| blob | e9b5173305b9b64ba700d5d94588738460f721e9 |
1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988-2013 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
9 any later version.
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
71 #ifndef CHECK_STACK_LIMIT
72 #define CHECK_STACK_LIMIT (-1)
73 #endif
75 /* Return index of given mode in mult and division cost tables. */
76 #define MODE_INDEX(mode) \
77 ((mode) == QImode ? 0 \
78 : (mode) == HImode ? 1 \
79 : (mode) == SImode ? 2 \
80 : (mode) == DImode ? 3 \
81 : 4)
83 /* Processor costs (relative to an add) */
84 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
85 #define COSTS_N_BYTES(N) ((N) * 2)
87 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall, false}}}
96 const
119 in QImode, HImode and SImode.
120 Relative to reg-reg move (2). */
124 in SFmode, DFmode and XFmode */
126 in SFmode, DFmode and XFmode */
129 in SImode and DImode */
131 in SImode and DImode */
134 in SImode, DImode and TImode */
136 in SImode, DImode and TImode */
149 ix86_size_memcpy,
150 ix86_size_memset,
162 };
164 /* Processor costs (relative to an add) */
167 DUMMY_STRINGOP_ALGS};
170 DUMMY_STRINGOP_ALGS};
172 static const
195 in QImode, HImode and SImode.
196 Relative to reg-reg move (2). */
200 in SFmode, DFmode and XFmode */
202 in SFmode, DFmode and XFmode */
205 in SImode and DImode */
207 in SImode and DImode */
210 in SImode, DImode and TImode */
212 in SImode, DImode and TImode */
225 i386_memcpy,
226 i386_memset,
238 };
242 DUMMY_STRINGOP_ALGS};
245 DUMMY_STRINGOP_ALGS};
247 static const
270 in QImode, HImode and SImode.
271 Relative to reg-reg move (2). */
275 in SFmode, DFmode and XFmode */
277 in SFmode, DFmode and XFmode */
280 in SImode and DImode */
282 in SImode and DImode */
285 in SImode, DImode and TImode */
287 in SImode, DImode and TImode */
290 shared for code and data, so 4kB is
291 not really precise. */
302 i486_memcpy,
303 i486_memset,
315 };
319 DUMMY_STRINGOP_ALGS};
322 DUMMY_STRINGOP_ALGS};
324 static const
347 in QImode, HImode and SImode.
348 Relative to reg-reg move (2). */
352 in SFmode, DFmode and XFmode */
354 in SFmode, DFmode and XFmode */
357 in SImode and DImode */
359 in SImode and DImode */
362 in SImode, DImode and TImode */
364 in SImode, DImode and TImode */
377 pentium_memcpy,
378 pentium_memset,
390 };
392 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
393 (we ensure the alignment). For small blocks inline loop is still a
394 noticeable win, for bigger blocks either rep movsl or rep movsb is
395 way to go. Rep movsb has apparently more expensive startup time in CPU,
396 but after 4K the difference is down in the noise. */
401 DUMMY_STRINGOP_ALGS};
406 DUMMY_STRINGOP_ALGS};
407 static const
430 in QImode, HImode and SImode.
431 Relative to reg-reg move (2). */
435 in SFmode, DFmode and XFmode */
437 in SFmode, DFmode and XFmode */
440 in SImode and DImode */
442 in SImode and DImode */
445 in SImode, DImode and TImode */
447 in SImode, DImode and TImode */
460 pentiumpro_memcpy,
461 pentiumpro_memset,
473 };
477 DUMMY_STRINGOP_ALGS};
480 DUMMY_STRINGOP_ALGS};
481 static const
504 in QImode, HImode and SImode.
505 Relative to reg-reg move (2). */
509 in SFmode, DFmode and XFmode */
511 in SFmode, DFmode and XFmode */
515 in SImode and DImode */
517 in SImode and DImode */
520 in SImode, DImode and TImode */
522 in SImode, DImode and TImode */
535 geode_memcpy,
536 geode_memset,
548 };
552 DUMMY_STRINGOP_ALGS};
555 DUMMY_STRINGOP_ALGS};
556 static const
579 in QImode, HImode and SImode.
580 Relative to reg-reg move (2). */
584 in SFmode, DFmode and XFmode */
586 in SFmode, DFmode and XFmode */
589 in SImode and DImode */
591 in SImode and DImode */
594 in SImode, DImode and TImode */
596 in SImode, DImode and TImode */
600 have integrated l2 cache, but
601 optimizing for k6 is not important
602 enough to worry about that. */
612 k6_memcpy,
613 k6_memset,
625 };
627 /* For some reason, Athlon deals better with REP prefix (relative to loops)
628 compared to K8. Alignment becomes important after 8 bytes for memcpy and
629 128 bytes for memset. */
632 DUMMY_STRINGOP_ALGS};
635 DUMMY_STRINGOP_ALGS};
636 static const
659 in QImode, HImode and SImode.
660 Relative to reg-reg move (2). */
664 in SFmode, DFmode and XFmode */
666 in SFmode, DFmode and XFmode */
669 in SImode and DImode */
671 in SImode and DImode */
674 in SImode, DImode and TImode */
676 in SImode, DImode and TImode */
689 athlon_memcpy,
690 athlon_memset,
702 };
704 /* K8 has optimized REP instruction for medium sized blocks, but for very
705 small blocks it is better to use loop. For large blocks, libcall can
706 do nontemporary accesses and beat inline considerably. */
717 static const
740 in QImode, HImode and SImode.
741 Relative to reg-reg move (2). */
745 in SFmode, DFmode and XFmode */
747 in SFmode, DFmode and XFmode */
750 in SImode and DImode */
752 in SImode and DImode */
755 in SImode, DImode and TImode */
757 in SImode, DImode and TImode */
762 /* New AMD processors never drop prefetches; if they cannot be performed
763 immediately, they are queued. We set number of simultaneous prefetches
764 to a large constant to reflect this (it probably is not a good idea not
765 to limit number of prefetches at all, as their execution also takes some
766 time). */
776 k8_memcpy,
777 k8_memset,
789 };
791 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
792 very small blocks it is better to use loop. For large blocks, libcall can
793 do nontemporary accesses and beat inline considerably. */
826 in QImode, HImode and SImode.
827 Relative to reg-reg move (2). */
831 in SFmode, DFmode and XFmode */
833 in SFmode, DFmode and XFmode */
836 in SImode and DImode */
838 in SImode and DImode */
841 in SImode, DImode and TImode */
843 in SImode, DImode and TImode */
845 /* On K8:
846 MOVD reg64, xmmreg Double FSTORE 4
847 MOVD reg32, xmmreg Double FSTORE 4
848 On AMDFAM10:
849 MOVD reg64, xmmreg Double FADD 3
850 1/1 1/1
851 MOVD reg32, xmmreg Double FADD 3
852 1/1 1/1 */
856 /* New AMD processors never drop prefetches; if they cannot be performed
857 immediately, they are queued. We set number of simultaneous prefetches
858 to a large constant to reflect this (it probably is not a good idea not
859 to limit number of prefetches at all, as their execution also takes some
860 time). */
870 amdfam10_memcpy,
871 amdfam10_memset,
883 };
885 /* BDVER1 has optimized REP instruction for medium sized blocks, but for
886 very small blocks it is better to use loop. For large blocks, libcall
887 can do nontemporary accesses and beat inline considerably. */
921 in QImode, HImode and SImode.
922 Relative to reg-reg move (2). */
926 in SFmode, DFmode and XFmode */
928 in SFmode, DFmode and XFmode */
931 in SImode and DImode */
933 in SImode and DImode */
936 in SImode, DImode and TImode */
938 in SImode, DImode and TImode */
940 /* On K8:
941 MOVD reg64, xmmreg Double FSTORE 4
942 MOVD reg32, xmmreg Double FSTORE 4
943 On AMDFAM10:
944 MOVD reg64, xmmreg Double FADD 3
945 1/1 1/1
946 MOVD reg32, xmmreg Double FADD 3
947 1/1 1/1 */
951 /* New AMD processors never drop prefetches; if they cannot be performed
952 immediately, they are queued. We set number of simultaneous prefetches
953 to a large constant to reflect this (it probably is not a good idea not
954 to limit number of prefetches at all, as their execution also takes some
955 time). */
965 bdver1_memcpy,
966 bdver1_memset,
978 };
980 /* BDVER2 has optimized REP instruction for medium sized blocks, but for
981 very small blocks it is better to use loop. For large blocks, libcall
982 can do nontemporary accesses and beat inline considerably. */
1017 in QImode, HImode and SImode.
1018 Relative to reg-reg move (2). */
1022 in SFmode, DFmode and XFmode */
1024 in SFmode, DFmode and XFmode */
1027 in SImode and DImode */
1029 in SImode and DImode */
1032 in SImode, DImode and TImode */
1034 in SImode, DImode and TImode */
1036 /* On K8:
1037 MOVD reg64, xmmreg Double FSTORE 4
1038 MOVD reg32, xmmreg Double FSTORE 4
1039 On AMDFAM10:
1040 MOVD reg64, xmmreg Double FADD 3
1041 1/1 1/1
1042 MOVD reg32, xmmreg Double FADD 3
1043 1/1 1/1 */
1047 /* New AMD processors never drop prefetches; if they cannot be performed
1048 immediately, they are queued. We set number of simultaneous prefetches
1049 to a large constant to reflect this (it probably is not a good idea not
1050 to limit number of prefetches at all, as their execution also takes some
1051 time). */
1061 bdver2_memcpy,
1062 bdver2_memset,
1074 };
1077 /* BDVER3 has optimized REP instruction for medium sized blocks, but for
1078 very small blocks it is better to use loop. For large blocks, libcall
1079 can do nontemporary accesses and beat inline considerably. */
1112 in QImode, HImode and SImode.
1113 Relative to reg-reg move (2). */
1117 in SFmode, DFmode and XFmode */
1119 in SFmode, DFmode and XFmode */
1122 in SImode and DImode */
1124 in SImode and DImode */
1127 in SImode, DImode and TImode */
1129 in SImode, DImode and TImode */
1134 /* New AMD processors never drop prefetches; if they cannot be performed
1135 immediately, they are queued. We set number of simultaneous prefetches
1136 to a large constant to reflect this (it probably is not a good idea not
1137 to limit number of prefetches at all, as their execution also takes some
1138 time). */
1148 bdver3_memcpy,
1149 bdver3_memset,
1161 };
1163 /* BTVER1 has optimized REP instruction for medium sized blocks, but for
1164 very small blocks it is better to use loop. For large blocks, libcall can
1165 do nontemporary accesses and beat inline considerably. */
1198 in QImode, HImode and SImode.
1199 Relative to reg-reg move (2). */
1203 in SFmode, DFmode and XFmode */
1205 in SFmode, DFmode and XFmode */
1208 in SImode and DImode */
1210 in SImode and DImode */
1213 in SImode, DImode and TImode */
1215 in SImode, DImode and TImode */
1217 /* On K8:
1218 MOVD reg64, xmmreg Double FSTORE 4
1219 MOVD reg32, xmmreg Double FSTORE 4
1220 On AMDFAM10:
1221 MOVD reg64, xmmreg Double FADD 3
1222 1/1 1/1
1223 MOVD reg32, xmmreg Double FADD 3
1224 1/1 1/1 */
1237 btver1_memcpy,
1238 btver1_memset,
1250 };
1284 in QImode, HImode and SImode.
1285 Relative to reg-reg move (2). */
1289 in SFmode, DFmode and XFmode */
1291 in SFmode, DFmode and XFmode */
1294 in SImode and DImode */
1296 in SImode and DImode */
1299 in SImode, DImode and TImode */
1301 in SImode, DImode and TImode */
1303 /* On K8:
1304 MOVD reg64, xmmreg Double FSTORE 4
1305 MOVD reg32, xmmreg Double FSTORE 4
1306 On AMDFAM10:
1307 MOVD reg64, xmmreg Double FADD 3
1308 1/1 1/1
1309 MOVD reg32, xmmreg Double FADD 3
1310 1/1 1/1 */
1322 btver2_memcpy,
1323 btver2_memset,
1335 };
1339 DUMMY_STRINGOP_ALGS};
1343 DUMMY_STRINGOP_ALGS};
1345 static const
1368 in QImode, HImode and SImode.
1369 Relative to reg-reg move (2). */
1373 in SFmode, DFmode and XFmode */
1375 in SFmode, DFmode and XFmode */
1378 in SImode and DImode */
1380 in SImode and DImode */
1383 in SImode, DImode and TImode */
1385 in SImode, DImode and TImode */
1398 pentium4_memcpy,
1399 pentium4_memset,
1411 };
1424 static const
1447 in QImode, HImode and SImode.
1448 Relative to reg-reg move (2). */
1452 in SFmode, DFmode and XFmode */
1454 in SFmode, DFmode and XFmode */
1457 in SImode and DImode */
1459 in SImode and DImode */
1462 in SImode, DImode and TImode */
1464 in SImode, DImode and TImode */
1477 nocona_memcpy,
1478 nocona_memset,
1490 };
1501 static const
1524 in QImode, HImode and SImode.
1525 Relative to reg-reg move (2). */
1529 in SFmode, DFmode and XFmode */
1531 in SFmode, DFmode and XFmode */
1534 in SImode and DImode */
1536 in SImode and DImode */
1539 in SImode, DImode and TImode */
1541 in SImode, DImode and TImode */
1554 atom_memcpy,
1555 atom_memset,
1567 };
1578 static const
1601 in QImode, HImode and SImode.
1602 Relative to reg-reg move (2). */
1606 in SFmode, DFmode and XFmode */
1608 in SFmode, DFmode and XFmode */
1611 in SImode and DImode */
1613 in SImode and DImode */
1616 in SImode, DImode and TImode */
1618 in SImode, DImode and TImode */
1631 slm_memcpy,
1632 slm_memset,
1644 };
1646 /* Generic should produce code tuned for Core-i7 (and newer chips)
1647 and btver1 (and newer chips). */
1659 static const
1662 /* On all chips taken into consideration lea is 2 cycles and more. With
1663 this cost however our current implementation of synth_mult results in
1664 use of unnecessary temporary registers causing regression on several
1665 SPECfp benchmarks. */
1686 in QImode, HImode and SImode.
1687 Relative to reg-reg move (2). */
1691 in SFmode, DFmode and XFmode */
1693 in SFmode, DFmode and XFmode */
1696 in SImode and DImode */
1698 in SImode and DImode */
1701 in SImode, DImode and TImode */
1703 in SImode, DImode and TImode */
1709 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
1710 value is increased to perhaps more appropriate value of 5. */
1718 generic_memcpy,
1719 generic_memset,
1731 };
1733 /* core_cost should produce code tuned for Core familly of CPUs. */
1746 static const
1749 /* On all chips taken into consideration lea is 2 cycles and more. With
1750 this cost however our current implementation of synth_mult results in
1751 use of unnecessary temporary registers causing regression on several
1752 SPECfp benchmarks. */
1773 in QImode, HImode and SImode.
1774 Relative to reg-reg move (2). */
1778 in SFmode, DFmode and XFmode */
1780 in SFmode, DFmode and XFmode */
1783 in SImode and DImode */
1785 in SImode and DImode */
1788 in SImode, DImode and TImode */
1790 in SImode, DImode and TImode */
1796 /* FIXME perhaps more appropriate value is 5. */
1804 core_memcpy,
1805 core_memset,
1817 };
1820 /* Set by -mtune. */
1823 /* Set by -mtune or -Os. */
1826 /* Processor feature/optimization bitmasks. */
1827 #define m_386 (1<<PROCESSOR_I386)
1828 #define m_486 (1<<PROCESSOR_I486)
1829 #define m_PENT (1<<PROCESSOR_PENTIUM)
1830 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
1831 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
1832 #define m_NOCONA (1<<PROCESSOR_NOCONA)
1833 #define m_P4_NOCONA (m_PENT4 | m_NOCONA)
1834 #define m_CORE2 (1<<PROCESSOR_CORE2)
1835 #define m_COREI7 (1<<PROCESSOR_COREI7)
1836 #define m_HASWELL (1<<PROCESSOR_HASWELL)
1837 #define m_CORE_ALL (m_CORE2 | m_COREI7 | m_HASWELL)
1838 #define m_ATOM (1<<PROCESSOR_ATOM)
1839 #define m_SLM (1<<PROCESSOR_SLM)
1841 #define m_GEODE (1<<PROCESSOR_GEODE)
1842 #define m_K6 (1<<PROCESSOR_K6)
1843 #define m_K6_GEODE (m_K6 | m_GEODE)
1844 #define m_K8 (1<<PROCESSOR_K8)
1845 #define m_ATHLON (1<<PROCESSOR_ATHLON)
1846 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
1847 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
1848 #define m_BDVER1 (1<<PROCESSOR_BDVER1)
1849 #define m_BDVER2 (1<<PROCESSOR_BDVER2)
1850 #define m_BDVER3 (1<<PROCESSOR_BDVER3)
1851 #define m_BTVER1 (1<<PROCESSOR_BTVER1)
1852 #define m_BTVER2 (1<<PROCESSOR_BTVER2)
1853 #define m_BDVER (m_BDVER1 | m_BDVER2 | m_BDVER3)
1854 #define m_BTVER (m_BTVER1 | m_BTVER2)
1855 #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER)
1857 #define m_GENERIC (1<<PROCESSOR_GENERIC)
1860 #undef DEF_TUNE
1861 #define DEF_TUNE(tune, name, selector) name,
1863 #undef DEF_TUNE
1864 };
1866 /* Feature tests against the various tunings. */
1869 /* Feature tests against the various tunings used to create ix86_tune_features
1870 based on the processor mask. */
1872 #undef DEF_TUNE
1873 #define DEF_TUNE(tune, name, selector) selector,
1875 #undef DEF_TUNE
1876 };
1878 /* Feature tests against the various architecture variations. */
1881 /* Feature tests against the various architecture variations, used to create
1882 ix86_arch_features based on the processor mask. */
1884 /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */
1887 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
1890 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
1893 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
1896 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
1898 };
1900 /* In case the average insn count for single function invocation is
1901 lower than this constant, emit fast (but longer) prologue and
1902 epilogue code. */
1903 #define FAST_PROLOGUE_INSN_COUNT 20
1905 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
1910 /* Array of the smallest class containing reg number REGNO, indexed by
1911 REGNO. Used by REGNO_REG_CLASS in i386.h. */
1914 {
1915 /* ax, dx, cx, bx */
1917 /* si, di, bp, sp */
1919 /* FP registers */
1922 /* arg pointer */
1923 NON_Q_REGS,
1924 /* flags, fpsr, fpcr, frame */
1926 /* SSE registers */
1929 /* MMX registers */
1932 /* REX registers */
1935 /* SSE REX registers */
1938 /* AVX-512 SSE registers */
1943 /* Mask registers. */
1946 /* MPX bound registers */
1948 };
1950 /* The "default" register map used in 32bit mode. */
1953 {
1965 };
1967 /* The "default" register map used in 64bit mode. */
1970 {
1982 };
1984 /* Define the register numbers to be used in Dwarf debugging information.
1985 The SVR4 reference port C compiler uses the following register numbers
1986 in its Dwarf output code:
1987 0 for %eax (gcc regno = 0)
1988 1 for %ecx (gcc regno = 2)
1989 2 for %edx (gcc regno = 1)
1990 3 for %ebx (gcc regno = 3)
1991 4 for %esp (gcc regno = 7)
1992 5 for %ebp (gcc regno = 6)
1993 6 for %esi (gcc regno = 4)
1994 7 for %edi (gcc regno = 5)
1995 The following three DWARF register numbers are never generated by
1996 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
1997 believes these numbers have these meanings.
1998 8 for %eip (no gcc equivalent)
1999 9 for %eflags (gcc regno = 17)
2000 10 for %trapno (no gcc equivalent)
2001 It is not at all clear how we should number the FP stack registers
2002 for the x86 architecture. If the version of SDB on x86/svr4 were
2003 a bit less brain dead with respect to floating-point then we would
2004 have a precedent to follow with respect to DWARF register numbers
2005 for x86 FP registers, but the SDB on x86/svr4 is so completely
2006 broken with respect to FP registers that it is hardly worth thinking
2007 of it as something to strive for compatibility with.
2008 The version of x86/svr4 SDB I have at the moment does (partially)
2009 seem to believe that DWARF register number 11 is associated with
2010 the x86 register %st(0), but that's about all. Higher DWARF
2011 register numbers don't seem to be associated with anything in
2012 particular, and even for DWARF regno 11, SDB only seems to under-
2013 stand that it should say that a variable lives in %st(0) (when
2014 asked via an `=' command) if we said it was in DWARF regno 11,
2015 but SDB still prints garbage when asked for the value of the
2016 variable in question (via a `/' command).
2017 (Also note that the labels SDB prints for various FP stack regs
2018 when doing an `x' command are all wrong.)
2019 Note that these problems generally don't affect the native SVR4
2020 C compiler because it doesn't allow the use of -O with -g and
2021 because when it is *not* optimizing, it allocates a memory
2022 location for each floating-point variable, and the memory
2023 location is what gets described in the DWARF AT_location
2024 attribute for the variable in question.
2025 Regardless of the severe mental illness of the x86/svr4 SDB, we
2026 do something sensible here and we use the following DWARF
2027 register numbers. Note that these are all stack-top-relative
2028 numbers.
2029 11 for %st(0) (gcc regno = 8)
2030 12 for %st(1) (gcc regno = 9)
2031 13 for %st(2) (gcc regno = 10)
2032 14 for %st(3) (gcc regno = 11)
2033 15 for %st(4) (gcc regno = 12)
2034 16 for %st(5) (gcc regno = 13)
2035 17 for %st(6) (gcc regno = 14)
2036 18 for %st(7) (gcc regno = 15)
2037 */
2039 {
2051 };
2053 /* Define parameter passing and return registers. */
2056 {
2058 };
2061 {
2063 };
2066 {
2068 };
2070 /* Additional registers that are clobbered by SYSV calls. */
2073 {
2078 };
2080 /* Define the structure for the machine field in struct function. */
2085 rtx rtl;
2087 };
2089 /* Structure describing stack frame layout.
2090 Stack grows downward:
2092 [arguments]
2093 <- ARG_POINTER
2094 saved pc
2096 saved static chain if ix86_static_chain_on_stack
2098 saved frame pointer if frame_pointer_needed
2099 <- HARD_FRAME_POINTER
2100 [saved regs]
2101 <- regs_save_offset
2102 [padding0]
2104 [saved SSE regs]
2105 <- sse_regs_save_offset
2106 [padding1] |
2107 | <- FRAME_POINTER
2108 [va_arg registers] |
2109 |
2110 [frame] |
2111 |
2112 [padding2] | = to_allocate
2113 <- STACK_POINTER
2114 */
2115 struct ix86_frame
2116 {
2123 /* The offsets relative to ARG_POINTER. */
2124 HOST_WIDE_INT frame_pointer_offset;
2125 HOST_WIDE_INT hard_frame_pointer_offset;
2126 HOST_WIDE_INT stack_pointer_offset;
2127 HOST_WIDE_INT hfp_save_offset;
2128 HOST_WIDE_INT reg_save_offset;
2129 HOST_WIDE_INT sse_reg_save_offset;
2131 /* When save_regs_using_mov is set, emit prologue using
2132 move instead of push instructions. */
2134 };
2136 /* Which cpu are we scheduling for. */
2139 /* Which cpu are we optimizing for. */
2142 /* Which instruction set architecture to use. */
2145 /* True if processor has SSE prefetch instruction. */
2148 /* -mstackrealign option */
2165 /* Preferred alignment for stack boundary in bits. */
2168 /* Alignment for incoming stack boundary in bits specified at
2169 command line. */
2172 /* Default alignment for incoming stack boundary in bits. */
2175 /* Alignment for incoming stack boundary in bits. */
2178 /* Calling abi specific va_list type nodes. */
2182 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
2186 /* Fence to use after loop using movnt. */
2187 tree x86_mfence;
2189 /* Register class used for passing given 64bit part of the argument.
2190 These represent classes as documented by the PS ABI, with the exception
2191 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
2192 use SF or DFmode move instead of DImode to avoid reformatting penalties.
2194 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
2195 whenever possible (upper half does contain padding). */
2196 enum x86_64_reg_class
2197 {
2198 X86_64_NO_CLASS,
2199 X86_64_INTEGER_CLASS,
2200 X86_64_INTEGERSI_CLASS,
2201 X86_64_SSE_CLASS,
2202 X86_64_SSESF_CLASS,
2203 X86_64_SSEDF_CLASS,
2204 X86_64_SSEUP_CLASS,
2205 X86_64_X87_CLASS,
2206 X86_64_X87UP_CLASS,
2207 X86_64_COMPLEX_X87_CLASS,
2208 X86_64_MEMORY_CLASS
2209 };
2211 #define MAX_CLASSES 4
2213 /* Table of constants used by fldpi, fldln2, etc.... */
2217 \f
2222 const_tree);
2234 enum ix86_function_specific_strings
2235 {
2236 IX86_FUNCTION_SPECIFIC_ARCH,
2237 IX86_FUNCTION_SPECIFIC_TUNE,
2238 IX86_FUNCTION_SPECIFIC_MAX
2239 };
2260 \f
2261 #ifndef SUBTARGET32_DEFAULT_CPU
2263 #endif
2265 /* Whether -mtune= or -march= were specified */
2269 /* Vectorization library interface and handlers. */
2275 /* Processor target table, indexed by processor number */
2276 struct ptt
2277 {
2284 };
2287 {
2298 /* Core 2 */
2300 /* Core i7 */
2302 /* Core avx2 */
2313 };
2316 {
2346 "btver2"
2347 };
2348 \f
2349 static bool
2351 {
2353 }
2355 static unsigned int
2357 {
2360 /* vzeroupper instructions are inserted immediately after reload to
2361 account for possible spills from 256bit registers. The pass
2362 reuses mode switching infrastructure by re-running mode insertion
2363 pass, so disable entities that have already been processed. */
2369 /* Call optimize_mode_switching. */
2372 }
2377 {
2389 };
2392 {
2396 {}
2398 /* opt_pass methods: */
2406 rtl_opt_pass *
2408 {
2410 }
2412 /* Return true if a red-zone is in use. */
2416 {
2418 }
2419 \f
2420 /* Return a string that documents the current -m options. The caller is
2421 responsible for freeing the string. */
2427 {
2428 struct ix86_target_opts
2429 {
2432 };
2434 /* This table is ordered so that options like -msse4.2 that imply
2435 preceding options while match those first. */
2437 {
2478 };
2480 /* Flag options. */
2482 {
2510 };
2527 /* Add -march= option. */
2529 {
2532 }
2534 /* Add -mtune= option. */
2536 {
2539 }
2541 /* Add -m32/-m64/-mx32. */
2543 {
2546 else
2548 isa &= ~ (OPTION_MASK_ISA_64BIT
2549 | OPTION_MASK_ABI_64
2551 }
2552 else
2556 /* Pick out the options in isa options. */
2558 {
2560 {
2563 }
2564 }
2567 {
2570 isa);
2571 }
2573 /* Add flag options. */
2575 {
2577 {
2580 }
2581 }
2584 {
2587 }
2589 /* Add -fpmath= option. */
2591 {
2594 {
2609 }
2610 }
2612 /* Any options? */
2618 /* Size the string. */
2622 {
2627 }
2629 /* Build the string. */
2634 {
2641 {
2643 line_len++;
2646 {
2650 }
2651 }
2655 {
2659 }
2660 }
2666 }
2668 /* Return true, if profiling code should be emitted before
2669 prologue. Otherwise it returns false.
2670 Note: For x86 with "hotfix" it is sorried. */
2671 static bool
2673 {
2675 }
2677 /* Function that is callable from the debugger to print the current
2678 options. */
2679 void ATTRIBUTE_UNUSED
2681 {
2687 {
2690 }
2691 else
2695 }
2698 #define DEF_ENUM
2699 #define DEF_ALG(alg, name) #name,
2701 #undef DEF_ENUM
2702 #undef DEF_ALG
2703 };
2705 /* Parse parameter string passed to -mmemcpy-strategy= or -mmemset-strategy=.
2706 The string is of the following form (or comma separated list of it):
2708 strategy_alg:max_size:[align|noalign]
2710 where the full size range for the strategy is either [0, max_size] or
2711 [min_size, max_size], in which min_size is the max_size + 1 of the
2712 preceding range. The last size range must have max_size == -1.
2714 Examples:
2716 1.
2717 -mmemcpy-strategy=libcall:-1:noalign
2719 this is equivalent to (for known size memcpy) -mstringop-strategy=libcall
2722 2.
2723 -mmemset-strategy=rep_8byte:16:noalign,vector_loop:2048:align,libcall:-1:noalign
2725 This is to tell the compiler to use the following strategy for memset
2726 1) when the expected size is between [1, 16], use rep_8byte strategy;
2727 2) when the size is between [17, 2048], use vector_loop;
2728 3) when the size is > 2048, use libcall. */
2730 struct stringop_size_range
2731 {
2733 stringop_alg alg;
2735 };
2737 static void
2739 {
2747 else
2752 do
2753 {
2755 stringop_alg alg;
2764 {
2768 }
2771 {
2775 }
2778 {
2780 {
2783 }
2784 }
2787 {
2789 alg_name,
2792 }
2800 else
2801 {
2805 }
2806 n++;
2808 }
2812 {
2817 }
2820 {
2824 }
2826 /* Now override the default algs array. */
2828 {
2834 }
2835 }
2837 \f
2838 /* parse -mtune-ctrl= option. When DUMP is true,
2839 print the features that are explicitly set. */
2841 static void
2843 {
2851 do
2852 {
2859 {
2860 curr_feature_string++;
2862 }
2864 {
2866 {
2872 }
2873 }
2878 }
2881 }
2883 /* Helper function to set ix86_tune_features. IX86_TUNE is the
2884 processor type. */
2886 static void
2888 {
2893 {
2896 else
2898 }
2901 {
2906 }
2909 }
2912 /* Override various settings based on options. If MAIN_ARGS_P, the
2913 options are from the command line, otherwise they are from
2914 attributes. */
2916 static void
2920 {
2928 #define PTA_3DNOW (HOST_WIDE_INT_1 << 0)
2929 #define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1)
2930 #define PTA_64BIT (HOST_WIDE_INT_1 << 2)
2931 #define PTA_ABM (HOST_WIDE_INT_1 << 3)
2932 #define PTA_AES (HOST_WIDE_INT_1 << 4)
2933 #define PTA_AVX (HOST_WIDE_INT_1 << 5)
2934 #define PTA_BMI (HOST_WIDE_INT_1 << 6)
2935 #define PTA_CX16 (HOST_WIDE_INT_1 << 7)
2936 #define PTA_F16C (HOST_WIDE_INT_1 << 8)
2937 #define PTA_FMA (HOST_WIDE_INT_1 << 9)
2938 #define PTA_FMA4 (HOST_WIDE_INT_1 << 10)
2939 #define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11)
2940 #define PTA_LWP (HOST_WIDE_INT_1 << 12)
2941 #define PTA_LZCNT (HOST_WIDE_INT_1 << 13)
2942 #define PTA_MMX (HOST_WIDE_INT_1 << 14)
2943 #define PTA_MOVBE (HOST_WIDE_INT_1 << 15)
2944 #define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16)
2945 #define PTA_PCLMUL (HOST_WIDE_INT_1 << 17)
2946 #define PTA_POPCNT (HOST_WIDE_INT_1 << 18)
2947 #define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19)
2948 #define PTA_RDRND (HOST_WIDE_INT_1 << 20)
2949 #define PTA_SSE (HOST_WIDE_INT_1 << 21)
2950 #define PTA_SSE2 (HOST_WIDE_INT_1 << 22)
2951 #define PTA_SSE3 (HOST_WIDE_INT_1 << 23)
2952 #define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24)
2953 #define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25)
2954 #define PTA_SSE4A (HOST_WIDE_INT_1 << 26)
2955 #define PTA_SSSE3 (HOST_WIDE_INT_1 << 27)
2956 #define PTA_TBM (HOST_WIDE_INT_1 << 28)
2957 #define PTA_XOP (HOST_WIDE_INT_1 << 29)
2958 #define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
2959 #define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
2960 #define PTA_RTM (HOST_WIDE_INT_1 << 32)
2961 #define PTA_HLE (HOST_WIDE_INT_1 << 33)
2962 #define PTA_PRFCHW (HOST_WIDE_INT_1 << 34)
2963 #define PTA_RDSEED (HOST_WIDE_INT_1 << 35)
2964 #define PTA_ADX (HOST_WIDE_INT_1 << 36)
2965 #define PTA_FXSR (HOST_WIDE_INT_1 << 37)
2966 #define PTA_XSAVE (HOST_WIDE_INT_1 << 38)
2967 #define PTA_XSAVEOPT (HOST_WIDE_INT_1 << 39)
2968 #define PTA_AVX512F (HOST_WIDE_INT_1 << 40)
2969 #define PTA_AVX512ER (HOST_WIDE_INT_1 << 41)
2970 #define PTA_AVX512PF (HOST_WIDE_INT_1 << 42)
2971 #define PTA_AVX512CD (HOST_WIDE_INT_1 << 43)
2972 #define PTA_MPX (HOST_WIDE_INT_1 << 44)
2974 /* if this reaches 64, need to widen struct pta flags below */
2976 static struct pta
2977 {
2982 }
2984 {
3117 PTA_64BIT
3119 };
3121 /* -mrecip options. */
3122 static struct
3123 {
3126 }
3128 {
3135 };
3139 /* Set up prefix/suffix so the error messages refer to either the command
3140 line argument, or the attribute(target). */
3142 {
3146 }
3147 else
3148 {
3152 }
3154 /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
3155 TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
3158 #ifdef TARGET_BI_ARCH
3159 else
3160 {
3161 #if TARGET_BI_ARCH == 1
3162 /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
3163 is on and OPTION_MASK_ABI_X32 is off. We turn off
3164 OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
3165 -mx32. */
3168 #else
3169 /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
3170 on and OPTION_MASK_ABI_64 is off. We turn off
3171 OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
3172 -m64. */
3175 #endif
3176 }
3177 #endif
3180 {
3181 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3182 OPTION_MASK_ABI_64 for TARGET_X32. */
3185 }
3187 {
3188 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3189 OPTION_MASK_ABI_X32 for TARGET_LP64. */
3192 }
3194 #ifdef SUBTARGET_OVERRIDE_OPTIONS
3195 SUBTARGET_OVERRIDE_OPTIONS;
3196 #endif
3198 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
3199 SUBSUBTARGET_OVERRIDE_OPTIONS;
3200 #endif
3202 /* -fPIC is the default for x86_64. */
3206 /* Need to check -mtune=generic first. */
3208 {
3211 /* As special support for cross compilers we read -mtune=native
3212 as -mtune=generic. With native compilers we won't see the
3213 -mtune=native, as it was changed by the driver. */
3215 {
3217 }
3218 /* If this call is for setting the option attribute, allow the
3219 generic that was previously set. */
3222 ;
3230 }
3231 else
3232 {
3236 {
3239 }
3241 /* opts->x_ix86_tune_string is set to opts->x_ix86_arch_string
3242 or defaulted. We need to use a sensible tune option. */
3246 {
3248 }
3249 }
3253 {
3254 /* rep; movq isn't available in 32-bit code. */
3257 }
3260 opts->x_ix86_arch_string
3263 else
3267 {
3275 }
3276 else
3283 /* For targets using ms ABI enable ms-extensions, if not
3284 explicit turned off. For non-ms ABI we turn off this
3285 option. */
3290 {
3292 {
3336 {
3339 }
3347 }
3348 }
3349 else
3350 {
3351 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
3352 use of rip-relative addressing. This eliminates fixups that
3353 would otherwise be needed if this object is to be placed in a
3354 DLL, and is essentially just as efficient as direct addressing. */
3360 else
3362 }
3364 {
3367 }
3375 {
3378 /* Default cpu tuning to the architecture. */
3517 }
3532 {
3536 {
3538 {
3540 {
3548 }
3549 else
3552 }
3553 }
3554 /* Intel CPUs have always interpreted SSE prefetch instructions as
3555 NOPs; so, we can enable SSE prefetch instructions even when
3556 -mtune (rather than -march) points us to a processor that has them.
3557 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
3558 higher processors. */
3563 }
3571 #ifndef USE_IX86_FRAME_POINTER
3572 #define USE_IX86_FRAME_POINTER 0
3573 #endif
3575 #ifndef USE_X86_64_FRAME_POINTER
3576 #define USE_X86_64_FRAME_POINTER 0
3577 #endif
3579 /* Set the default values for switches whose default depends on TARGET_64BIT
3580 in case they weren't overwritten by command line options. */
3582 {
3586 opts->x_flag_unwind_tables
3590 }
3591 else
3592 {
3594 opts->x_flag_omit_frame_pointer
3600 }
3605 else
3608 /* Arrange to set up i386_stack_locals for all functions. */
3611 /* Validate -mregparm= value. */
3613 {
3617 {
3621 }
3622 }
3626 /* Default align_* from the processor table. */
3628 {
3631 }
3633 {
3636 }
3638 {
3640 }
3642 /* Provide default for -mbranch-cost= value. */
3647 {
3648 opts->x_target_flags
3651 /* Enable by default the SSE and MMX builtins. Do allow the user to
3652 explicitly disable any of these. In particular, disabling SSE and
3653 MMX for kernel code is extremely useful. */
3655 opts->x_ix86_isa_flags
3662 }
3663 else
3664 {
3665 opts->x_target_flags
3669 opts->x_ix86_isa_flags
3672 /* i386 ABI does not specify red zone. It still makes sense to use it
3673 when programmer takes care to stack from being destroyed. */
3676 }
3678 /* Keep nonleaf frame pointers. */
3684 /* If we're doing fast math, we don't care about comparison order
3685 wrt NaNs. This lets us use a shorter comparison sequence. */
3689 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3690 since the insns won't need emulation. */
3694 /* Likewise, if the target doesn't have a 387, or we've specified
3695 software floating point, don't use 387 inline intrinsics. */
3699 /* Turn on MMX builtins for -msse. */
3701 opts->x_ix86_isa_flags
3704 /* Enable SSE prefetch. */
3709 /* Enable prefetch{,w} instructions for -m3dnow. */
3711 opts->x_ix86_isa_flags
3714 /* Enable popcnt instruction for -msse4.2 or -mabm. */
3717 opts->x_ix86_isa_flags
3720 /* Enable lzcnt instruction for -mabm. */
3722 opts->x_ix86_isa_flags
3725 /* Validate -mpreferred-stack-boundary= value or default it to
3726 PREFERRED_STACK_BOUNDARY_DEFAULT. */
3729 {
3736 {
3740 else
3743 }
3744 else
3745 ix86_preferred_stack_boundary
3747 }
3749 /* Set the default value for -mstackrealign. */
3755 /* Validate -mincoming-stack-boundary= value or default it to
3756 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
3759 {
3764 ix86_incoming_stack_boundary_arg,
3766 else
3767 {
3768 ix86_user_incoming_stack_boundary
3770 ix86_incoming_stack_boundary
3772 }
3773 }
3775 /* Accept -msseregparm only if at least SSE support is enabled. */
3781 {
3783 {
3785 {
3788 }
3791 {
3794 }
3795 }
3796 }
3797 /* For all chips supporting SSE2, -mfpmath=sse performs better than
3798 fpmath=387. The second is however default at many targets since the
3799 extra 80bit precision of temporaries is considered to be part of ABI.
3800 Overwrite the default at least for -ffast-math.
3801 TODO: -mfpmath=both seems to produce same performing code with bit
3802 smaller binaries. It is however not clear if register allocation is
3803 ready for this setting.
3804 Also -mfpmath=387 is overall a lot more compact (bout 4-5%) than SSE
3805 codegen. We may switch to 387 with -ffast-math for size optimized
3806 functions. */
3810 else
3813 /* If the i387 is disabled, then do not return values in it. */
3817 /* Use external vectorized library in vectorizing intrinsics. */
3820 {
3831 }
3838 /* If stack probes are required, the space used for large function
3839 arguments on the stack must also be probed, so enable
3840 -maccumulate-outgoing-args so this happens in the prologue. */
3843 {
3848 }
3850 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
3851 {
3857 }
3859 /* When scheduling description is not available, disable scheduler pass
3860 so it won't slow down the compilation and make x87 code slower. */
3881 /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
3883 && HAVE_prefetch
3888 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
3889 can be opts->x_optimized to ap = __builtin_next_arg (0). */
3894 {
3897 {
3899 ix86_gen_tls_local_dynamic_base_64
3901 }
3902 else
3903 {
3905 ix86_gen_tls_local_dynamic_base_64
3907 }
3908 }
3909 else
3913 {
3923 }
3924 else
3925 {
3935 }
3937 #ifdef USE_IX86_CLD
3938 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
3941 #endif
3944 {
3949 }
3951 {
3955 }
3957 {
3958 #if defined(PROFILE_BEFORE_PROLOGUE)
3960 #else
3962 #endif
3963 }
3965 /* When not opts->x_optimize for size, enable vzeroupper optimization for
3966 TARGET_AVX with -fexpensive-optimizations and split 32-byte
3967 AVX unaligned load/store. */
3969 {
3979 /* Enable 128-bit AVX instruction generation
3980 for the auto-vectorizer. */
3984 }
3987 {
3994 {
3997 {
3999 q++;
4000 }
4001 else
4006 else
4007 {
4010 {
4013 }
4016 {
4020 }
4021 }
4026 else
4028 }
4029 }
4036 /* Default long double to 64-bit for Bionic. */
4041 /* Save the initial options in case the user does function specific
4042 options. */
4044 target_option_default_node = target_option_current_node
4047 /* Handle stack protector */
4049 opts->x_ix86_stack_protector_guard
4052 /* Handle -mmemcpy-strategy= and -mmemset-strategy= */
4054 {
4058 }
4061 {
4065 }
4066 }
4068 /* Implement the TARGET_OPTION_OVERRIDE hook. */
4070 static void
4072 {
4074 static struct register_pass_info insert_vzeroupper_info
4077 };
4082 /* This needs to be done at start up. It's convenient to do it here. */
4084 }
4086 /* Update register usage after having seen the compiler flags. */
4088 static void
4090 {
4094 /* The PIC register, if it exists, is fixed. */
4099 /* For 32-bit targets, squash the REX registers. */
4101 {
4108 }
4110 /* See the definition of CALL_USED_REGISTERS in i386.h. */
4118 {
4119 /* Set/reset conditionally defined registers from
4120 CALL_USED_REGISTERS initializer. */
4124 /* Calculate registers of CLOBBERED_REGS register set
4125 as call used registers from GENERAL_REGS register set. */
4129 }
4131 /* If MMX is disabled, squash the registers. */
4137 /* If SSE is disabled, squash the registers. */
4143 /* If the FPU is disabled, squash the registers. */
4149 /* If AVX512F is disabled, squash the registers. */
4151 {
4157 }
4159 /* If MPX is disabled, squash the registers. */
4163 }
4165 \f
4166 /* Save the current options */
4168 static void
4171 {
4182 /* The fields are char but the variables are not; make sure the
4183 values fit in the fields. */
4188 }
4190 /* Restore the current options */
4192 static void
4195 {
4211 /* Recreate the arch feature tests if the arch changed */
4213 {
4218 }
4220 /* Recreate the tune optimization tests */
4223 }
4225 /* Print the current options */
4227 static void
4230 {
4252 {
4255 }
4256 }
4258 \f
4259 /* Inner function to process the attribute((target(...))), take an argument and
4260 set the current options from the argument. If we have a list, recursively go
4261 over the list. */
4263 static bool
4268 {
4272 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
4273 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
4274 #define IX86_ATTR_ENUM(S,O) { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
4275 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
4276 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
4278 enum ix86_opt_type
4279 {
4280 ix86_opt_unknown,
4281 ix86_opt_yes,
4282 ix86_opt_no,
4283 ix86_opt_str,
4284 ix86_opt_enum,
4285 ix86_opt_isa
4286 };
4288 static const struct
4289 {
4296 /* isa options */
4337 /* enum options */
4340 /* string options */
4344 /* flag options */
4346 OPT_mcld,
4347 MASK_CLD),
4350 OPT_mfancy_math_387,
4351 MASK_NO_FANCY_MATH_387),
4354 OPT_mieee_fp,
4355 MASK_IEEE_FP),
4358 OPT_minline_all_stringops,
4359 MASK_INLINE_ALL_STRINGOPS),
4362 OPT_minline_stringops_dynamically,
4363 MASK_INLINE_STRINGOPS_DYNAMICALLY),
4366 OPT_mno_align_stringops,
4367 MASK_NO_ALIGN_STRINGOPS),
4370 OPT_mrecip,
4371 MASK_RECIP),
4373 };
4375 /* If this is a list, recurse to get the options. */
4377 {
4384 enum_opts_set))
4388 }
4391 {
4394 }
4396 /* Handle multiple arguments separated by commas. */
4400 {
4414 {
4418 }
4419 else
4420 {
4423 }
4425 /* Recognize no-xxx. */
4427 {
4431 }
4432 else
4435 /* Find the option. */
4439 {
4447 {
4452 }
4453 }
4455 /* Process the option. */
4457 {
4460 }
4463 {
4469 }
4472 {
4478 else
4480 }
4483 {
4485 {
4488 }
4489 else
4491 }
4494 {
4502 global_dc);
4503 else
4504 {
4507 }
4508 }
4510 else
4512 }
4515 }
4517 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
4519 tree
4523 {
4538 /* Process each of the options on the chain. */
4543 /* If the changed options are different from the default, rerun
4544 ix86_option_override_internal, and then save the options away.
4545 The string options are are attribute options, and will be undone
4546 when we copy the save structure. */
4552 {
4553 /* If we are using the default tune= or arch=, undo the string assigned,
4554 and use the default. */
4565 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
4570 {
4573 }
4575 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
4578 /* Add any builtin functions with the new isa if any. */
4581 /* Save the current options unless we are validating options for
4582 #pragma. */
4589 /* Free up memory allocated to hold the strings */
4592 }
4595 }
4597 /* Hook to validate attribute((target("string"))). */
4599 static bool
4602 tree args,
4604 {
4609 /* attribute((target("default"))) does nothing, beyond
4610 affecting multi-versioning. */
4619 /* Get the optimization options of the current function. */
4625 /* Init func_options. */
4633 /* Initialize func_options to the default before its target options can
4634 be set. */
4647 {
4652 }
4655 }
4657 \f
4658 /* Hook to determine if one function can safely inline another. */
4660 static bool
4662 {
4667 /* If callee has no option attributes, then it is ok to inline. */
4671 /* If caller has no option attributes, but callee does then it is not ok to
4672 inline. */
4676 else
4677 {
4681 /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
4682 can inline a SSE2 function but a SSE2 function can't inline a SSE4
4683 function. */
4688 /* See if we have the same non-isa options. */
4692 /* See if arch, tune, etc. are the same. */
4705 else
4707 }
4710 }
4712 \f
4713 /* Remember the last target of ix86_set_current_function. */
4716 /* Invalidate ix86_previous_fndecl cache. */
4717 void
4719 {
4721 }
4723 /* Establish appropriate back-end context for processing the function
4724 FNDECL. The argument might be NULL to indicate processing at top
4725 level, outside of any function scope. */
4726 static void
4728 {
4729 /* Only change the context if the function changes. This hook is called
4730 several times in the course of compiling a function, and we don't want to
4731 slow things down too much or call target_reinit when it isn't safe. */
4733 {
4734 tree old_tree = (ix86_previous_fndecl
4738 tree new_tree = (fndecl
4744 ;
4747 {
4751 }
4754 {
4760 }
4761 }
4762 }
4764 \f
4765 /* Return true if this goes in large data/bss. */
4767 static bool
4769 {
4773 /* Functions are never large data. */
4778 {
4784 }
4785 else
4786 {
4789 /* If this is an incomplete type with size 0, then we can't put it
4790 in data because it might be too big when completed. */
4793 }
4796 }
4798 /* Switch to the appropriate section for output of DECL.
4799 DECL is either a `VAR_DECL' node or a constant of some sort.
4800 RELOC indicates whether forming the initial value of DECL requires
4801 link-time relocations. */
4806 {
4809 {
4813 {
4847 /* We don't split these for medium model. Place them into
4848 default sections and hope for best. */
4850 }
4852 {
4853 /* We might get called with string constants, but get_named_section
4854 doesn't like them as they are not DECLs. Also, we need to set
4855 flags in that case. */
4859 }
4860 }
4862 }
4864 /* Select a set of attributes for section NAME based on the properties
4865 of DECL and whether or not RELOC indicates that DECL's initializer
4866 might contain runtime relocations. */
4868 static unsigned int ATTRIBUTE_UNUSED
4870 {
4884 }
4886 /* Build up a unique section name, expressed as a
4887 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
4888 RELOC indicates whether the initial value of EXP requires
4889 link-time relocations. */
4891 static void ATTRIBUTE_UNUSED
4893 {
4896 {
4898 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
4902 {
4926 /* We don't split these for medium model. Place them into
4927 default sections and hope for best. */
4929 }
4931 {
4938 /* If we're using one_only, then there needs to be a .gnu.linkonce
4939 prefix to the section name. */
4946 }
4947 }
4949 }
4951 #ifdef COMMON_ASM_OP
4952 /* This says how to output assembler code to declare an
4953 uninitialized external linkage data object.
4955 For medium model x86-64 we need to use .largecomm opcode for
4956 large objects. */
4957 void
4961 {
4965 else
4970 }
4971 #endif
4973 /* Utility function for targets to use in implementing
4974 ASM_OUTPUT_ALIGNED_BSS. */
4976 void
4980 {
4984 else
4987 #ifdef ASM_DECLARE_OBJECT_NAME
4990 #else
4991 /* Standard thing is just output label for the object. */
4995 }
4996 \f
4997 /* Decide whether we must probe the stack before any space allocation
4998 on this target. It's essentially TARGET_STACK_PROBE except when
4999 -fstack-check causes the stack to be already probed differently. */
5001 bool
5003 {
5004 /* Do not probe the stack twice if static stack checking is enabled. */
5009 }
5010 \f
5011 /* Decide whether we can make a sibling call to a function. DECL is the
5012 declaration of the function being targeted by the call and EXP is the
5013 CALL_EXPR representing the call. */
5015 static bool
5017 {
5021 /* If we are generating position-independent code, we cannot sibcall
5022 optimize any indirect call, or a direct call to a global function,
5023 as the PLT requires %ebx be live. (Darwin does not have a PLT.) */
5025 && !TARGET_64BIT
5026 && flag_pic
5030 /* If we need to align the outgoing stack, then sibcalling would
5031 unalign the stack, which may break the called function. */
5037 {
5040 }
5041 else
5042 {
5043 /* We're looking at the CALL_EXPR, we need the type of the function. */
5048 }
5050 /* Check that the return value locations are the same. Like
5051 if we are returning floats on the 80387 register stack, we cannot
5052 make a sibcall from a function that doesn't return a float to a
5053 function that does or, conversely, from a function that does return
5054 a float to a function that doesn't; the necessary stack adjustment
5055 would not be executed. This is also the place we notice
5056 differences in the return value ABI. Note that it is ok for one
5057 of the functions to have void return type as long as the return
5058 value of the other is passed in a register. */
5063 {
5066 }
5068 ;
5073 {
5074 /* The SYSV ABI has more call-clobbered registers;
5075 disallow sibcalls from MS to SYSV. */
5079 }
5080 else
5081 {
5082 /* If this call is indirect, we'll need to be able to use a
5083 call-clobbered register for the address of the target function.
5084 Make sure that all such registers are not used for passing
5085 parameters. Note that DLLIMPORT functions are indirect. */
5088 {
5090 {
5091 /* ??? Need to count the actual number of registers to be used,
5092 not the possible number of registers. Fix later. */
5094 }
5095 }
5096 }
5098 /* Otherwise okay. That also includes certain types of indirect calls. */
5100 }
5102 /* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
5103 and "sseregparm" calling convention attributes;
5104 arguments as in struct attribute_spec.handler. */
5106 static tree
5108 tree args,
5111 {
5116 {
5118 name);
5121 }
5123 /* Can combine regparm with all attributes but fastcall, and thiscall. */
5125 {
5126 tree cst;
5129 {
5131 }
5134 {
5136 }
5140 {
5143 name);
5145 }
5147 {
5151 }
5154 }
5157 {
5158 /* Do not warn when emulating the MS ABI. */
5163 name);
5166 }
5168 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
5170 {
5172 {
5174 }
5176 {
5178 }
5180 {
5182 }
5184 {
5186 }
5187 }
5189 /* Can combine stdcall with fastcall (redundant), regparm and
5190 sseregparm. */
5192 {
5194 {
5196 }
5198 {
5200 }
5202 {
5204 }
5205 }
5207 /* Can combine cdecl with regparm and sseregparm. */
5209 {
5211 {
5213 }
5215 {
5217 }
5219 {
5221 }
5222 }
5224 {
5227 name);
5229 {
5231 }
5233 {
5235 }
5237 {
5239 }
5240 }
5242 /* Can combine sseregparm with all attributes. */
5245 }
5247 /* The transactional memory builtins are implicitly regparm or fastcall
5248 depending on the ABI. Override the generic do-nothing attribute that
5249 these builtins were declared with, and replace it with one of the two
5250 attributes that we expect elsewhere. */
5252 static tree
5254 tree args ATTRIBUTE_UNUSED,
5256 {
5257 tree alt;
5259 /* In no case do we want to add the placeholder attribute. */
5262 /* The 64-bit ABI is unchanged for transactional memory. */
5266 /* ??? Is there a better way to validate 32-bit windows? We have
5267 cfun->machine->call_abi, but that seems to be set only for 64-bit. */
5270 else
5271 {
5274 }
5278 }
5280 /* This function determines from TYPE the calling-convention. */
5282 unsigned int
5284 {
5287 tree attrs;
5294 {
5304 /* Regparam isn't allowed for thiscall and fastcall. */
5306 {
5311 }
5315 }
5322 || is_stdarg
5328 }
5330 /* Return 0 if the attributes for two types are incompatible, 1 if they
5331 are compatible, and 2 if they are nearly compatible (which causes a
5332 warning to be generated). */
5334 static int
5336 {
5352 }
5353 \f
5354 /* Return the regparm value for a function with the indicated TYPE and DECL.
5355 DECL may be NULL when calling function indirectly
5356 or considering a libcall. */
5358 static int
5360 {
5361 tree attr;
5372 {
5375 {
5378 }
5379 }
5385 /* Use register calling convention for local functions when possible. */
5388 && optimize
5390 {
5391 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5394 {
5397 /* Make sure no regparm register is taken by a
5398 fixed register variable. */
5403 /* We don't want to use regparm(3) for nested functions as
5404 these use a static chain pointer in the third argument. */
5408 /* In 32-bit mode save a register for the split stack. */
5412 /* Each fixed register usage increases register pressure,
5413 so less registers should be used for argument passing.
5414 This functionality can be overriden by an explicit
5415 regparm value. */
5418 globals++;
5420 local_regparm
5425 }
5426 }
5429 }
5431 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
5432 DFmode (2) arguments in SSE registers for a function with the
5433 indicated TYPE and DECL. DECL may be NULL when calling function
5434 indirectly or considering a libcall. Otherwise return 0. */
5436 static int
5438 {
5441 /* Use SSE registers to pass SFmode and DFmode arguments if requested
5442 by the sseregparm attribute. */
5445 {
5447 {
5449 {
5453 else
5456 }
5458 }
5461 }
5463 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
5464 (and DFmode for SSE2) arguments in SSE registers. */
5467 {
5468 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5472 }
5475 }
5477 /* Return true if EAX is live at the start of the function. Used by
5478 ix86_expand_prologue to determine if we need special help before
5479 calling allocate_stack_worker. */
5481 static bool
5483 {
5484 /* Cheat. Don't bother working forward from ix86_function_regparm
5485 to the function type to whether an actual argument is located in
5486 eax. Instead just look at cfg info, which is still close enough
5487 to correct at this point. This gives false positives for broken
5488 functions that might use uninitialized data that happens to be
5489 allocated in eax, but who cares? */
5491 }
5493 static bool
5495 {
5496 tree attr;
5499 {
5505 /* For 32-bit MS-ABI the default is to keep aggregate
5506 return pointer. */
5509 }
5511 }
5513 /* Value is the number of bytes of arguments automatically
5514 popped when returning from a subroutine call.
5515 FUNDECL is the declaration node of the function (as a tree),
5516 FUNTYPE is the data type of the function (as a tree),
5517 or for a library call it is an identifier node for the subroutine name.
5518 SIZE is the number of bytes of arguments passed on the stack.
5520 On the 80386, the RTD insn may be used to pop them if the number
5521 of args is fixed, but if the number is variable then the caller
5522 must pop them all. RTD can't be used for library calls now
5523 because the library is compiled with the Unix compiler.
5524 Use of RTD is a selectable option, since it is incompatible with
5525 standard Unix calling sequences. If the option is not selected,
5526 the caller must always pop the args.
5528 The attribute stdcall is equivalent to RTD on a per module basis. */
5530 static int
5532 {
5535 /* None of the 64-bit ABIs pop arguments. */
5546 /* Lose any fake structure return argument if it is passed on the stack. */
5549 {
5553 }
5556 }
5558 /* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook. */
5560 static bool
5562 {
5563 /* Check operand constraints in case hard registers were propagated
5564 into insn pattern. This check prevents combine pass from
5565 generating insn patterns with invalid hard register operands.
5566 These invalid insns can eventually confuse reload to error out
5567 with a spill failure. See also PRs 46829 and 46843. */
5569 {
5576 {
5584 /* A unary operator may be accepted by the predicate, but it
5585 is irrelevant for matching constraints. */
5590 {
5598 }
5605 /* Operand has no constraints, anything is OK. */
5609 {
5612 && operands_match_p
5616 {
5619 }
5620 }
5624 }
5625 }
5628 }
5629 \f
5630 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
5632 static unsigned HOST_WIDE_INT
5634 {
5638 }
5639 \f
5640 /* Argument support functions. */
5642 /* Return true when register may be used to pass function parameters. */
5643 bool
5645 {
5650 {
5654 else
5660 }
5666 /* TODO: The function should depend on current function ABI but
5667 builtins.c would need updating then. Therefore we use the
5668 default ABI. */
5670 /* RAX is used as hidden argument to va_arg functions. */
5676 else
5683 }
5685 /* Return if we do not know how to pass TYPE solely in registers. */
5687 static bool
5689 {
5693 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
5694 The layout_type routine is crafty and tries to trick us into passing
5695 currently unsupported vector types on the stack by using TImode. */
5698 }
5700 /* It returns the size, in bytes, of the area reserved for arguments passed
5701 in registers for the function represented by fndecl dependent to the used
5702 abi format. */
5703 int
5705 {
5709 else
5714 }
5716 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
5717 call abi used. */
5718 enum calling_abi
5720 {
5722 {
5725 {
5728 }
5732 }
5734 }
5736 /* We add this as a workaround in order to use libc_has_function
5737 hook in i386.md. */
5738 bool
5740 {
5742 }
5744 static bool
5746 {
5748 {
5752 else
5754 }
5756 }
5758 static enum calling_abi
5760 {
5764 }
5766 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
5767 call abi used. */
5768 enum calling_abi
5770 {
5774 }
5776 /* Write the extra assembler code needed to declare a function properly. */
5778 void
5780 tree decl)
5781 {
5785 {
5791 }
5793 #ifdef SUBTARGET_ASM_UNWIND_INIT
5795 #endif
5799 /* Output magic byte marker, if hot-patch attribute is set. */
5801 {
5803 {
5804 /* leaq [%rsp + 0], %rsp */
5807 }
5808 else
5809 {
5810 /* movl.s %edi, %edi
5811 push %ebp
5812 movl.s %esp, %ebp */
5815 }
5816 }
5817 }
5819 /* regclass.c */
5822 /* Implementation of call abi switching target hook. Specific to FNDECL
5823 the specific call register sets are set. See also
5824 ix86_conditional_register_usage for more details. */
5825 void
5827 {
5830 else
5832 }
5834 /* 64-bit MS and SYSV ABI have different set of call used registers. Avoid
5835 expensive re-initialization of init_regs each time we switch function context
5836 since this is needed only during RTL expansion. */
5837 static void
5839 {
5843 }
5845 /* Initialize a variable CUM of type CUMULATIVE_ARGS
5846 for a call to a function whose data type is FNTYPE.
5847 For a library call, FNTYPE is 0. */
5849 void
5853 tree fndecl,
5855 {
5861 {
5864 }
5865 else
5866 {
5869 }
5873 /* Set up the number of registers to use for passing arguments. */
5880 {
5882 ? X86_64_REGPARM_MAX
5884 }
5886 {
5889 {
5891 ? X86_64_SSE_REGPARM_MAX
5893 }
5894 }
5901 /* Because type might mismatch in between caller and callee, we need to
5902 use actual type of function for local calls.
5903 FIXME: cgraph_analyze can be told to actually record if function uses
5904 va_start so for local functions maybe_vaarg can be made aggressive
5905 helping K&R code.
5906 FIXME: once typesytem is fixed, we won't need this code anymore. */
5914 {
5915 /* If there are variable arguments, then we won't pass anything
5916 in registers in 32-bit mode. */
5918 {
5926 }
5928 /* Use ecx and edx registers if function has fastcall attribute,
5929 else look for regparm information. */
5931 {
5934 {
5937 }
5939 {
5942 }
5943 else
5945 }
5947 /* Set up the number of SSE registers used for passing SFmode
5948 and DFmode arguments. Warn for mismatching ABI. */
5950 }
5951 }
5953 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
5954 But in the case of vector types, it is some vector mode.
5956 When we have only some of our vector isa extensions enabled, then there
5957 are some modes for which vector_mode_supported_p is false. For these
5958 modes, the generic vector support in gcc will choose some non-vector mode
5959 in order to implement the type. By computing the natural mode, we'll
5960 select the proper ABI location for the operand and not depend on whatever
5961 the middle-end decides to do with these vector types.
5963 The midde-end can't deal with the vector types > 16 bytes. In this
5964 case, we return the original mode and warn ABI change if CUM isn't
5965 NULL. */
5967 static enum machine_mode
5969 {
5973 {
5976 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
5978 {
5983 else
5986 /* Get the mode which has this inner mode and number of units. */
5990 {
5992 {
5996 && !warnedavx
5998 {
6002 }
6004 }
6006 {
6010 && !warnedsse
6012 {
6016 }
6018 }
6019 else
6021 }
6024 }
6025 }
6028 }
6030 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
6031 this may not agree with the mode that the type system has chosen for the
6032 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
6033 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
6035 static rtx
6038 {
6039 rtx tmp;
6043 else
6044 {
6048 }
6051 }
6053 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
6054 of this code is to classify each 8bytes of incoming argument by the register
6055 class and assign registers accordingly. */
6057 /* Return the union class of CLASS1 and CLASS2.
6058 See the x86-64 PS ABI for details. */
6060 static enum x86_64_reg_class
6062 {
6063 /* Rule #1: If both classes are equal, this is the resulting class. */
6067 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
6068 the other class. */
6074 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
6078 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
6086 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
6087 MEMORY is used. */
6096 /* Rule #6: Otherwise class SSE is used. */
6098 }
6100 /* Classify the argument of type TYPE and mode MODE.
6101 CLASSES will be filled by the register class used to pass each word
6102 of the operand. The number of words is returned. In case the parameter
6103 should be passed in memory, 0 is returned. As a special case for zero
6104 sized containers, classes[0] will be NO_CLASS and 1 is returned.
6106 BIT_OFFSET is used internally for handling records and specifies offset
6107 of the offset in bits modulo 256 to avoid overflow cases.
6109 See the x86-64 PS ABI for details.
6110 */
6112 static int
6115 {
6116 HOST_WIDE_INT bytes =
6118 int words
6121 /* Variable sized entities are always passed/returned in memory. */
6129 /* Special case check for pointer to shared, on 64-bit target. */
6133 {
6136 }
6139 {
6141 tree field;
6144 /* On x86-64 we pass structures larger than 32 bytes on the stack. */
6151 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
6152 signalize memory class, so handle it as special case. */
6154 {
6157 }
6159 /* Classify each field of record and merge classes. */
6161 {
6163 /* And now merge the fields of structure. */
6165 {
6167 {
6173 /* Bitfields are always classified as integer. Handle them
6174 early, since later code would consider them to be
6175 misaligned integers. */
6177 {
6186 }
6187 else
6188 {
6193 /* Flexible array member is ignored. */
6200 {
6204 {
6207 "the ABI of passing struct with"
6208 " a flexible array member has"
6210 }
6212 }
6214 subclasses,
6224 }
6225 }
6226 }
6230 /* Arrays are handled as small records. */
6231 {
6238 /* The partial classes are now full classes. */
6249 }
6252 /* Unions are similar to RECORD_TYPE but offset is always 0.
6253 */
6255 {
6257 {
6265 bit_offset);
6270 }
6271 }
6276 }
6279 {
6280 /* When size > 16 bytes, if the first one isn't
6281 X86_64_SSE_CLASS or any other ones aren't
6282 X86_64_SSEUP_CLASS, everything should be passed in
6283 memory. */
6290 }
6292 /* Final merger cleanup. */
6294 {
6295 /* If one class is MEMORY, everything should be passed in
6296 memory. */
6300 /* The X86_64_SSEUP_CLASS should be always preceded by
6301 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
6305 {
6306 /* The first one should never be X86_64_SSEUP_CLASS. */
6309 }
6311 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
6312 everything should be passed in memory. */
6315 {
6318 /* The first one should never be X86_64_X87UP_CLASS. */
6321 {
6324 "the ABI of passing union with long double"
6326 }
6328 }
6329 }
6331 }
6333 /* Compute alignment needed. We align all types to natural boundaries with
6334 exception of XFmode that is aligned to 64bits. */
6336 {
6345 /* Misaligned fields are always returned in memory. */
6348 }
6350 /* for V1xx modes, just use the base mode */
6355 /* Classification of atomic types. */
6357 {
6373 {
6377 {
6380 }
6382 {
6385 }
6387 {
6391 }
6393 {
6396 }
6397 else
6399 }
6406 /* OImode shouldn't be used directly. */
6413 else
6431 else
6432 {
6436 {
6439 "the ABI of passing structure with complex float"
6441 }
6444 }
6453 /* This modes is larger than 16 bytes. */
6496 else
6500 }
6501 }
6503 /* Examine the argument and return set number of register required in each
6504 class. Return 0 iff parameter should be passed in memory. */
6505 static int
6508 {
6518 {
6540 }
6542 }
6544 /* Construct container for the argument used by GCC interface. See
6545 FUNCTION_ARG for the detailed description. */
6547 static rtx
6551 {
6552 /* The following variables hold the static issued_error state. */
6566 rtx ret;
6577 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
6578 some less clueful developer tries to use floating-point anyway. */
6580 {
6582 {
6584 {
6587 }
6588 }
6590 {
6593 }
6595 }
6597 /* Likewise, error if the ABI requires us to return values in the
6598 x87 registers and the user specified -mno-80387. */
6604 {
6606 {
6609 }
6611 }
6613 /* First construct simple cases. Avoid SCmode, since we want to use
6614 single register to pass this type. */
6617 {
6632 /* Zero sized array, struct or class. */
6636 }
6663 /* Otherwise figure out the entries of the PARALLEL. */
6665 {
6669 {
6674 /* Merge TImodes on aligned occasions here too. */
6676 tmpmode
6680 else
6682 /* We've requested 24 bytes we
6683 don't have mode for. Use DImode. */
6690 intreg++;
6698 sse_regno++;
6706 sse_regno++;
6711 {
6717 {
6719 i++;
6720 }
6721 else
6734 }
6740 sse_regno++;
6744 }
6745 }
6747 /* Empty aligned struct, union or class. */
6755 }
6757 /* Update the data in CUM to advance over an argument of mode MODE
6758 and data type TYPE. (TYPE is null for libcalls where that information
6759 may not be available.) */
6761 static void
6764 HOST_WIDE_INT words)
6765 {
6767 {
6774 /* FALLTHRU */
6785 {
6788 }
6792 /* OImode shouldn't be used directly. */
6801 /* FALLTHRU */
6817 {
6822 {
6825 }
6826 }
6836 {
6841 {
6844 }
6845 }
6847 }
6848 }
6850 static void
6853 {
6856 /* Unnamed 256bit vector mode parameters are passed on stack. */
6862 {
6867 }
6868 else
6869 {
6873 }
6874 }
6876 static void
6878 HOST_WIDE_INT words)
6879 {
6880 /* Otherwise, this should be passed indirect. */
6885 {
6888 }
6889 }
6891 /* Update the data in CUM to advance over an argument of mode MODE and
6892 data type TYPE. (TYPE is null for libcalls where that information
6893 may not be available.) */
6895 static void
6898 {
6904 else
6915 else
6917 }
6919 /* Define where to put the arguments to a function.
6920 Value is zero to push the argument on the stack,
6921 or a hard register in which to store the argument.
6923 MODE is the argument's machine mode.
6924 TYPE is the data type of the argument (as a tree).
6925 This is null for libcalls where that information may
6926 not be available.
6927 CUM is a variable of type CUMULATIVE_ARGS which gives info about
6928 the preceding args and about the function being called.
6929 NAMED is nonzero if this argument is a named parameter
6930 (otherwise it is an extra parameter matching an ellipsis). */
6932 static rtx
6936 {
6939 /* Avoid the AL settings for the Unix64 ABI. */
6944 {
6951 /* FALLTHRU */
6957 {
6960 /* Fastcall allocates the first two DWORD (SImode) or
6961 smaller arguments to ECX and EDX if it isn't an
6962 aggregate type . */
6964 {
6970 /* ECX not EAX is the first allocated register. */
6973 }
6975 }
6984 /* FALLTHRU */
6986 /* In 32bit, we pass TImode in xmm registers. */
6994 {
6996 {
7000 }
7004 }
7008 /* OImode shouldn't be used directly. */
7018 {
7022 }
7032 {
7034 {
7038 }
7042 }
7044 }
7047 }
7049 static rtx
7052 {
7053 /* Handle a hidden AL argument containing number of registers
7054 for varargs x86-64 functions. */
7058 ? X86_64_SSE_REGPARM_MAX
7063 {
7073 /* Unnamed 256bit vector mode parameters are passed on stack. */
7077 }
7083 }
7085 static rtx
7088 HOST_WIDE_INT bytes)
7089 {
7092 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
7093 We use value of -2 to specify that current function call is MSABI. */
7097 /* If we've run out of registers, it goes on the stack. */
7103 /* Only floating point modes are passed in anything but integer regs. */
7105 {
7108 else
7109 {
7112 /* Unnamed floating parameters are passed in both the
7113 SSE and integer registers. */
7119 }
7120 }
7121 /* Handle aggregated types passed in register. */
7123 {
7128 }
7131 }
7133 /* Return where to put the arguments to a function.
7134 Return zero to push the argument on the stack, or a hard register in which to store the argument.
7136 MODE is the argument's machine mode. TYPE is the data type of the
7137 argument. It is null for libcalls where that information may not be
7138 available. CUM gives information about the preceding args and about
7139 the function being called. NAMED is nonzero if this argument is a
7140 named parameter (otherwise it is an extra parameter matching an
7141 ellipsis). */
7143 static rtx
7146 {
7150 rtx arg;
7154 else
7158 /* To simplify the code below, represent vector types with a vector mode
7159 even if MMX/SSE are not active. */
7167 else
7171 }
7173 /* A C expression that indicates when an argument must be passed by
7174 reference. If nonzero for an argument, a copy of that argument is
7175 made in memory and a pointer to the argument is passed instead of
7176 the argument itself. The pointer is passed in whatever way is
7177 appropriate for passing a pointer to that type. */
7179 static bool
7182 {
7185 /* See Windows x64 Software Convention. */
7187 {
7190 {
7191 /* Arrays are passed by reference. */
7196 {
7197 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
7198 are passed by reference. */
7200 }
7201 }
7203 /* __m128 is passed by reference. */
7209 }
7210 }
7215 }
7217 /* Return true when TYPE should be 128bit aligned for 32bit argument
7218 passing ABI. XXX: This function is obsolete and is only used for
7219 checking psABI compatibility with previous versions of GCC. */
7221 static bool
7223 {
7235 {
7236 /* Walk the aggregates recursively. */
7238 {
7242 {
7243 tree field;
7245 /* Walk all the structure fields. */
7247 {
7251 }
7253 }
7256 /* Just for use if some languages passes arrays by value. */
7263 }
7264 }
7266 }
7268 /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
7269 XXX: This function is obsolete and is only used for checking psABI
7270 compatibility with previous versions of GCC. */
7272 static unsigned int
7275 {
7276 /* In 32bit, only _Decimal128 and __float128 are aligned to their
7277 natural boundaries. */
7279 {
7280 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
7281 make an exception for SSE modes since these require 128bit
7282 alignment.
7284 The handling here differs from field_alignment. ICC aligns MMX
7285 arguments to 4 byte boundaries, while structure fields are aligned
7286 to 8 byte boundaries. */
7288 {
7291 }
7292 else
7293 {
7296 }
7297 }
7301 }
7303 /* Return true when TYPE should be 128bit aligned for 32bit argument
7304 passing ABI. */
7306 static bool
7308 {
7318 {
7319 /* Walk the aggregates recursively. */
7321 {
7325 {
7326 tree field;
7328 /* Walk all the structure fields. */
7330 field;
7332 {
7336 }
7338 }
7341 /* Just for use if some languages passes arrays by value. */
7348 }
7349 }
7350 else
7354 }
7356 /* Gives the alignment boundary, in bits, of an argument with the
7357 specified mode and type. */
7359 static unsigned int
7361 {
7364 {
7365 /* Since the main variant type is used for call, we convert it to
7366 the main variant type. */
7369 }
7370 else
7374 else
7375 {
7380 {
7381 /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
7383 {
7386 }
7392 }
7395 && !warned
7397 saved_align))
7398 {
7401 "The ABI for passing parameters with %d-byte"
7404 }
7405 }
7408 }
7410 /* Return true if N is a possible register number of function value. */
7412 static bool
7414 {
7416 {
7424 /* Complex values are returned in %st(0)/%st(1) pair. */
7427 /* TODO: The function should depend on current function ABI but
7428 builtins.c would need updating then. Therefore we use the
7429 default ABI. */
7434 /* Complex values are returned in %xmm0/%xmm1 pair. */
7443 }
7446 }
7448 /* Define how to find the value returned by a function.
7449 VALTYPE is the data type of the value (as a tree).
7450 If the precise function being called is known, FUNC is its FUNCTION_DECL;
7451 otherwise, FUNC is 0. */
7453 static rtx
7456 {
7459 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
7460 we normally prevent this case when mmx is not available. However
7461 some ABIs may require the result to be returned like DImode. */
7465 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
7466 we prevent this case when sse is not available. However some ABIs
7467 may require the result to be returned like integer TImode. */
7472 /* 32-byte vector modes in %ymm0. */
7476 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
7479 else
7480 /* Most things go in %eax. */
7483 /* Override FP return register with %xmm0 for local functions when
7484 SSE math is enabled or for functions with sseregparm attribute. */
7486 {
7491 }
7493 /* OImode shouldn't be used directly. */
7497 }
7499 static rtx
7501 const_tree valtype)
7502 {
7503 rtx ret;
7505 /* Handle libcalls, which don't provide a type node. */
7507 {
7511 {
7530 }
7533 }
7536 {
7537 /* Pointers are always returned in word_mode. */
7539 }
7545 /* For zero sized structures, construct_container returns NULL, but we
7546 need to keep rest of compiler happy by returning meaningful value. */
7551 }
7553 static rtx
7555 const_tree valtype)
7556 {
7560 {
7562 {
7581 }
7582 }
7584 }
7586 static rtx
7589 {
7601 else
7603 }
7605 static rtx
7608 {
7614 }
7616 /* Pointer function arguments and return values are promoted to
7617 word_mode. */
7619 static enum machine_mode
7623 {
7625 {
7627 {
7630 }
7633 }
7635 for_return);
7636 }
7638 /* Return true if a structure, union or array with MODE containing FIELD
7639 should be accessed using BLKmode. */
7641 static bool
7643 {
7644 /* Union with XFmode must be in BLKmode. */
7648 }
7650 rtx
7652 {
7654 }
7656 /* Return true iff type is returned in memory. */
7658 static bool ATTRIBUTE_UNUSED
7660 {
7661 HOST_WIDE_INT size;
7672 {
7673 /* User-created vectors small enough to fit in EAX. */
7677 /* MMX/3dNow values are returned in MM0,
7678 except when it doesn't exits or the ABI prescribes otherwise. */
7682 /* SSE values are returned in XMM0, except when it doesn't exist. */
7686 /* AVX values are returned in YMM0, except when it doesn't exist. */
7689 }
7697 /* OImode shouldn't be used directly. */
7701 }
7703 static bool ATTRIBUTE_UNUSED
7705 {
7708 }
7710 static bool ATTRIBUTE_UNUSED
7712 {
7715 /* __m128 is returned in xmm0. */
7722 /* Otherwise, the size must be exactly in [1248]. */
7724 }
7726 static bool
7728 {
7729 #ifdef SUBTARGET_RETURN_IN_MEMORY
7731 #else
7735 {
7738 else
7740 }
7741 else
7743 #endif
7744 }
7746 /* When returning SSE vector types, we have a choice of either
7747 (1) being abi incompatible with a -march switch, or
7748 (2) generating an error.
7749 Given no good solution, I think the safest thing is one warning.
7750 The user won't be able to use -Werror, but....
7752 Choose the STRUCT_VALUE_RTX hook because that's (at present) only
7753 called in response to actually generating a caller or callee that
7754 uses such a type. As opposed to TARGET_RETURN_IN_MEMORY, which is called
7755 via aggregate_value_p for general type probing from tree-ssa. */
7757 static rtx
7759 {
7763 {
7764 /* Look at the return type of the function, not the function type. */
7768 {
7771 {
7775 }
7776 }
7779 {
7781 {
7785 }
7786 }
7787 }
7790 }
7792 \f
7793 /* Create the va_list data type. */
7795 /* Returns the calling convention specific va_list date type.
7796 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
7798 static tree
7800 {
7803 /* For i386 we use plain pointer to argument area. */
7813 unsigned_type_node);
7816 unsigned_type_node);
7819 ptr_type_node);
7822 ptr_type_node);
7841 /* The correct type is an array type of one element. */
7843 }
7845 /* Setup the builtin va_list data type and for 64-bit the additional
7846 calling convention specific va_list data types. */
7848 static tree
7850 {
7853 /* Initialize abi specific va_list builtin types. */
7855 {
7856 tree t;
7858 {
7863 }
7864 else
7865 {
7870 }
7872 {
7877 }
7878 else
7879 {
7884 }
7885 }
7888 }
7890 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
7892 static void
7894 {
7896 alias_set_type set;
7899 /* GPR size of varargs save area. */
7902 else
7905 /* FPR size of varargs save area. We don't need it if we don't pass
7906 anything in SSE registers. */
7909 else
7923 {
7931 }
7934 {
7938 /* Now emit code to save SSE registers. The AX parameter contains number
7939 of SSE parameter registers used to call this function, though all we
7940 actually check here is the zero/non-zero status. */
7945 label));
7947 /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
7948 we used movdqa (i.e. TImode) instead? Perhaps even better would
7949 be if we could determine the real mode of the data, via a hook
7950 into pass_stdarg. Ignore all that for now. */
7960 {
7969 }
7972 }
7973 }
7975 static void
7977 {
7981 /* Reset to zero, as there might be a sysv vaarg used
7982 before. */
7987 {
7998 }
7999 }
8001 static void
8005 {
8007 CUMULATIVE_ARGS next_cum;
8008 tree fntype;
8010 /* This argument doesn't appear to be used anymore. Which is good,
8011 because the old code here didn't suppress rtl generation. */
8019 /* For varargs, we do not want to skip the dummy va_dcl argument.
8020 For stdargs, we do want to skip the last named argument. */
8028 else
8030 }
8032 /* Checks if TYPE is of kind va_list char *. */
8034 static bool
8036 {
8037 tree canonic;
8039 /* For 32-bit it is always true. */
8045 }
8047 /* Implement va_start. */
8049 static void
8051 {
8055 tree type;
8056 rtx ovf_rtx;
8060 {
8063 /* When we are splitting the stack, we can't refer to the stack
8064 arguments using internal_arg_pointer, because they may be on
8065 the old stack. The split stack prologue will arrange to
8066 leave a pointer to the old stack arguments in a scratch
8067 register, which we here copy to a pseudo-register. The split
8068 stack prologue can't set the pseudo-register directly because
8069 it (the prologue) runs before any registers have been saved. */
8073 {
8087 }
8088 }
8090 /* Only 64bit target needs something special. */
8092 {
8095 else
8096 {
8105 }
8107 }
8116 /* The following should be folded into the MEM_REF offset. */
8126 /* Count number of gp and fp argument registers used. */
8132 {
8138 }
8141 {
8147 }
8149 /* Find the overflow area. */
8153 else
8163 {
8164 /* Find the register save area.
8165 Prologue of the function save it right above stack frame. */
8173 }
8174 }
8176 /* Implement va_arg. */
8178 static tree
8181 {
8188 rtx container;
8190 tree ptrtype;
8194 /* Only 64bit target needs something special. */
8218 {
8225 /* Unnamed 256bit vector mode parameters are passed on stack. */
8227 {
8230 }
8238 }
8240 /* Pull the value out of the saved registers. */
8245 {
8259 /* In case we are passing structure, verify that it is consecutive block
8260 on the register save area. If not we need to do moves. */
8262 {
8263 /* Verify that all registers are strictly consecutive */
8265 {
8269 {
8274 }
8275 }
8276 else
8277 {
8281 {
8286 }
8287 }
8288 }
8290 {
8293 }
8294 else
8295 {
8298 }
8300 /* First ensure that we fit completely in registers. */
8302 {
8309 }
8311 {
8319 }
8321 /* Compute index to start of area used for integer regs. */
8323 {
8324 /* int_addr = gpr + sav; */
8327 }
8329 {
8330 /* sse_addr = fpr + sav; */
8333 }
8335 {
8339 /* addr = &temp; */
8344 {
8348 tree piece_type;
8349 tree addr_type;
8350 tree daddr_type;
8359 {
8364 }
8368 else
8375 {
8378 }
8379 else
8380 {
8383 }
8390 {
8395 }
8396 else
8397 {
8398 tree copy
8403 }
8405 }
8406 }
8409 {
8413 }
8416 {
8420 }
8425 }
8427 /* ... otherwise out of the overflow area. */
8429 /* When we align parameter on stack for caller, if the parameter
8430 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
8431 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
8432 here with caller. */
8437 /* Care for on-stack alignment if needed. */
8440 else
8441 {
8446 }
8463 }
8464 \f
8465 /* Return true if OPNUM's MEM should be matched
8466 in movabs* patterns. */
8468 bool
8470 {
8482 }
8483 \f
8484 /* Initialize the table of extra 80387 mathematical constants. */
8486 static void
8488 {
8490 {
8496 };
8500 {
8502 /* Ensure each constant is rounded to XFmode precision. */
8505 }
8508 }
8510 /* Return non-zero if the constant is something that
8511 can be loaded with a special instruction. */
8513 int
8515 {
8518 REAL_VALUE_TYPE r;
8530 /* For XFmode constants, try to find a special 80387 instruction when
8531 optimizing for size or on those CPUs that benefit from them. */
8534 {
8543 }
8545 /* Load of the constant -0.0 or -1.0 will be split as
8546 fldz;fchs or fld1;fchs sequence. */
8553 }
8555 /* Return the opcode of the special instruction to be used to load
8556 the constant X. */
8560 {
8562 {
8582 }
8583 }
8585 /* Return the CONST_DOUBLE representing the 80387 constant that is
8586 loaded by the specified special instruction. The argument IDX
8587 matches the return value from standard_80387_constant_p. */
8589 rtx
8591 {
8598 {
8609 }
8612 XFmode);
8613 }
8615 /* Return 1 if X is all 0s and 2 if x is all 1s
8616 in supported SSE/AVX vector mode. */
8618 int
8620 {
8627 {
8642 }
8645 }
8647 /* Return the opcode of the special instruction to be used to load
8648 the constant X. */
8652 {
8654 {
8657 {
8674 }
8683 else
8688 }
8690 }
8692 /* Returns true if OP contains a symbol reference */
8694 bool
8696 {
8705 {
8707 {
8713 }
8717 }
8720 }
8722 /* Return true if it is appropriate to emit `ret' instructions in the
8723 body of a function. Do this only if the epilogue is simple, needing a
8724 couple of insns. Prior to reloading, we can't tell how many registers
8725 must be saved, so return false then. Return false if there is no frame
8726 marker to de-allocate. */
8728 bool
8730 {
8736 /* Don't allow more than 32k pop, since that's all we can do
8737 with one instruction. */
8744 }
8745 \f
8746 /* Value should be nonzero if functions must have frame pointers.
8747 Zero means the frame pointer need not be set up (and parms may
8748 be accessed via the stack pointer) in functions that seem suitable. */
8750 static bool
8752 {
8753 /* If we accessed previous frames, then the generated code expects
8754 to be able to access the saved ebp value in our frame. */
8758 /* Several x86 os'es need a frame pointer for other reasons,
8759 usually pertaining to setjmp. */
8763 /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
8767 /* Win64 SEH, very large frames need a frame-pointer as maximum stack
8768 allocation is 4GB. */
8772 /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
8773 turns off the frame pointer by default. Turn it back on now if
8774 we've not got a leaf function. */
8784 }
8786 /* Record that the current function accesses previous call frames. */
8788 void
8790 {
8792 }
8793 \f
8794 #ifndef USE_HIDDEN_LINKONCE
8795 # if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
8796 # define USE_HIDDEN_LINKONCE 1
8797 # else
8798 # define USE_HIDDEN_LINKONCE 0
8799 # endif
8800 #endif
8804 /* Fills in the label name that should be used for a pc thunk for
8805 the given register. */
8807 static void
8809 {
8814 else
8816 }
8819 /* This function generates code for -fpic that loads %ebx with
8820 the return address of the caller and then returns. */
8822 static void
8824 {
8829 {
8831 tree decl;
8847 #if TARGET_MACHO
8849 {
8858 }
8859 else
8860 #endif
8862 {
8873 }
8874 else
8875 {
8878 }
8884 /* Make sure unwind info is emitted for the thunk if needed. */
8887 /* Pad stack IP move with 4 instructions (two NOPs count
8888 as one instruction). */
8890 {
8895 }
8906 }
8910 }
8912 /* Emit code for the SET_GOT patterns. */
8916 {
8922 {
8923 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
8928 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
8929 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
8930 an unadorned address. */
8935 }
8940 {
8942 /* We don't need a pic base, we're not producing pic. */
8949 }
8950 else
8951 {
8960 #if TARGET_MACHO
8961 /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
8962 This is what will be referenced by the Mach-O PIC subsystem. */
8966 /* When we are restoring the pic base at the site of a nonlocal label,
8967 and we decided to emit the pic base above, we will still output a
8968 local label used for calculating the correction offset (even though
8969 the offset will be 0 in that case). */
8973 #endif
8974 }
8980 }
8982 /* Generate an "push" pattern for input ARG. */
8984 static rtx
8986 {
8999 stack_pointer_rtx)),
9000 arg);
9001 }
9003 /* Generate an "pop" pattern for input ARG. */
9005 static rtx
9007 {
9012 arg,
9015 stack_pointer_rtx)));
9016 }
9018 /* Return >= 0 if there is an unused call-clobbered register available
9019 for the entire function. */
9021 static unsigned int
9023 {
9027 {
9029 /* Can't use the same register for both PIC and DRAP. */
9032 else
9037 }
9040 }
9042 /* Return TRUE if we need to save REGNO. */
9044 static bool
9046 {
9057 {
9060 {
9066 }
9067 }
9076 }
9078 /* Return number of saved general prupose registers. */
9080 static int
9082 {
9088 nregs ++;
9090 }
9092 /* Return number of saved SSE registrers. */
9094 static int
9096 {
9104 nregs ++;
9106 }
9108 /* Given FROM and TO register numbers, say whether this elimination is
9109 allowed. If stack alignment is needed, we can only replace argument
9110 pointer with hard frame pointer, or replace frame pointer with stack
9111 pointer. Otherwise, frame pointer elimination is automatically
9112 handled and all other eliminations are valid. */
9114 static bool
9116 {
9122 else
9124 }
9126 /* Return the offset between two registers, one to be eliminated, and the other
9127 its replacement, at the start of a routine. */
9129 HOST_WIDE_INT
9131 {
9140 else
9141 {
9149 }
9150 }
9152 /* In a dynamically-aligned function, we can't know the offset from
9153 stack pointer to frame pointer, so we must ensure that setjmp
9154 eliminates fp against the hard fp (%ebp) rather than trying to
9155 index from %esp up to the top of the frame across a gap that is
9156 of unknown (at compile-time) size. */
9157 static rtx
9159 {
9161 }
9163 /* When using -fsplit-stack, the allocation routines set a field in
9164 the TCB to the bottom of the stack plus this much space, measured
9165 in bytes. */
9167 #define SPLIT_STACK_AVAILABLE 256
9169 /* Fill structure ix86_frame about frame of currently computed function. */
9171 static void
9173 {
9175 HOST_WIDE_INT offset;
9178 HOST_WIDE_INT to_allocate;
9186 /* 64-bit MS ABI seem to require stack alignment to be always 16 except for
9187 function prologues and leaf. */
9191 {
9196 }
9202 /* For SEH we have to limit the amount of code movement into the prologue.
9203 At present we do this via a BLOCKAGE, at which point there's very little
9204 scheduling that can be done, which means that there's very little point
9205 in doing anything except PUSHs. */
9209 /* During reload iteration the amount of registers saved can change.
9210 Recompute the value as needed. Do not recompute when amount of registers
9211 didn't change as reload does multiple calls to the function and does not
9212 expect the decision to change within single iteration. */
9215 {
9221 /* The fast prologue uses move instead of push to save registers. This
9222 is significantly longer, but also executes faster as modern hardware
9223 can execute the moves in parallel, but can't do that for push/pop.
9225 Be careful about choosing what prologue to emit: When function takes
9226 many instructions to execute we may use slow version as well as in
9227 case function is known to be outside hot spot (this is known with
9228 feedback only). Weight the size of function by number of registers
9229 to save as it is cheap to use one or two push instructions but very
9230 slow to use many of them. */
9234 || (flag_branch_probabilities
9237 else
9240 }
9242 frame->save_regs_using_mov
9244 /* If static stack checking is enabled and done with probes,
9245 the registers need to be saved before allocating the frame. */
9248 /* Skip return address. */
9251 /* Skip pushed static chain. */
9255 /* Skip saved base pointer. */
9260 /* The traditional frame pointer location is at the top of the frame. */
9263 /* Register save area */
9267 /* On SEH target, registers are pushed just before the frame pointer
9268 location. */
9272 /* Align and set SSE register save area. */
9274 {
9275 /* The only ABI that has saved SSE registers (Win64) also has a
9276 16-byte aligned default stack, and thus we don't need to be
9277 within the re-aligned local stack frame to save them. */
9281 }
9284 /* The re-aligned stack starts here. Values before this point are not
9285 directly comparable with values below this point. In order to make
9286 sure that no value happens to be the same before and after, force
9287 the alignment computation below to add a non-zero value. */
9291 /* Va-arg area */
9295 /* Align start of frame for local function. */
9304 /* Frame pointer points here. */
9309 /* Add outgoing arguments area. Can be skipped if we eliminated
9310 all the function calls as dead code.
9311 Skipping is however impossible when function calls alloca. Alloca
9312 expander assumes that last crtl->outgoing_args_size
9313 of stack frame are unused. */
9317 {
9320 }
9321 else
9324 /* Align stack boundary. Only needed if we're calling another function
9325 or using alloca. */
9330 /* We've reached end of stack frame. */
9333 /* Size prologue needs to allocate. */
9344 {
9350 }
9351 else
9355 /* The SEH frame pointer location is near the bottom of the frame.
9356 This is enforced by the fact that the difference between the
9357 stack pointer and the frame pointer is limited to 240 bytes in
9358 the unwind data structure. */
9360 {
9361 HOST_WIDE_INT diff;
9363 /* If we can leave the frame pointer where it is, do so. Also, returns
9364 the establisher frame for __builtin_frame_address (0). */
9369 {
9370 /* Ideally we'd determine what portion of the local stack frame
9371 (within the constraint of the lowest 240) is most heavily used.
9372 But without that complication, simply bias the frame pointer
9373 by 128 bytes so as to maximize the amount of the local stack
9374 frame that is addressable with 8-bit offsets. */
9376 }
9377 }
9378 }
9380 /* This is semi-inlined memory_address_length, but simplified
9381 since we know that we're always dealing with reg+offset, and
9382 to avoid having to create and discard all that rtl. */
9386 {
9390 {
9391 /* EBP and R13 cannot be encoded without an offset. */
9393 }
9397 /* ESP and R12 must be encoded with a SIB byte. */
9399 len++;
9402 }
9404 /* Return an RTX that points to CFA_OFFSET within the stack frame.
9405 The valid base registers are taken from CFUN->MACHINE->FS. */
9407 static rtx
9409 {
9415 {
9416 /* Choose the base register most likely to allow the most scheduling
9417 opportunities. Generally FP is valid throughout the function,
9418 while DRAP must be reloaded within the epilogue. But choose either
9419 over the SP due to increased encoding size. */
9422 {
9425 }
9427 {
9430 }
9432 {
9435 }
9436 }
9437 else
9438 {
9439 HOST_WIDE_INT toffset;
9442 /* Choose the base register with the smallest address encoding.
9443 With a tie, choose FP > DRAP > SP. */
9445 {
9449 }
9451 {
9455 {
9459 }
9460 }
9462 {
9466 {
9470 }
9471 }
9472 }
9476 }
9478 /* Emit code to save registers in the prologue. */
9480 static void
9482 {
9484 rtx insn;
9488 {
9491 }
9492 }
9494 /* Emit a single register save at CFA - CFA_OFFSET. */
9496 static void
9498 HOST_WIDE_INT cfa_offset)
9499 {
9507 /* For SSE saves, we need to indicate the 128-bit alignment. */
9518 /* When saving registers into a re-aligned local stack frame, avoid
9519 any tricky guessing by dwarf2out. */
9521 {
9525 {
9526 /* A bit of a hack. We force the DRAP register to be saved in
9527 the re-aligned stack frame, which provides us with a copy
9528 of the CFA that will last past the prologue. Install it. */
9534 }
9535 else
9536 {
9537 /* The frame pointer is a stable reference within the
9538 aligned frame. Use it. */
9545 }
9546 }
9548 /* The memory may not be relative to the current CFA register,
9549 which means that we may need to generate a new pattern for
9550 use by the unwind info. */
9552 {
9557 }
9558 }
9560 /* Emit code to save registers using MOV insns.
9561 First register is stored at CFA - CFA_OFFSET. */
9562 static void
9564 {
9569 {
9572 }
9573 }
9575 /* Emit code to save SSE registers using MOV insns.
9576 First register is stored at CFA - CFA_OFFSET. */
9577 static void
9579 {
9584 {
9587 }
9588 }
9592 /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
9593 manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
9594 Don't add the note if the previously saved value will be left untouched
9595 within stack red-zone till return, as unwinders can find the same value
9596 in the register and on the stack. */
9598 static void
9600 {
9606 {
9609 }
9610 else
9611 queued_cfa_restores
9613 }
9615 /* Add queued REG_CFA_RESTORE notes if any to INSN. */
9617 static void
9619 {
9620 rtx last;
9624 ;
9629 }
9631 /* Expand prologue or epilogue stack adjustment.
9632 The pattern exist to put a dependency on all ebp-based memory accesses.
9633 STYLE should be negative if instructions should be marked as frame related,
9634 zero if %r11 register is live and cannot be freely used and positive
9635 otherwise. */
9637 static void
9640 {
9642 rtx insn;
9649 else
9650 {
9651 rtx tmp;
9652 /* r11 is used by indirect sibcall return as well, set before the
9653 epilogue and used after the epilogue. */
9656 else
9657 {
9661 }
9667 }
9674 {
9675 rtx r;
9685 }
9687 {
9690 {
9694 }
9695 }
9698 {
9703 {
9706 }
9708 {
9711 }
9712 else
9713 {
9714 /* Else there are two possibilities: SP itself, which we set
9715 up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
9716 taken care of this by hand along the eh_return path. */
9719 }
9723 }
9724 }
9726 /* Find an available register to be used as dynamic realign argument
9727 pointer regsiter. Such a register will be written in prologue and
9728 used in begin of body, so it must not be
9729 1. parameter passing register.
9730 2. GOT pointer.
9731 We reuse static-chain register if it is available. Otherwise, we
9732 use DI for i386 and R13 for x86-64. We chose R13 since it has
9733 shorter encoding.
9735 Return: the regno of chosen register. */
9737 static unsigned int
9739 {
9743 {
9744 /* Use R13 for nested function or function need static chain.
9745 Since function with tail call may use any caller-saved
9746 registers in epilogue, DRAP must not use caller-saved
9747 register in such case. */
9752 }
9753 else
9754 {
9755 /* Use DI for nested function or function need static chain.
9756 Since function with tail call may use any caller-saved
9757 registers in epilogue, DRAP must not use caller-saved
9758 register in such case. */
9762 /* Reuse static chain register if it isn't used for parameter
9763 passing. */
9765 {
9769 }
9771 }
9772 }
9774 /* Return minimum incoming stack alignment. */
9776 static unsigned int
9778 {
9781 /* Prefer the one specified at command line. */
9784 /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
9785 if -mstackrealign is used, it isn't used for sibcall check and
9786 estimated stack alignment is 128bit. */
9788 && !TARGET_64BIT
9789 && ix86_force_align_arg_pointer
9792 else
9795 /* Incoming stack alignment can be changed on individual functions
9796 via force_align_arg_pointer attribute. We use the smallest
9797 incoming stack boundary. */
9803 /* The incoming stack frame has to be aligned at least at
9804 parm_stack_boundary. */
9808 /* Stack at entrance of main is aligned by runtime. We use the
9809 smallest incoming stack boundary. */
9817 }
9819 /* Update incoming stack boundary and estimated stack alignment. */
9821 static void
9823 {
9824 ix86_incoming_stack_boundary
9827 /* x86_64 vararg needs 16byte stack alignment for register save
9828 area. */
9833 }
9835 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
9836 needed or an rtx for DRAP otherwise. */
9838 static rtx
9840 {
9845 {
9846 /* Assign DRAP to vDRAP and returns vDRAP */
9848 rtx drap_vreg;
9849 rtx arg_ptr;
9862 {
9865 }
9867 }
9868 else
9870 }
9872 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
9874 static rtx
9876 {
9878 }
9881 rtx reg;
9883 };
9885 /* Return a short-lived scratch register for use on function entry.
9886 In 32-bit mode, it is valid only after the registers are saved
9887 in the prologue. This register must be released by means of
9888 release_scratch_register_on_entry once it is dead. */
9890 static void
9892 {
9898 {
9899 /* We always use R11 in 64-bit mode. */
9901 }
9902 else
9903 {
9905 bool fastcall_p
9907 bool thiscall_p
9911 int drap_regno
9914 /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
9915 for the static chain register. */
9919 /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
9920 for the static chain register. */
9925 /* ecx is the static chain register. */
9927 && !static_chain_p
9932 /* esi is the static chain register. */
9938 else
9939 {
9942 }
9943 }
9947 {
9950 }
9951 }
9953 /* Release a scratch register obtained from the preceding function. */
9955 static void
9957 {
9959 {
9963 /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */
9969 }
9970 }
9972 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
9974 /* Emit code to adjust the stack pointer by SIZE bytes while probing it. */
9976 static void
9978 {
9979 /* We skip the probe for the first interval + a small dope of 4 words and
9980 probe that many bytes past the specified size to maintain a protection
9981 area at the botton of the stack. */
9985 /* See if we have a constant small number of probes to generate. If so,
9986 that's the easy case. The run-time loop is made up of 11 insns in the
9987 generic case while the compile-time loop is made up of 3+2*(n-1) insns
9988 for n # of intervals. */
9990 {
9994 /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
9995 values of N from 1 until it exceeds SIZE. If only one probe is
9996 needed, this will not generate any code. Then adjust and probe
9997 to PROBE_INTERVAL + SIZE. */
9999 {
10001 {
10004 }
10005 else
10012 }
10016 else
10024 /* Adjust back to account for the additional first interval. */
10028 }
10030 /* Otherwise, do the same as above, but in a loop. Note that we must be
10031 extra careful with variables wrapping around because we might be at
10032 the very top (or the very bottom) of the address space and we have
10033 to be able to handle this case properly; in particular, we use an
10034 equality test for the loop condition. */
10035 else
10036 {
10037 HOST_WIDE_INT rounded_size;
10043 /* Step 1: round SIZE to the previous multiple of the interval. */
10048 /* Step 2: compute initial and final value of the loop counter. */
10050 /* SP = SP_0 + PROBE_INTERVAL. */
10055 /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
10059 stack_pointer_rtx)));
10062 /* Step 3: the loop
10064 while (SP != LAST_ADDR)
10065 {
10066 SP = SP + PROBE_INTERVAL
10067 probe at SP
10068 }
10070 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
10071 values of N from 1 until it is equal to ROUNDED_SIZE. */
10076 /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
10077 assert at compile-time that SIZE is equal to ROUNDED_SIZE. */
10080 {
10085 }
10087 /* Adjust back to account for the additional first interval. */
10093 }
10097 /* Even if the stack pointer isn't the CFA register, we need to correctly
10098 describe the adjustments made to it, in particular differentiate the
10099 frame-related ones from the frame-unrelated ones. */
10101 {
10114 }
10116 /* Make sure nothing is scheduled before we are done. */
10118 }
10120 /* Adjust the stack pointer up to REG while probing it. */
10124 {
10134 /* Jump to END_LAB if SP == LAST_ADDR. */
10142 /* SP = SP + PROBE_INTERVAL. */
10146 /* Probe at SP. */
10157 }
10159 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
10160 inclusive. These are offsets from the current stack pointer. */
10162 static void
10164 {
10165 /* See if we have a constant small number of probes to generate. If so,
10166 that's the easy case. The run-time loop is made up of 7 insns in the
10167 generic case while the compile-time loop is made up of n insns for n #
10168 of intervals. */
10170 {
10171 HOST_WIDE_INT i;
10173 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
10174 it exceeds SIZE. If only one probe is needed, this will not
10175 generate any code. Then probe at FIRST + SIZE. */
10182 }
10184 /* Otherwise, do the same as above, but in a loop. Note that we must be
10185 extra careful with variables wrapping around because we might be at
10186 the very top (or the very bottom) of the address space and we have
10187 to be able to handle this case properly; in particular, we use an
10188 equality test for the loop condition. */
10189 else
10190 {
10197 /* Step 1: round SIZE to the previous multiple of the interval. */
10202 /* Step 2: compute initial and final value of the loop counter. */
10204 /* TEST_OFFSET = FIRST. */
10207 /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
10211 /* Step 3: the loop
10213 while (TEST_ADDR != LAST_ADDR)
10214 {
10215 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
10216 probe at TEST_ADDR
10217 }
10219 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
10220 until it is equal to ROUNDED_SIZE. */
10225 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
10226 that SIZE is equal to ROUNDED_SIZE. */
10231 stack_pointer_rtx,
10236 }
10238 /* Make sure nothing is scheduled before we are done. */
10240 }
10242 /* Probe a range of stack addresses from REG to END, inclusive. These are
10243 offsets from the current stack pointer. */
10247 {
10257 /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */
10265 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
10269 /* Probe at TEST_ADDR. */
10282 }
10284 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
10285 to be generated in correct form. */
10286 static void
10288 {
10289 /* Check if stack realign is really needed after reload, and
10290 stores result in cfun */
10291 unsigned int incoming_stack_boundary
10300 {
10301 /* After stack_realign_needed is finalized, we can't no longer
10302 change it. */
10305 }
10307 /* If the only reason for frame_pointer_needed is that we conservatively
10308 assumed stack realignment might be needed, but in the end nothing that
10309 needed the stack alignment had been spilled, clear frame_pointer_needed
10310 and say we don't need stack realignment. */
10313 && frame_pointer_needed
10315 && flag_omit_frame_pointer
10317 && !ix86_current_function_calls_tls_descriptor
10326 {
10328 basic_block bb;
10335 HARD_FRAME_POINTER_REGNUM);
10337 {
10338 rtx insn;
10342 set_up_by_prologue))
10343 {
10347 }
10348 }
10362 }
10366 }
10368 /* Expand the prologue into a bunch of separate insns. */
10370 void
10372 {
10377 HOST_WIDE_INT allocate;
10383 /* DRAP should not coexist with stack_realign_fp */
10388 /* Initialize CFA state for before the prologue. */
10392 /* Track SP offset to the CFA. We continue tracking this after we've
10393 swapped the CFA register away from SP. In the case of re-alignment
10394 this is fudged; we're interested to offsets within the local frame. */
10401 {
10402 /* We should have already generated an error for any use of
10403 ms_hook on a nested function. */
10406 /* Check if profiling is active and we shall use profiling before
10407 prologue variant. If so sorry. */
10412 /* In ix86_asm_output_function_label we emitted:
10413 8b ff movl.s %edi,%edi
10414 55 push %ebp
10415 8b ec movl.s %esp,%ebp
10417 This matches the hookable function prologue in Win32 API
10418 functions in Microsoft Windows XP Service Pack 2 and newer.
10419 Wine uses this to enable Windows apps to hook the Win32 API
10420 functions provided by Wine.
10422 What that means is that we've already set up the frame pointer. */
10426 {
10429 /* We've decided to use the frame pointer already set up.
10430 Describe this to the unwinder by pretending that both
10431 push and mov insns happen right here.
10433 Putting the unwind info here at the end of the ms_hook
10434 is done so that we can make absolutely certain we get
10435 the required byte sequence at the start of the function,
10436 rather than relying on an assembler that can produce
10437 the exact encoding required.
10439 However it does mean (in the unpatched case) that we have
10440 a 1 insn window where the asynchronous unwind info is
10441 incorrect. However, if we placed the unwind info at
10442 its correct location we would have incorrect unwind info
10443 in the patched case. Which is probably all moot since
10444 I don't expect Wine generates dwarf2 unwind info for the
10445 system libraries that use this feature. */
10451 stack_pointer_rtx);
10459 /* Note that gen_push incremented m->fs.cfa_offset, even
10460 though we didn't emit the push insn here. */
10464 }
10465 else
10466 {
10467 /* The frame pointer is not needed so pop %ebp again.
10468 This leaves us with a pristine state. */
10470 }
10471 }
10473 /* The first insn of a function that accepts its static chain on the
10474 stack is to push the register that would be filled in by a direct
10475 call. This insn will be skipped by the trampoline. */
10477 {
10481 /* We don't want to interpret this push insn as a register save,
10482 only as a stack adjustment. The real copy of the register as
10483 a save will be done later, if needed. */
10488 }
10490 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
10491 of DRAP is needed and stack realignment is really needed after reload */
10493 {
10496 /* Only need to push parameter pointer reg if it is caller saved. */
10498 {
10499 /* Push arg pointer reg */
10502 }
10504 /* Grab the argument pointer. */
10511 /* Align the stack. */
10513 stack_pointer_rtx,
10517 /* Replicate the return address on the stack so that return
10518 address can be reached via (argp - 1) slot. This is needed
10519 to implement macro RETURN_ADDR_RTX and intrinsic function
10520 expand_builtin_return_addr etc. */
10526 /* For the purposes of frame and register save area addressing,
10527 we've started over with a new frame. */
10530 }
10536 {
10537 /* Note: AT&T enter does NOT have reversed args. Enter is probably
10538 slower on all targets. Also sdb doesn't like it. */
10542 /* Push registers now, before setting the frame pointer
10543 on SEH target. */
10545 && TARGET_SEH
10547 {
10551 }
10554 {
10562 }
10563 }
10566 {
10567 /* If saving registers via PUSH, do so now. */
10569 {
10573 }
10575 /* When using red zone we may start register saving before allocating
10576 the stack frame saving one cycle of the prologue. However, avoid
10577 doing this if we have to probe the stack; at least on x86_64 the
10578 stack probe can turn into a call that clobbers a red zone location. */
10580 && (! TARGET_STACK_PROBE
10582 {
10585 }
10586 }
10589 {
10593 /* The computation of the size of the re-aligned stack frame means
10594 that we must allocate the size of the register save area before
10595 performing the actual alignment. Otherwise we cannot guarantee
10596 that there's enough storage above the realignment point. */
10603 /* Align the stack. */
10605 stack_pointer_rtx,
10608 /* For the purposes of register save area addressing, the stack
10609 pointer is no longer valid. As for the value of sp_offset,
10610 see ix86_compute_frame_layout, which we need to match in order
10611 to pass verification of stack_pointer_offset at the end. */
10614 }
10619 {
10620 /* We start to count from ARG_POINTER. */
10623 /* If it was realigned, take into account the fake frame. */
10625 {
10632 /* This over-estimates by 1 minimal-stack-alignment-unit but
10633 mitigates that by counting in the new return address slot. */
10634 current_function_dynamic_stack_size
10636 }
10639 }
10641 /* On SEH target with very large frame size, allocate an area to save
10642 SSE registers (as the very large allocation won't be described). */
10646 {
10647 HOST_WIDE_INT sse_size =
10652 /* No need to do stack checking as the area will be immediately
10653 written. */
10660 }
10662 /* The stack has already been decremented by the instruction calling us
10663 so probe if the size is non-negative to preserve the protection area. */
10665 {
10666 /* We expect the registers to be saved when probes are used. */
10670 {
10673 }
10674 else
10675 {
10683 else
10685 }
10686 }
10689 ;
10692 {
10696 }
10697 else
10698 {
10711 /* Note that SEH directives need to continue tracking the stack
10712 pointer even after the frame pointer has been set up. */
10714 {
10718 {
10722 }
10723 }
10726 {
10731 {
10735 }
10736 }
10741 /* Use the fact that AX still contains ALLOCATE. */
10743 ? gen_pro_epilogue_adjust_stack_di_sub
10750 {
10758 }
10762 {
10769 }
10771 {
10776 }
10777 }
10780 /* If we havn't already set up the frame pointer, do so now. */
10782 {
10794 }
10802 /* We don't use pic-register for pe-coff target. */
10804 && !TARGET_PECOFF
10807 {
10814 }
10817 {
10819 {
10821 {
10831 label));
10835 }
10836 else
10838 }
10839 else
10840 {
10844 }
10845 }
10847 /* In the pic_reg_used case, make sure that the got load isn't deleted
10848 when mcount needs it. Blockage to avoid call movement across mcount
10849 call is emitted in generic code after the NOTE_INSN_PROLOGUE_END
10850 note. */
10855 {
10856 /* vDRAP is setup but after reload it turns out stack realign
10857 isn't necessary, here we will emit prologue to setup DRAP
10858 without stack realign adjustment */
10861 }
10863 /* Prevent instructions from being scheduled into register save push
10864 sequence when access to the redzone area is done through frame pointer.
10865 The offset between the frame pointer and the stack pointer is calculated
10866 relative to the value of the stack pointer at the end of the function
10867 prologue, and moving instructions that access redzone area via frame
10868 pointer inside push sequence violates this assumption. */
10872 /* Emit cld instruction if stringops are used in the function. */
10876 /* SEH requires that the prologue end within 256 bytes of the start of
10877 the function. Prevent instruction schedules that would extend that.
10878 Further, prevent alloca modifications to the stack pointer from being
10879 combined with prologue modifications. */
10882 }
10884 /* Emit code to restore REG using a POP insn. */
10886 static void
10888 {
10897 {
10898 /* Previously we'd represented the CFA as an expression
10899 like *(%ebp - 8). We've just popped that value from
10900 the stack, which means we need to reset the CFA to
10901 the drap register. This will remain until we restore
10902 the stack pointer. */
10906 /* This means that the DRAP register is valid for addressing too. */
10909 }
10912 {
10919 }
10921 /* When the frame pointer is the CFA, and we pop it, we are
10922 swapping back to the stack pointer as the CFA. This happens
10923 for stack frames that don't allocate other data, so we assume
10924 the stack pointer is now pointing at the return address, i.e.
10925 the function entry state, which makes the offset be 1 word. */
10927 {
10930 {
10938 }
10939 }
10940 }
10942 /* Emit code to restore saved registers using POP insns. */
10944 static void
10946 {
10952 }
10954 /* Emit code and notes for the LEAVE instruction. */
10956 static void
10958 {
10970 {
10978 }
10981 }
10983 /* Emit code to restore saved registers using MOV insns.
10984 First register is restored from CFA - CFA_OFFSET. */
10985 static void
10988 {
10994 {
11003 {
11004 /* Previously we'd represented the CFA as an expression
11005 like *(%ebp - 8). We've just popped that value from
11006 the stack, which means we need to reset the CFA to
11007 the drap register. This will remain until we restore
11008 the stack pointer. */
11012 /* This means that the DRAP register is valid for addressing. */
11014 }
11015 else
11019 }
11020 }
11022 /* Emit code to restore saved registers using MOV insns.
11023 First register is restored from CFA - CFA_OFFSET. */
11024 static void
11027 {
11032 {
11034 rtx mem;
11044 }
11045 }
11047 /* Restore function stack, frame, and registers. */
11049 void
11051 {
11067 /* The FP must be valid if the frame pointer is present. */
11072 /* We must have *some* valid pointer to the stack frame. */
11075 /* The DRAP is never valid at this point. */
11078 /* See the comment about red zone and frame
11079 pointer usage in ix86_expand_prologue. */
11086 /* Determine the CFA offset of the end of the red-zone. */
11089 {
11090 /* The red-zone begins below the return address. */
11093 /* When the register save area is in the aligned portion of
11094 the stack, determine the maximum runtime displacement that
11095 matches up with the aligned frame. */
11099 }
11101 /* Special care must be taken for the normal return case of a function
11102 using eh_return: the eax and edx registers are marked as saved, but
11103 not restored along this path. Adjust the save location to match. */
11107 /* EH_RETURN requires the use of moves to function properly. */
11110 /* SEH requires the use of pops to identify the epilogue. */
11113 /* If we're only restoring one register and sp is not valid then
11114 using a move instruction to restore the register since it's
11115 less work than reloading sp and popping the register. */
11128 && TARGET_USE_LEAVE
11132 else
11136 {
11137 /* Ensure that the entire register save area is addressable via
11138 the stack pointer, if we will restore via sp. */
11143 {
11147 style,
11149 }
11150 }
11152 /* If there are any SSE registers to restore, then we have to do it
11153 via moves, since there's obviously no pop for SSE regs. */
11159 {
11160 rtx t;
11165 /* eh_return epilogues need %ecx added to the stack pointer. */
11167 {
11170 /* Stack align doesn't work with eh_return. */
11172 /* Neither does regparm nested functions. */
11176 {
11184 /* Note that we use SA as a temporary CFA, as the return
11185 address is at the proper place relative to it. We
11186 pretend this happens at the FP restore insn because
11187 prior to this insn the FP would be stored at the wrong
11188 offset relative to SA, and after this insn we have no
11189 other reasonable register to use for the CFA. We don't
11190 bother resetting the CFA to the SP for the duration of
11191 the return insn. */
11204 }
11205 else
11206 {
11214 {
11218 UNITS_PER_WORD));
11220 }
11221 }
11224 }
11225 }
11226 else
11227 {
11228 /* SEH requires that the function end with (1) a stack adjustment
11229 if necessary, (2) a sequence of pops, and (3) a return or
11230 jump instruction. Prevent insns from the function body from
11231 being scheduled into this sequence. */
11233 {
11234 /* Prevent a catch region from being adjacent to the standard
11235 epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
11236 several other flags that would be interesting to test are
11237 not yet set up. */
11240 else
11242 }
11244 /* First step is to deallocate the stack frame so that we can
11245 pop the registers. Also do it on SEH target for very large
11246 frame as the emitted instructions aren't allowed by the ABI in
11247 epilogues. */
11249 || (TARGET_SEH
11252 {
11257 }
11259 {
11263 style,
11265 }
11268 }
11270 /* If we used a stack pointer and haven't already got rid of it,
11271 then do so now. */
11273 {
11274 /* If the stack pointer is valid and pointing at the frame
11275 pointer store address, then we only need a pop. */
11278 /* Leave results in shorter dependency chains on CPUs that are
11279 able to grok it fast. */
11284 else
11285 {
11287 hard_frame_pointer_rtx,
11290 }
11291 }
11294 {
11296 rtx insn;
11322 }
11324 /* At this point the stack pointer must be valid, and we must have
11325 restored all of the registers. We may not have deallocated the
11326 entire stack frame. We've delayed this until now because it may
11327 be possible to merge the local stack deallocation with the
11328 deallocation forced by ix86_static_chain_on_stack. */
11333 {
11337 }
11338 else
11341 /* Sibcall epilogues don't want a return instruction. */
11343 {
11346 }
11349 {
11352 /* i386 can only pop 64K bytes. If asked to pop more, pop return
11353 address, do explicit add, and jump indirectly to the caller. */
11356 {
11358 rtx insn;
11360 /* There is no "pascal" calling convention in any 64bit ABI. */
11376 }
11377 else
11379 }
11380 else
11383 /* Restore the state back to the state from the prologue,
11384 so that it's correct for the next epilogue. */
11386 }
11388 /* Reset from the function's potential modifications. */
11390 static void
11392 HOST_WIDE_INT size ATTRIBUTE_UNUSED)
11393 {
11396 #if TARGET_MACHO
11397 /* Mach-O doesn't support labels at the end of objects, so if
11398 it looks like we might want one, insert a NOP. */
11399 {
11405 {
11406 /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
11407 notes only, instead set their CODE_LABEL_NUMBER to -1,
11408 otherwise there would be code generation differences
11409 in between -g and -g0. */
11413 }
11423 }
11424 #endif
11426 }
11428 /* Return a scratch register to use in the split stack prologue. The
11429 split stack prologue is used for -fsplit-stack. It is the first
11430 instructions in the function, even before the regular prologue.
11431 The scratch register can be any caller-saved register which is not
11432 used for parameters or for the static chain. */
11434 static unsigned int
11436 {
11439 else
11440 {
11453 {
11455 {
11459 }
11461 }
11463 {
11467 }
11469 {
11472 else
11473 {
11475 {
11479 }
11481 }
11482 }
11483 else
11484 {
11485 /* FIXME: We could make this work by pushing a register
11486 around the addition and comparison. */
11489 }
11490 }
11491 }
11493 /* A SYMBOL_REF for the function which allocates new stackspace for
11494 -fsplit-stack. */
11498 /* A SYMBOL_REF for the more stack function when using the large
11499 model. */
11503 /* Handle -fsplit-stack. These are the first instructions in the
11504 function, even before the regular prologue. */
11506 void
11508 {
11510 HOST_WIDE_INT allocate;
11515 rtx fn;
11523 /* This is the label we will branch to if we have enough stack
11524 space. We expect the basic block reordering pass to reverse this
11525 branch if optimizing, so that we branch in the unlikely case. */
11528 /* We need to compare the stack pointer minus the frame size with
11529 the stack boundary in the TCB. The stack boundary always gives
11530 us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
11531 can compare directly. Otherwise we need to do an addition. */
11534 UNSPEC_STACK_CHECK);
11539 else
11540 {
11542 rtx offset;
11544 /* We need a scratch register to hold the stack pointer minus
11545 the required frame size. Since this is the very start of the
11546 function, the scratch register can be any caller-saved
11547 register which is not used for parameters. */
11554 {
11555 /* We don't use ix86_gen_add3 in this case because it will
11556 want to split to lea, but when not optimizing the insn
11557 will not be split after this point. */
11560 offset)));
11561 }
11562 else
11563 {
11566 stack_pointer_rtx));
11567 }
11569 }
11575 /* Mark the jump as very likely to be taken. */
11583 /* Get more stack space. We pass in the desired stack space and the
11584 size of the arguments to copy to the new stack. In 32-bit mode
11585 we push the parameters; __morestack will return on a new stack
11586 anyhow. In 64-bit mode we pass the parameters in r10 and
11587 r11. */
11592 {
11598 /* If this function uses a static chain, it will be in %r10.
11599 Preserve it across the call to __morestack. */
11601 {
11602 rtx rax;
11607 }
11611 {
11612 HOST_WIDE_INT argval;
11615 /* When using the large model we need to load the address
11616 into a register, and we've run out of registers. So we
11617 switch to a different calling convention, and we call a
11618 different function: __morestack_large. We pass the
11619 argument size in the upper 32 bits of r10 and pass the
11620 frame size in the lower 32 bits. */
11625 split_stack_fn_large =
11629 {
11639 UNSPEC_GOT);
11645 }
11646 else
11653 }
11654 else
11655 {
11659 }
11662 }
11663 else
11664 {
11667 }
11673 /* In order to make call/return prediction work right, we now need
11674 to execute a return instruction. See
11675 libgcc/config/i386/morestack.S for the details on how this works.
11677 For flow purposes gcc must not see this as a return
11678 instruction--we need control flow to continue at the subsequent
11679 label. Therefore, we use an unspec. */
11683 /* If we are in 64-bit mode and this function uses a static chain,
11684 we saved %r10 in %rax before calling _morestack. */
11689 /* If this function calls va_start, we need to store a pointer to
11690 the arguments on the old stack, because they may not have been
11691 all copied to the new stack. At this point the old stack can be
11692 found at the frame pointer value used by __morestack, because
11693 __morestack has set that up before calling back to us. Here we
11694 store that pointer in a scratch register, and in
11695 ix86_expand_prologue we store the scratch register in a stack
11696 slot. */
11698 {
11700 rtx frame_reg;
11707 /* 64-bit:
11708 fp -> old fp value
11709 return address within this function
11710 return address of caller of this function
11711 stack arguments
11712 So we add three words to get to the stack arguments.
11714 32-bit:
11715 fp -> old fp value
11716 return address within this function
11717 first argument to __morestack
11718 second argument to __morestack
11719 return address of caller of this function
11720 stack arguments
11721 So we add five words to get to the stack arguments.
11722 */
11733 }
11738 /* If this function calls va_start, we now have to set the scratch
11739 register for the case where we do not call __morestack. In this
11740 case we need to set it based on the stack pointer. */
11742 {
11749 }
11750 }
11752 /* We may have to tell the dataflow pass that the split stack prologue
11753 is initializing a scratch register. */
11755 static void
11757 {
11759 {
11762 }
11763 }
11764 \f
11765 /* Determine if op is suitable SUBREG RTX for address. */
11767 static bool
11769 {
11780 /* Don't allow SUBREGs that span more than a word. It can lead to spill
11781 failures when the register is one word out of a two word structure. */
11785 /* Allow only SUBREGs of non-eliminable hard registers. */
11787 }
11789 /* Extract the parts of an RTL expression that is a valid memory address
11790 for an instruction. Return 0 if the structure of the address is
11791 grossly off. Return -1 if the address contains ASHIFT, so it is not
11792 strictly valid, but still used for computing length of lea instruction. */
11794 int
11796 {
11801 rtx tmp;
11805 /* Allow zero-extended SImode addresses,
11806 they will be emitted with addr32 prefix. */
11808 {
11811 {
11815 }
11818 {
11825 }
11826 }
11828 /* Allow SImode subregs of DImode addresses,
11829 they will be emitted with addr32 prefix. */
11831 {
11834 {
11838 }
11839 }
11844 {
11847 else
11849 }
11851 {
11856 do
11857 {
11862 }
11869 {
11872 {
11897 /* FALLTHRU */
11901 && TARGET_TLS_DIRECT_SEG_REFS
11904 else
11911 /* FALLTHRU */
11918 else
11933 }
11934 }
11935 }
11937 {
11940 }
11942 {
11943 /* We're called for lea too, which implements ashift on occasion. */
11953 }
11955 {
11959 /* Constant addresses are sign extended to 64bit, we have to
11960 prevent addresses from 0x80000000 to 0xffffffff in x32 mode. */
11966 }
11967 else
11971 {
11973 ;
11976 ;
11977 else
11979 }
11981 /* Address override works only on the (%reg) part of %fs:(%reg). */
11987 /* Extract the integral value of scale. */
11989 {
11993 }
11998 /* Avoid useless 0 displacement. */
12002 /* Allow arg pointer and stack pointer as index if there is not scaling. */
12007 {
12008 rtx tmp;
12011 }
12013 /* Special case: %ebp cannot be encoded as a base without a displacement.
12014 Similarly %r13. */
12016 && base_reg
12025 /* Special case: on K6, [%esi] makes the instruction vector decoded.
12026 Avoid this by transforming to [%esi+0].
12027 Reload calls address legitimization without cfun defined, so we need
12028 to test cfun for being non-NULL. */
12034 /* Special case: encode reg+reg instead of reg*2. */
12038 /* Special case: scaling cannot be encoded without base or displacement. */
12049 }
12050 \f
12051 /* Return cost of the memory address x.
12052 For i386, it is better to use a complex address than let gcc copy
12053 the address into a reg and make a new pseudo. But not if the address
12054 requires to two regs - that would mean more pseudos with longer
12055 lifetimes. */
12056 static int
12058 addr_space_t as ATTRIBUTE_UNUSED,
12060 {
12072 /* Attempt to minimize number of registers in the address. */
12078 cost++;
12085 cost++;
12087 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
12088 since it's predecode logic can't detect the length of instructions
12089 and it degenerates to vector decoded. Increase cost of such
12090 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
12091 to split such addresses or even refuse such addresses at all.
12093 Following addressing modes are affected:
12094 [base+scale*index]
12095 [scale*index+disp]
12096 [base+index]
12098 The first and last case may be avoidable by explicitly coding the zero in
12099 memory address, but I don't have AMD-K6 machine handy to check this
12100 theory. */
12109 }
12110 \f
12111 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
12112 this is used for to form addresses to local data when -fPIC is in
12113 use. */
12115 static bool
12117 {
12120 }
12122 /* Determine if a given RTX is a valid constant. We already know this
12123 satisfies CONSTANT_P. */
12125 static bool
12127 {
12129 {
12134 {
12138 }
12143 /* Only some unspecs are valid as "constants". */
12146 {
12162 }
12164 /* We must have drilled down to a symbol. */
12169 /* FALLTHRU */
12172 /* TLS symbols are never valid. */
12176 /* DLLIMPORT symbols are never valid. */
12181 #if TARGET_MACHO
12182 /* mdynamic-no-pic */
12185 #endif
12201 }
12203 /* Otherwise we handle everything else in the move patterns. */
12205 }
12207 /* Determine if it's legal to put X into the constant pool. This
12208 is not possible for the address of thread-local symbols, which
12209 is checked above. */
12211 static bool
12213 {
12214 /* We can always put integral constants and vectors in memory. */
12216 {
12224 }
12226 }
12228 /* Nonzero if the symbol is marked as dllimport, or as stub-variable,
12229 otherwise zero. */
12231 static bool
12233 {
12239 }
12242 /* Nonzero if the constant value X is a legitimate general operand
12243 when generating PIC code. It is given that flag_pic is on and
12244 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
12246 bool
12248 {
12249 rtx inner;
12252 {
12259 /* Only some unspecs are valid as "constants". */
12262 {
12275 }
12276 /* FALLTHRU */
12284 }
12285 }
12287 /* Determine if a given CONST RTX is a valid memory displacement
12288 in PIC mode. */
12290 bool
12292 {
12295 /* In 64bit mode we can allow direct addresses of symbols and labels
12296 when they are not dynamic symbols. */
12298 {
12302 {
12326 /* FALLTHRU */
12329 /* TLS references should always be enclosed in UNSPEC.
12330 The dllimported symbol needs always to be resolved. */
12336 {
12344 /* Function-symbols need to be resolved only for
12345 large-model.
12346 For the small-model we don't need to resolve anything
12347 here. */
12352 /* Non-external symbols don't need to be resolved for
12353 large, and medium-model. */
12358 }
12367 }
12368 }
12374 {
12375 /* We are unsafe to allow PLUS expressions. This limit allowed distance
12376 of GOT tables. We should not need these anyway. */
12388 }
12392 {
12397 }
12406 {
12410 /* We need to check for both symbols and labels because VxWorks loads
12411 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
12412 details. */
12416 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
12417 While ABI specify also 32bit relocation but we don't produce it in
12418 small PIC model at all. */
12440 }
12443 }
12445 /* Our implementation of LEGITIMIZE_RELOAD_ADDRESS. Returns a value to
12446 replace the input X, or the original X if no replacement is called for.
12447 The output parameter *WIN is 1 if the calling macro should goto WIN,
12448 0 if it should not. */
12450 bool
12455 {
12456 /* Reload can generate:
12458 (plus:DI (plus:DI (unspec:DI [(const_int 0 [0])] UNSPEC_TP)
12459 (reg:DI 97))
12460 (reg:DI 2 cx))
12462 This RTX is rejected from ix86_legitimate_address_p due to
12463 non-strictness of base register 97. Following this rejection,
12464 reload pushes all three components into separate registers,
12465 creating invalid memory address RTX.
12467 Following code reloads only the invalid part of the
12468 memory address RTX. */
12474 {
12480 {
12485 }
12489 {
12494 }
12498 }
12501 }
12503 /* Recognizes RTL expressions that are valid memory addresses for an
12504 instruction. The MODE argument is the machine mode for the MEM
12505 expression that wants to use this address.
12507 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
12508 convert common non-canonical forms to canonical form so that they will
12509 be recognized. */
12511 static bool
12514 {
12517 HOST_WIDE_INT scale;
12520 /* Decomposition failed. */
12528 /* Validate base register. */
12530 {
12531 rtx reg;
12537 else
12538 /* Base is not a register. */
12546 /* Base is not valid. */
12548 }
12550 /* Validate index register. */
12552 {
12553 rtx reg;
12559 else
12560 /* Index is not a register. */
12568 /* Index is not valid. */
12570 }
12572 /* Index and base should have the same mode. */
12577 /* Validate scale factor. */
12579 {
12581 /* Scale without index. */
12585 /* Scale is not a valid multiplier. */
12587 }
12589 /* Validate displacement. */
12591 {
12596 {
12597 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
12598 used. While ABI specify also 32bit relocations, we don't produce
12599 them at all and use IP relative instead. */
12606 /* 64bit address unspec. */
12626 /* Invalid address unspec. */
12628 }
12631 && (flag_pic
12632 || (TARGET_MACHO
12633 #if TARGET_MACHO
12634 && MACHOPIC_INDIRECT
12636 #endif
12637 )))
12638 {
12640 is_legitimate_pic:
12642 {
12643 /* foo@dtpoff(%rX) is ok. */
12650 /* Non-constant pic memory reference. */
12652 }
12655 /* Displacement is an invalid pic construct. */
12657 #if TARGET_MACHO
12660 /* displacment must be referenced via non_lazy_pointer */
12662 #endif
12664 /* This code used to verify that a symbolic pic displacement
12665 includes the pic_offset_table_rtx register.
12667 While this is good idea, unfortunately these constructs may
12668 be created by "adds using lea" optimization for incorrect
12669 code like:
12671 int a;
12672 int foo(int i)
12673 {
12674 return *(&a+i);
12675 }
12677 This code is nonsensical, but results in addressing
12678 GOT table with pic_offset_table_rtx base. We can't
12679 just refuse it easily, since it gets matched by
12680 "addsi3" pattern, that later gets split to lea in the
12681 case output register differs from input. While this
12682 can be handled by separate addsi pattern for this case
12683 that never results in lea, this seems to be easier and
12684 correct fix for crash to disable this test. */
12685 }
12692 /* Displacement is not constant. */
12696 /* Displacement is out of range. */
12698 }
12700 /* Everything looks valid. */
12702 }
12704 /* Determine if a given RTX is a valid constant address. */
12706 bool
12708 {
12710 }
12711 \f
12712 /* Return a unique alias set for the GOT. */
12714 static alias_set_type
12716 {
12721 }
12723 /* Return a legitimate reference for ORIG (an address) using the
12724 register REG. If REG is 0, a new pseudo is generated.
12726 There are two types of references that must be handled:
12728 1. Global data references must load the address from the GOT, via
12729 the PIC reg. An insn is emitted to do this load, and the reg is
12730 returned.
12732 2. Static data references, constant pool addresses, and code labels
12733 compute the address as an offset from the GOT, whose base is in
12734 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
12735 differentiate them from global data objects. The returned
12736 address is the PIC reg + an unspec constant.
12738 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
12739 reg also appears in the address. */
12741 static rtx
12743 {
12747 #if TARGET_MACHO
12749 {
12752 /* Use the generic Mach-O PIC machinery. */
12754 }
12755 #endif
12758 {
12762 }
12768 {
12769 rtx tmpreg;
12770 /* This symbol may be referenced via a displacement from the PIC
12771 base address (@GOTOFF). */
12778 {
12780 UNSPEC_GOTOFF);
12782 }
12783 else
12788 else
12793 {
12797 }
12798 else
12800 }
12802 {
12803 /* This symbol may be referenced via a displacement from the PIC
12804 base address (@GOTOFF). */
12811 {
12813 UNSPEC_GOTOFF);
12815 }
12816 else
12822 {
12825 }
12826 }
12828 /* We can't use @GOTOFF for text labels on VxWorks;
12829 see gotoff_operand. */
12831 {
12836 /* For x64 PE-COFF there is no GOT table. So we use address
12837 directly. */
12839 {
12847 }
12849 {
12857 /* Use directly gen_movsi, otherwise the address is loaded
12858 into register for CSE. We don't want to CSE this addresses,
12859 instead we CSE addresses from the GOT table, so skip this. */
12862 }
12863 else
12864 {
12865 /* This symbol must be referenced via a load from the
12866 Global Offset Table (@GOT). */
12882 }
12883 }
12884 else
12885 {
12888 {
12890 {
12893 }
12894 else
12896 }
12898 {
12901 /* We must match stuff we generate before. Assume the only
12902 unspecs that can get here are ours. Not that we could do
12903 anything with them anyway.... */
12909 }
12911 {
12914 /* Check first to see if this is a constant offset from a @GOTOFF
12915 symbol reference. */
12918 {
12920 {
12924 UNSPEC_GOTOFF);
12930 {
12933 }
12934 }
12935 else
12936 {
12939 {
12943 }
12944 }
12945 }
12946 else
12947 {
12950 new_rtx
12954 {
12957 {
12960 new_rtx
12962 }
12963 else
12965 }
12966 else
12967 {
12970 {
12973 }
12975 }
12976 }
12977 }
12978 }
12980 }
12981 \f
12982 /* Load the thread pointer. If TO_REG is true, force it into a register. */
12984 static rtx
12986 {
12990 {
12995 }
13001 }
13003 /* Construct the SYMBOL_REF for the tls_get_addr function. */
13007 static rtx
13009 {
13011 {
13017 }
13020 {
13022 UNSPEC_PLTOFF);
13025 }
13028 }
13030 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
13034 rtx
13036 {
13038 {
13039 ix86_tls_module_base_symbol
13044 }
13047 }
13049 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
13050 false if we expect this to be used for a memory address and true if
13051 we expect to load the address into a register. */
13053 static rtx
13055 {
13062 {
13067 {
13070 else
13071 {
13074 }
13075 }
13078 {
13081 else
13091 }
13092 else
13093 {
13097 {
13099 rtx insns;
13102 emit_call_insn
13112 }
13113 else
13115 }
13122 {
13125 else
13126 {
13129 }
13130 }
13133 {
13138 else
13144 }
13145 else
13146 {
13150 {
13155 emit_call_insn
13160 /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
13161 share the LD_BASE result with other LD model accesses. */
13163 UNSPEC_TLS_LD_BASE);
13167 }
13168 else
13170 }
13178 {
13185 }
13190 {
13192 {
13193 /* The Sun linker took the AMD64 TLS spec literally
13194 and can only handle %rax as destination of the
13195 initial executable code sequence. */
13200 }
13202 /* Generate DImode references to avoid %fs:(%reg32)
13203 problems and linker IE->LE relaxation bug. */
13207 }
13209 {
13214 }
13216 {
13220 }
13221 else
13222 {
13225 }
13235 {
13240 }
13241 else
13242 {
13246 }
13256 {
13260 }
13261 else
13262 {
13266 }
13271 }
13274 }
13276 /* Create or return the unique __imp_DECL dllimport symbol corresponding
13277 to symbol DECL if BEIMPORT is true. Otherwise create or return the
13278 unique refptr-DECL symbol corresponding to symbol DECL. */
13281 htab_t dllimport_map;
13283 static tree
13285 {
13292 tree to;
13293 rtx rtl;
13320 else
13333 {
13335 #ifdef SUB_TARGET_RECORD_STUB
13337 #endif
13338 }
13347 }
13349 /* Expand SYMBOL into its corresponding far-addresse symbol.
13350 WANT_REG is true if we require the result be a register. */
13352 static rtx
13354 {
13355 tree imp_decl;
13356 rtx x;
13365 }
13367 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
13368 true if we require the result be a register. */
13370 static rtx
13372 {
13373 tree imp_decl;
13374 rtx x;
13383 }
13385 /* Expand SYMBOL into its corresponding dllimport or refptr symbol. WANT_REG
13386 is true if we require the result be a register. */
13388 static rtx
13390 {
13395 {
13402 {
13405 }
13406 }
13422 {
13425 }
13427 }
13429 /* Try machine-dependent ways of modifying an illegitimate address
13430 to be legitimate. If we find one, return the new, valid address.
13431 This macro is used in only one place: `memory_address' in explow.c.
13433 OLDX is the address as it was before break_out_memory_refs was called.
13434 In some cases it is useful to look at this to decide what needs to be done.
13436 It is always safe for this macro to do nothing. It exists to recognize
13437 opportunities to optimize the output.
13439 For the 80386, we handle X+REG by loading X into a register R and
13440 using R+REG. R will go in a general reg and indexing will be used.
13441 However, if REG is a broken-out memory address or multiplication,
13442 nothing needs to be done because REG can certainly go in a general reg.
13444 When -fpic is used, special handling is needed for symbolic references.
13445 See comments by legitimize_pic_address in i386.c for details. */
13447 static rtx
13450 {
13461 {
13465 }
13468 {
13472 }
13477 #if TARGET_MACHO
13480 #endif
13482 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
13486 {
13491 }
13494 {
13495 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
13500 {
13506 }
13511 {
13517 }
13519 /* Put multiply first if it isn't already. */
13521 {
13526 }
13528 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
13529 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
13530 created by virtual register instantiation, register elimination, and
13531 similar optimizations. */
13533 {
13539 }
13541 /* Canonicalize
13542 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
13543 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
13548 {
13549 rtx constant;
13553 {
13556 }
13558 {
13561 }
13562 else
13566 {
13573 }
13574 }
13580 {
13583 }
13586 {
13589 }
13597 {
13600 }
13606 {
13610 {
13613 }
13617 }
13620 {
13624 {
13627 }
13631 }
13632 }
13635 }
13636 \f
13637 /* Print an integer constant expression in assembler syntax. Addition
13638 and subtraction are the only arithmetic that may appear in these
13639 expressions. FILE is the stdio stream to write to, X is the rtx, and
13640 CODE is the operand print code from the output string. */
13642 static void
13644 {
13648 {
13657 else
13658 {
13661 /* Mark the decl as referenced so that cgraph will
13662 output the function. */
13666 #if TARGET_MACHO
13670 #endif
13672 }
13680 /* FALLTHRU */
13691 /* This used to output parentheses around the expression,
13692 but that does not work on the 386 (either ATT or BSD assembler). */
13698 {
13699 /* We can use %d if the number is <32 bits and positive. */
13704 else
13706 }
13707 else
13708 /* We can't handle floating point constants;
13709 TARGET_PRINT_OPERAND must handle them. */
13714 /* Some assemblers need integer constants to appear first. */
13716 {
13720 }
13721 else
13722 {
13727 }
13742 {
13746 }
13751 {
13770 /* FIXME: This might be @TPOFF in Sun ld too. */
13779 else
13789 else
13795 #if TARGET_MACHO
13800 #endif
13804 }
13809 }
13810 }
13812 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
13813 We need to emit DTP-relative relocations. */
13815 static void ATTRIBUTE_UNUSED
13817 {
13822 {
13830 }
13831 }
13833 /* Return true if X is a representation of the PIC register. This copes
13834 with calls from ix86_find_base_term, where the register might have
13835 been replaced by a cselib value. */
13837 static bool
13839 {
13843 else
13845 }
13847 /* Helper function for ix86_delegitimize_address.
13848 Attempt to delegitimize TLS local-exec accesses. */
13850 static rtx
13852 {
13877 {
13882 }
13888 }
13890 /* In the name of slightly smaller debug output, and to cater to
13891 general assembler lossage, recognize PIC+GOTOFF and turn it back
13892 into a direct symbol reference.
13894 On Darwin, this is necessary to avoid a crash, because Darwin
13895 has a different PIC label for each routine but the DWARF debugging
13896 information is not associated with any particular routine, so it's
13897 necessary to remove references to the PIC label from RTL stored by
13898 the DWARF output code. */
13900 static rtx
13902 {
13904 /* addend is NULL or some rtx if x is something+GOTOFF where
13905 something doesn't include the PIC register. */
13907 /* reg_addend is NULL or a multiple of some register. */
13909 /* const_addend is NULL or a const_int. */
13911 /* This is the result, or NULL. */
13920 {
13927 {
13933 }
13940 {
13943 {
13948 }
13950 }
13955 /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
13956 and -mcmodel=medium -fpic. */
13957 }
13964 /* %ebx + GOT/GOTOFF */
13965 ;
13967 {
13968 /* %ebx + %reg * scale + GOT/GOTOFF */
13974 else
13975 {
13978 }
13979 }
13980 else
13986 {
13989 }
14010 {
14011 /* If the rest of original X doesn't involve the PIC register, add
14012 addend and subtract pic_offset_table_rtx. This can happen e.g.
14013 for code like:
14014 leal (%ebx, %ecx, 4), %ecx
14015 ...
14016 movl foo@GOTOFF(%ecx), %edx
14017 in which case we return (%ecx - %ebx) + foo. */
14020 pic_offset_table_rtx),
14021 result);
14022 else
14024 }
14026 {
14030 }
14032 }
14034 /* If X is a machine specific address (i.e. a symbol or label being
14035 referenced as a displacement from the GOT implemented using an
14036 UNSPEC), then return the base term. Otherwise return X. */
14038 rtx
14040 {
14041 rtx term;
14044 {
14058 }
14061 }
14062 \f
14063 static void
14066 {
14070 {
14073 }
14078 {
14081 {
14100 }
14104 {
14123 }
14130 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
14131 Those same assemblers have the same but opposite lossage on cmov. */
14134 else
14139 {
14152 }
14159 else
14164 {
14177 }
14184 else
14194 else
14205 }
14207 }
14209 /* Print the name of register X to FILE based on its machine mode and number.
14210 If CODE is 'w', pretend the mode is HImode.
14211 If CODE is 'b', pretend the mode is QImode.
14212 If CODE is 'k', pretend the mode is SImode.
14213 If CODE is 'q', pretend the mode is DImode.
14214 If CODE is 'x', pretend the mode is V4SFmode.
14215 If CODE is 't', pretend the mode is V8SFmode.
14216 If CODE is 'g', pretend the mode is V16SFmode.
14217 If CODE is 'h', pretend the reg is the 'high' byte register.
14218 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
14219 If CODE is 'd', duplicate the operand for AVX instruction.
14220 */
14222 void
14224 {
14233 {
14237 }
14264 else
14267 /* Irritatingly, AMD extended registers use different naming convention
14268 from the normal registers: "r%d[bwd]" */
14270 {
14275 {
14289 /* no suffix */
14294 }
14296 }
14300 {
14303 {
14306 }
14307 /* FALLTHRU */
14313 /* FALLTHRU */
14316 normal:
14331 {
14336 }
14339 {
14344 }
14348 }
14352 {
14355 else
14357 }
14358 }
14360 /* Locate some local-dynamic symbol still in use by this function
14361 so that we can print its name in some tls_local_dynamic_base
14362 pattern. */
14364 static int
14366 {
14371 {
14374 }
14377 }
14381 {
14382 rtx insn;
14393 }
14395 /* Meaning of CODE:
14396 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
14397 C -- print opcode suffix for set/cmov insn.
14398 c -- like C, but print reversed condition
14399 F,f -- likewise, but for floating-point.
14400 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
14401 otherwise nothing
14402 R -- print the prefix for register names.
14403 z -- print the opcode suffix for the size of the current operand.
14404 Z -- likewise, with special suffixes for x87 instructions.
14405 * -- print a star (in certain assembler syntax)
14406 A -- print an absolute memory reference.
14407 E -- print address with DImode register names if TARGET_64BIT.
14408 w -- print the operand as if it's a "word" (HImode) even if it isn't.
14409 s -- print a shift double count, followed by the assemblers argument
14410 delimiter.
14411 b -- print the QImode name of the register for the indicated operand.
14412 %b0 would print %al if operands[0] is reg 0.
14413 w -- likewise, print the HImode name of the register.
14414 k -- likewise, print the SImode name of the register.
14415 q -- likewise, print the DImode name of the register.
14416 x -- likewise, print the V4SFmode name of the register.
14417 t -- likewise, print the V8SFmode name of the register.
14418 g -- likewise, print the V16SFmode name of the register.
14419 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
14420 y -- print "st(0)" instead of "st" as a register.
14421 d -- print duplicated register operand for AVX instruction.
14422 D -- print condition for SSE cmp instruction.
14423 P -- if PIC, print an @PLT suffix.
14424 p -- print raw symbol name.
14425 X -- don't print any sort of PIC '@' suffix for a symbol.
14426 & -- print some in-use local-dynamic symbol name.
14427 H -- print a memory address offset by 8; used for sse high-parts
14428 Y -- print condition for XOP pcom* instruction.
14429 + -- print a branch hint as 'cs' or 'ds' prefix
14430 ; -- print a semicolon (after prefixes due to bug in older gas).
14431 ~ -- print "i" if TARGET_AVX2, "f" otherwise.
14432 @ -- print a segment register of thread base pointer load
14433 ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
14434 ! -- print MPX prefix for jxx/call/ret instructions if required.
14435 */
14437 void
14439 {
14441 {
14443 {
14446 {
14452 /* Intel syntax. For absolute addresses, registers should not
14453 be surrounded by braces. */
14455 {
14460 }
14465 }
14471 /* Wrap address in an UNSPEC to declare special handling. */
14509 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14514 {
14528 output_operand_lossage
14531 }
14534 #endif
14539 {
14540 /* Opcodes don't get size suffixes if using Intel opcodes. */
14545 {
14563 output_operand_lossage
14566 }
14567 }
14570 warning
14572 /* FALLTHRU */
14575 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
14580 {
14582 {
14584 #ifdef HAVE_AS_IX86_FILDS
14586 #endif
14594 #ifdef HAVE_AS_IX86_FILDQ
14596 #else
14598 #endif
14603 }
14604 }
14606 {
14607 /* 387 opcodes don't get size suffixes
14608 if the operands are registers. */
14613 {
14629 }
14630 }
14631 else
14632 {
14633 output_operand_lossage
14636 }
14638 output_operand_lossage
14659 {
14662 }
14667 {
14718 }
14722 /* Little bit of braindamage here. The SSE compare instructions
14723 does use completely different names for the comparisons that the
14724 fp conditional moves. */
14726 {
14729 {
14732 }
14738 {
14741 }
14747 {
14750 }
14759 {
14762 }
14768 {
14771 }
14777 {
14780 }
14791 }
14796 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14799 #endif
14804 {
14808 }
14812 file);
14817 {
14821 }
14822 /* It doesn't actually matter what mode we use here, as we're
14823 only going to use this for printing. */
14825 /* Output 'qword ptr' for intel assembler dialect. */
14834 #ifdef HAVE_AS_IX86_HLE
14836 #else
14838 #endif
14840 #ifdef HAVE_AS_IX86_HLE
14842 #else
14844 #endif
14845 /* We do not want to print value of the operand. */
14854 {
14859 else
14862 }
14865 {
14866 rtx x;
14875 {
14880 {
14882 bool cputaken
14885 /* Emit hints only in the case default branch prediction
14886 heuristics would fail. */
14888 {
14889 /* We use 3e (DS) prefix for taken branches and
14890 2e (CS) prefix for not taken branches. */
14893 else
14895 }
14896 }
14897 }
14899 }
14902 #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
14904 #endif
14911 /* The kernel uses a different segment register for performance
14912 reasons; a system call would not have to trash the userspace
14913 segment register, which would be expensive. */
14916 else
14936 }
14937 }
14943 {
14944 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
14947 {
14950 {
14959 else
14966 }
14968 /* Check for explicit size override (codes 'b', 'w', 'k',
14969 'q' and 'x') */
14983 }
14986 /* Avoid (%rip) for call operands. */
14992 else
14994 }
14997 {
14998 REAL_VALUE_TYPE r;
15006 /* Sign extend 32bit SFmode immediate to 8 bytes. */
15010 else
15012 }
15015 {
15016 REAL_VALUE_TYPE r;
15025 }
15027 /* These float cases don't actually occur as immediate operands. */
15029 {
15034 }
15036 else
15037 {
15038 /* We have patterns that allow zero sets of memory, for instance.
15039 In 64-bit mode, we should probably support all 8-byte vectors,
15040 since we can in fact encode that into an immediate. */
15042 {
15045 }
15048 {
15050 {
15053 }
15056 {
15059 else
15061 }
15062 }
15067 else
15069 }
15070 }
15072 static bool
15074 {
15077 }
15078 \f
15079 /* Print a memory operand whose address is ADDR. */
15081 static void
15083 {
15092 {
15099 }
15101 {
15105 }
15107 {
15111 {
15114 }
15117 }
15119 {
15124 }
15125 else
15136 {
15147 }
15149 /* Use one byte shorter RIP relative addressing for 64bit mode. */
15151 {
15163 }
15165 {
15166 /* Displacement only requires special attention. */
15169 {
15173 }
15176 else
15178 }
15179 else
15180 {
15181 /* Print SImode register names to force addr32 prefix. */
15183 {
15184 #ifdef ENABLE_CHECKING
15187 {
15198 }
15199 #endif
15202 }
15204 && TARGET_X32
15205 && disp
15208 {
15209 /* X32 runs in 64-bit mode, where displacement, DISP, in
15210 address DISP(%r64), is encoded as 32-bit immediate sign-
15211 extended from 32-bit to 64-bit. For -0x40000300(%r64),
15212 address is %r64 + 0xffffffffbffffd00. When %r64 <
15213 0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
15214 which is invalid for x32. The correct address is %r64
15215 - 0x40000300 == 0xf7ffdd64. To properly encode
15216 -0x40000300(%r64) for x32, we zero-extend negative
15217 displacement by forcing addr32 prefix which truncates
15218 0xfffffffff7ffdd64 to 0xf7ffdd64. In theory, we should
15219 zero-extend all negative displacements, including -1(%rsp).
15220 However, for small negative displacements, sign-extension
15221 won't cause overflow. We only zero-extend negative
15222 displacements if they < -16*1024*1024, which is also used
15223 to check legitimate address displacements for PIC. */
15225 }
15228 {
15230 {
15235 else
15237 }
15243 {
15248 }
15250 }
15251 else
15252 {
15256 {
15257 /* Pull out the offset of a symbol; print any symbol itself. */
15261 {
15265 }
15273 else
15275 }
15279 {
15282 {
15286 }
15287 }
15290 else
15294 {
15299 }
15301 }
15302 }
15303 }
15305 /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
15307 static bool
15309 {
15310 rtx op;
15317 {
15320 /* FIXME: This might be @TPOFF in Sun ld. */
15331 else
15343 else
15350 #if TARGET_MACHO
15356 #endif
15359 {
15364 #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
15366 #else
15368 #endif
15371 }
15376 }
15379 }
15380 \f
15381 /* Split one or more double-mode RTL references into pairs of half-mode
15382 references. The RTL can be REG, offsettable MEM, integer constant, or
15383 CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
15384 split and "num" is its length. lo_half and hi_half are output arrays
15385 that parallel "operands". */
15387 void
15390 {
15395 {
15404 }
15409 {
15412 /* simplify_subreg refuse to split volatile memory addresses,
15413 but we still have to handle it. */
15415 {
15418 }
15419 else
15420 {
15427 }
15428 }
15429 }
15430 \f
15431 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
15432 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
15433 is the expression of the binary operation. The output may either be
15434 emitted here, or returned to the caller, like all output_* functions.
15436 There is no guarantee that the operands are the same mode, as they
15437 might be within FLOAT or FLOAT_EXTEND expressions. */
15439 #ifndef SYSV386_COMPAT
15440 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
15441 wants to fix the assemblers because that causes incompatibility
15442 with gcc. No-one wants to fix gcc because that causes
15443 incompatibility with assemblers... You can use the option of
15444 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
15445 #define SYSV386_COMPAT 1
15446 #endif
15450 {
15456 #ifdef ENABLE_CHECKING
15457 /* Even if we do not want to check the inputs, this documents input
15458 constraints. Which helps in understanding the following code. */
15468 else
15470 #endif
15473 {
15478 else
15487 else
15496 else
15505 else
15512 }
15515 {
15517 {
15521 else
15523 }
15524 else
15525 {
15529 else
15531 }
15533 }
15537 {
15541 {
15545 }
15547 /* know operands[0] == operands[1]. */
15550 {
15553 }
15556 {
15558 /* How is it that we are storing to a dead operand[2]?
15559 Well, presumably operands[1] is dead too. We can't
15560 store the result to st(0) as st(0) gets popped on this
15561 instruction. Instead store to operands[2] (which I
15562 think has to be st(1)). st(1) will be popped later.
15563 gcc <= 2.8.1 didn't have this check and generated
15564 assembly code that the Unixware assembler rejected. */
15566 else
15569 }
15573 else
15580 {
15583 }
15586 {
15589 }
15592 {
15593 #if SYSV386_COMPAT
15594 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
15595 derived assemblers, confusingly reverse the direction of
15596 the operation for fsub{r} and fdiv{r} when the
15597 destination register is not st(0). The Intel assembler
15598 doesn't have this brain damage. Read !SYSV386_COMPAT to
15599 figure out what the hardware really does. */
15602 else
15604 #else
15606 /* As above for fmul/fadd, we can't store to st(0). */
15608 else
15610 #endif
15612 }
15615 {
15616 #if SYSV386_COMPAT
15619 else
15621 #else
15624 else
15626 #endif
15628 }
15631 {
15634 else
15637 }
15639 {
15640 #if SYSV386_COMPAT
15642 #else
15644 #endif
15645 }
15646 else
15647 {
15648 #if SYSV386_COMPAT
15650 #else
15652 #endif
15653 }
15658 }
15662 }
15664 /* Check if a 256bit AVX register is referenced inside of EXP. */
15666 static int
15668 {
15679 }
15681 /* Return needed mode for entity in optimize_mode_switching pass. */
15683 static int
15685 {
15687 {
15688 rtx link;
15690 /* Needed mode is set to AVX_U128_CLEAN if there are
15691 no 256bit modes used in function arguments. */
15693 link;
15695 {
15697 {
15702 }
15703 }
15706 }
15708 /* Require DIRTY mode if a 256bit AVX register is referenced. Hardware
15709 changes state only when a 256bit register is written to, but we need
15710 to prevent the compiler from moving optimal insertion point above
15711 eventual read from 256bit register. */
15716 }
15718 /* Return mode that i387 must be switched into
15719 prior to the execution of insn. */
15721 static int
15723 {
15726 /* The mode UNINITIALIZED is used to store control word after a
15727 function call or ASM pattern. The mode ANY specify that function
15728 has no requirements on the control word and make no changes in the
15729 bits we are interested in. */
15743 {
15766 }
15769 }
15771 /* Return mode that entity must be switched into
15772 prior to the execution of insn. */
15774 int
15776 {
15778 {
15788 }
15790 }
15792 /* Check if a 256bit AVX register is referenced in stores. */
15794 static void
15796 {
15798 {
15801 }
15802 }
15804 /* Calculate mode of upper 128bit AVX registers after the insn. */
15806 static int
15808 {
15815 /* We know that state is clean after CALL insn if there are no
15816 256bit registers used in the function return register. */
15818 {
15823 }
15825 /* Otherwise, return current mode. Remember that if insn
15826 references AVX 256bit registers, the mode was already changed
15827 to DIRTY from MODE_NEEDED. */
15829 }
15831 /* Return the mode that an insn results in. */
15833 int
15835 {
15837 {
15847 }
15848 }
15850 static int
15852 {
15853 tree arg;
15855 /* Entry mode is set to AVX_U128_DIRTY if there are
15856 256bit modes used in function arguments. */
15859 {
15864 }
15867 }
15869 /* Return a mode that ENTITY is assumed to be
15870 switched to at function entry. */
15872 int
15874 {
15876 {
15886 }
15887 }
15889 static int
15891 {
15894 /* Exit mode is set to AVX_U128_DIRTY if there are
15895 256bit modes used in the function return register. */
15900 }
15902 /* Return a mode that ENTITY is assumed to be
15903 switched to at function exit. */
15905 int
15907 {
15909 {
15919 }
15920 }
15922 /* Output code to initialize control word copies used by trunc?f?i and
15923 rounding patterns. CURRENT_MODE is set to current control word,
15924 while NEW_MODE is set to new control word. */
15926 static void
15928 {
15930 rtx new_mode;
15941 {
15943 {
15945 /* round toward zero (truncate) */
15951 /* round down toward -oo */
15958 /* round up toward +oo */
15965 /* mask precision exception for nearbyint() */
15972 }
15973 }
15974 else
15975 {
15977 {
15979 /* round toward zero (truncate) */
15985 /* round down toward -oo */
15991 /* round up toward +oo */
15997 /* mask precision exception for nearbyint() */
16004 }
16005 }
16011 }
16013 /* Emit vzeroupper. */
16015 void
16017 {
16020 /* Cancel automatic vzeroupper insertion if there are
16021 live call-saved SSE registers at the insertion point. */
16033 }
16035 /* Generate one or more insns to set ENTITY to MODE. */
16037 void
16039 {
16041 {
16056 }
16057 }
16059 /* Output code for INSN to convert a float to a signed int. OPERANDS
16060 are the insn operands. The output may be [HSD]Imode and the input
16061 operand may be [SDX]Fmode. */
16065 {
16070 /* Jump through a hoop or two for DImode, since the hardware has no
16071 non-popping instruction. We used to do this a different way, but
16072 that was somewhat fragile and broke with post-reload splitters. */
16082 else
16083 {
16088 else
16092 }
16095 }
16097 /* Output code for x87 ffreep insn. The OPNO argument, which may only
16098 have the values zero or one, indicates the ffreep insn's operand
16099 from the OPERANDS array. */
16103 {
16105 #ifdef HAVE_AS_IX86_FFREEP
16107 #else
16108 {
16118 }
16119 #endif
16122 }
16125 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
16126 should be used. UNORDERED_P is true when fucom should be used. */
16130 {
16136 {
16139 }
16140 else
16141 {
16144 }
16147 {
16151 else
16153 else
16156 else
16158 }
16165 {
16167 {
16170 }
16171 else
16173 }
16176 && stack_top_dies
16179 {
16180 /* If both the top of the 387 stack dies, and the other operand
16181 is also a stack register that dies, then this must be a
16182 `fcompp' float compare */
16185 {
16186 /* There is no double popping fcomi variant. Fortunately,
16187 eflags is immune from the fstp's cc clobbering. */
16190 else
16193 }
16194 else
16195 {
16198 else
16200 }
16201 }
16202 else
16203 {
16204 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
16207 {
16215 NULL,
16216 NULL,
16223 NULL,
16224 NULL,
16225 NULL,
16226 NULL
16227 };
16242 }
16243 }
16245 void
16247 {
16250 #ifdef ASM_QUAD
16253 #else
16255 #endif
16258 }
16260 void
16262 {
16265 #ifdef ASM_QUAD
16268 #else
16270 #endif
16271 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
16277 #if TARGET_MACHO
16279 {
16283 }
16284 #endif
16285 else
16288 }
16289 \f
16290 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
16291 for the target. */
16293 void
16295 {
16296 rtx tmp;
16298 /* We play register width games, which are only valid after reload. */
16301 /* Avoid HImode and its attendant prefix byte. */
16306 /* This predicate should match that for movsi_xor and movdi_xor_rex64. */
16308 {
16311 }
16314 }
16316 /* X is an unchanging MEM. If it is a constant pool reference, return
16317 the constant pool rtx, else NULL. */
16319 rtx
16321 {
16328 }
16330 void
16332 {
16340 {
16341 rtx tmp;
16345 {
16351 }
16354 }
16358 {
16361 rtx tmp;
16366 else
16370 {
16377 }
16378 }
16382 {
16384 {
16385 #if TARGET_MACHO
16386 /* dynamic-no-pic */
16388 {
16389 rtx temp = ((reload_in_progress
16397 }
16399 {
16403 }
16406 else
16407 {
16415 }
16416 /* dynamic-no-pic */
16417 #endif
16418 }
16419 else
16420 {
16424 {
16430 }
16431 }
16432 }
16433 else
16434 {
16445 /* Force large constants in 64bit compilation into register
16446 to get them CSEed. */
16458 {
16459 /* If we are loading a floating point constant to a register,
16460 force the value to memory now, since we'll get better code
16461 out the back end. */
16465 {
16470 }
16471 }
16472 }
16475 }
16477 void
16479 {
16483 /* Force constants other than zero into memory. We do not know how
16484 the instructions used to build constants modify the upper 64 bits
16485 of the register, once we have that information we may be able
16486 to handle some of them more efficiently. */
16495 /* We need to check memory alignment for SSE mode since attribute
16496 can make operands unaligned. */
16501 {
16504 /* ix86_expand_vector_move_misalign() does not like constants ... */
16510 /* ... nor both arguments in memory. */
16518 }
16520 /* Make operand1 a register if it isn't already. */
16524 {
16527 }
16530 }
16532 /* Split 32-byte AVX unaligned load and store if needed. */
16534 static void
16536 {
16537 rtx m;
16544 {
16565 }
16568 {
16570 {
16577 }
16578 /* Normal *mov<mode>_internal pattern will handle
16579 unaligned loads just fine if misaligned_operand
16580 is true, and without the UNSPEC it can be combined
16581 with arithmetic instructions. */
16584 else
16586 }
16588 {
16590 {
16595 }
16596 else
16598 }
16599 else
16601 }
16603 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
16604 straight to ix86_expand_vector_move. */
16605 /* Code generation for scalar reg-reg moves of single and double precision data:
16606 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
16607 movaps reg, reg
16608 else
16609 movss reg, reg
16610 if (x86_sse_partial_reg_dependency == true)
16611 movapd reg, reg
16612 else
16613 movsd reg, reg
16615 Code generation for scalar loads of double precision data:
16616 if (x86_sse_split_regs == true)
16617 movlpd mem, reg (gas syntax)
16618 else
16619 movsd mem, reg
16621 Code generation for unaligned packed loads of single precision data
16622 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
16623 if (x86_sse_unaligned_move_optimal)
16624 movups mem, reg
16626 if (x86_sse_partial_reg_dependency == true)
16627 {
16628 xorps reg, reg
16629 movlps mem, reg
16630 movhps mem+8, reg
16631 }
16632 else
16633 {
16634 movlps mem, reg
16635 movhps mem+8, reg
16636 }
16638 Code generation for unaligned packed loads of double precision data
16639 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
16640 if (x86_sse_unaligned_move_optimal)
16641 movupd mem, reg
16643 if (x86_sse_split_regs == true)
16644 {
16645 movlpd mem, reg
16646 movhpd mem+8, reg
16647 }
16648 else
16649 {
16650 movsd mem, reg
16651 movhpd mem+8, reg
16652 }
16653 */
16655 void
16657 {
16666 {
16668 {
16672 {
16674 {
16677 }
16678 else
16680 }
16682 /* FALLTHRU */
16686 {
16701 }
16707 else
16715 }
16718 }
16722 {
16724 {
16728 {
16730 {
16733 }
16734 else
16736 }
16738 /* FALLTHRU */
16748 }
16751 }
16754 {
16755 /* Normal *mov<mode>_internal pattern will handle
16756 unaligned loads just fine if misaligned_operand
16757 is true, and without the UNSPEC it can be combined
16758 with arithmetic instructions. */
16764 /* ??? If we have typed data, then it would appear that using
16765 movdqu is the only way to get unaligned data loaded with
16766 integer type. */
16768 {
16770 {
16773 }
16775 /* We will eventually emit movups based on insn attributes. */
16779 }
16781 {
16782 rtx zero;
16785 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16786 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16788 {
16789 /* We will eventually emit movups based on insn attributes. */
16792 }
16794 /* When SSE registers are split into halves, we can avoid
16795 writing to the top half twice. */
16797 {
16800 }
16801 else
16802 {
16803 /* ??? Not sure about the best option for the Intel chips.
16804 The following would seem to satisfy; the register is
16805 entirely cleared, breaking the dependency chain. We
16806 then store to the upper half, with a dependency depth
16807 of one. A rumor has it that Intel recommends two movsd
16808 followed by an unpacklpd, but this is unconfirmed. And
16809 given that the dependency depth of the unpacklpd would
16810 still be one, I'm not sure why this would be better. */
16812 }
16818 }
16819 else
16820 {
16821 rtx t;
16824 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16825 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16827 {
16829 {
16832 }
16839 }
16843 else
16848 else
16857 }
16858 }
16860 {
16862 {
16865 /* We will eventually emit movups based on insn attributes. */
16867 }
16869 {
16871 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
16872 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16874 /* We will eventually emit movups based on insn attributes. */
16876 else
16877 {
16882 }
16883 }
16884 else
16885 {
16890 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
16891 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16893 {
16896 }
16897 else
16898 {
16903 }
16904 }
16905 }
16906 else
16908 }
16910 /* Expand a push in MODE. This is some mode for which we do not support
16911 proper push instructions, at least from the registers that we expect
16912 the value to live in. */
16914 void
16916 {
16917 rtx tmp;
16927 /* When we push an operand onto stack, it has to be aligned at least
16928 at the function argument boundary. However since we don't have
16929 the argument type, we can't determine the actual argument
16930 boundary. */
16932 }
16934 /* Helper function of ix86_fixup_binary_operands to canonicalize
16935 operand order. Returns true if the operands should be swapped. */
16937 static bool
16939 rtx operands[])
16940 {
16945 /* If the operation is not commutative, we can't do anything. */
16949 /* Highest priority is that src1 should match dst. */
16955 /* Next highest priority is that immediate constants come second. */
16961 /* Lowest priority is that memory references should come second. */
16968 }
16971 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
16972 destination to use for the operation. If different from the true
16973 destination in operands[0], a copy operation will be required. */
16975 rtx
16977 rtx operands[])
16978 {
16983 /* Canonicalize operand order. */
16985 {
16986 rtx temp;
16988 /* It is invalid to swap operands of different modes. */
16994 }
16996 /* Both source operands cannot be in memory. */
16998 {
16999 /* Optimization: Only read from memory once. */
17001 {
17004 }
17007 else
17009 }
17011 /* If the destination is memory, and we do not have matching source
17012 operands, do things in registers. */
17016 /* Source 1 cannot be a constant. */
17020 /* Source 1 cannot be a non-matching memory. */
17024 /* Improve address combine. */
17033 }
17035 /* Similarly, but assume that the destination has already been
17036 set up properly. */
17038 void
17041 {
17044 }
17046 /* Attempt to expand a binary operator. Make the expansion closer to the
17047 actual machine, then just general_operand, which will allow 3 separate
17048 memory references (one output, two input) in a single insn. */
17050 void
17052 rtx operands[])
17053 {
17060 /* Emit the instruction. */
17064 {
17065 /* Reload doesn't know about the flags register, and doesn't know that
17066 it doesn't want to clobber it. We can only do this with PLUS. */
17069 }
17073 {
17074 /* This is going to be an LEA; avoid splitting it later. */
17076 }
17077 else
17078 {
17081 }
17083 /* Fix up the destination if needed. */
17086 }
17088 /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
17089 the given OPERANDS. */
17091 void
17093 rtx operands[])
17094 {
17097 {
17100 }
17102 {
17105 }
17106 /* Optimize (__m128i) d | (__m128i) e and similar code
17107 when d and e are float vectors into float vector logical
17108 insn. In C/C++ without using intrinsics there is no other way
17109 to express vector logical operation on float vectors than
17110 to cast them temporarily to integer vectors. */
17112 && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17121 {
17122 rtx dst;
17124 {
17131 {
17134 }
17135 else
17136 {
17141 }
17152 }
17153 }
17162 }
17164 /* Return TRUE or FALSE depending on whether the binary operator meets the
17165 appropriate constraints. */
17167 bool
17170 {
17175 /* Both source operands cannot be in memory. */
17179 /* Canonicalize operand order for commutative operators. */
17181 {
17185 }
17187 /* If the destination is memory, we must have a matching source operand. */
17191 /* Source 1 cannot be a constant. */
17195 /* Source 1 cannot be a non-matching memory. */
17197 /* Support "andhi/andsi/anddi" as a zero-extending move. */
17205 }
17207 /* Attempt to expand a unary operator. Make the expansion closer to the
17208 actual machine, then just general_operand, which will allow 2 separate
17209 memory references (one output, one input) in a single insn. */
17211 void
17213 rtx operands[])
17214 {
17221 /* If the destination is memory, and we do not have matching source
17222 operands, do things in registers. */
17225 {
17228 else
17230 }
17232 /* When source operand is memory, destination must match. */
17236 /* Emit the instruction. */
17240 {
17241 /* Reload doesn't know about the flags register, and doesn't know that
17242 it doesn't want to clobber it. */
17245 }
17246 else
17247 {
17250 }
17252 /* Fix up the destination if needed. */
17255 }
17257 /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
17258 divisor are within the range [0-255]. */
17260 void
17263 {
17272 {
17285 }
17292 /* Use 8bit unsigned divimod if dividend and divisor are within
17293 the range [0-255]. */
17302 pc_rtx);
17307 /* Generate original signed/unsigned divimod. */
17312 /* Branch to the end. */
17316 /* Generate 8bit unsigned divide. */
17318 /* Don't use operands[0] for result of 8bit divide since not all
17319 registers support QImode ZERO_EXTRACT. */
17326 {
17329 }
17330 else
17331 {
17334 }
17336 /* Extract remainder from AH. */
17340 else
17341 {
17342 /* Need a new scratch register since the old one has result
17343 of 8bit divide. */
17347 }
17350 /* Zero extend quotient from AL. */
17356 }
17358 #define LEA_MAX_STALL (3)
17359 #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
17361 /* Increase given DISTANCE in half-cycles according to
17362 dependencies between PREV and NEXT instructions.
17363 Add 1 half-cycle if there is no dependency and
17364 go to next cycle if there is some dependecy. */
17366 static unsigned int
17368 {
17385 }
17387 /* Function checks if instruction INSN defines register number
17388 REGNO1 or REGNO2. */
17390 static bool
17392 rtx insn)
17393 {
17401 {
17403 }
17406 }
17408 /* Function checks if instruction INSN uses register number
17409 REGNO as a part of address expression. */
17411 static bool
17413 {
17421 }
17423 /* Search backward for non-agu definition of register number REGNO1
17424 or register number REGNO2 in basic block starting from instruction
17425 START up to head of basic block or instruction INSN.
17427 Function puts true value into *FOUND var if definition was found
17428 and false otherwise.
17430 Distance in half-cycles between START and found instruction or head
17431 of BB is added to DISTANCE and returned. */
17433 static int
17437 {
17447 {
17449 {
17452 {
17455 {
17458 }
17459 }
17462 }
17467 }
17470 }
17472 /* Search backward for non-agu definition of register number REGNO1
17473 or register number REGNO2 in INSN's basic block until
17474 1. Pass LEA_SEARCH_THRESHOLD instructions, or
17475 2. Reach neighbour BBs boundary, or
17476 3. Reach agu definition.
17477 Returns the distance between the non-agu definition point and INSN.
17478 If no definition point, returns -1. */
17480 static int
17482 rtx insn)
17483 {
17494 {
17495 edge e;
17496 edge_iterator ei;
17501 {
17504 }
17510 else
17511 {
17516 {
17517 int bb_dist
17523 {
17530 }
17531 }
17534 }
17535 }
17537 /* get_attr_type may modify recog data. We want to make sure
17538 that recog data is valid for instruction INSN, on which
17539 distance_non_agu_define is called. INSN is unchanged here. */
17546 }
17548 /* Return the distance in half-cycles between INSN and the next
17549 insn that uses register number REGNO in memory address added
17550 to DISTANCE. Return -1 if REGNO0 is set.
17552 Put true value into *FOUND if register usage was found and
17553 false otherwise.
17554 Put true value into *REDEFINED if register redefinition was
17555 found and false otherwise. */
17557 static int
17561 {
17570 {
17573 /* If insn and start belong to the same bb, set prev to insn,
17574 so the call to increase_distance will increase the distance
17575 between insns by 1. */
17577 }
17582 {
17584 {
17587 {
17588 /* Return DISTANCE if OP0 is used in memory
17589 address in NEXT. */
17592 }
17595 {
17596 /* Return -1 if OP0 is set in NEXT. */
17599 }
17602 }
17608 }
17611 }
17613 /* Return the distance between INSN and the next insn that uses
17614 register number REGNO0 in memory address. Return -1 if no such
17615 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
17617 static int
17619 {
17631 {
17632 edge e;
17633 edge_iterator ei;
17638 {
17641 }
17647 else
17648 {
17654 {
17655 int bb_dist
17660 {
17667 }
17668 }
17671 }
17672 }
17678 }
17680 /* Define this macro to tune LEA priority vs ADD, it take effect when
17681 there is a dilemma of choicing LEA or ADD
17682 Negative value: ADD is more preferred than LEA
17683 Zero: Netrual
17684 Positive value: LEA is more preferred than ADD*/
17685 #define IX86_LEA_PRIORITY 0
17687 /* Return true if usage of lea INSN has performance advantage
17688 over a sequence of instructions. Instructions sequence has
17689 SPLIT_COST cycles higher latency than lea latency. */
17691 static bool
17694 {
17697 /* For Silvermont if using a 2-source or 3-source LEA for
17698 non-destructive destination purposes, or due to wanting
17699 ability to use SCALE, the use of LEA is justified. */
17701 {
17709 }
17715 {
17716 /* If there is no non AGU operand definition, no AGU
17717 operand usage and split cost is 0 then both lea
17718 and non lea variants have same priority. Currently
17719 we prefer lea for 64 bit code and non lea on 32 bit
17720 code. */
17723 else
17725 }
17727 /* With longer definitions distance lea is more preferable.
17728 Here we change it to take into account splitting cost and
17729 lea priority. */
17732 /* If there is no use in memory addess then we just check
17733 that split cost exceeds AGU stall. */
17737 /* If this insn has both backward non-agu dependence and forward
17738 agu dependence, the one with short distance takes effect. */
17740 }
17742 /* Return true if it is legal to clobber flags by INSN and
17743 false otherwise. */
17745 static bool
17747 {
17750 bitmap live;
17753 {
17755 {
17762 }
17768 }
17772 }
17774 /* Return true if we need to split op0 = op1 + op2 into a sequence of
17775 move and add to avoid AGU stalls. */
17777 bool
17779 {
17782 /* Check if we need to optimize. */
17786 /* Check it is correct to split here. */
17794 /* We need to split only adds with non destructive
17795 destination operand. */
17798 else
17800 }
17802 /* Return true if we should emit lea instruction instead of mov
17803 instruction. */
17805 bool
17807 {
17810 /* Check if we need to optimize. */
17814 /* Use lea for reg to reg moves only. */
17822 }
17824 /* Return true if we need to split lea into a sequence of
17825 instructions to avoid AGU stalls. */
17827 bool
17829 {
17835 /* Check we need to optimize. */
17839 /* Check it is correct to split here. */
17846 /* There should be at least two components in the address. */
17851 /* We should not split into add if non legitimate pic
17852 operand is used as displacement. */
17867 /* Compute how many cycles we will add to execution time
17868 if split lea into a sequence of instructions. */
17870 {
17871 /* Have to use mov instruction if non desctructive
17872 destination form is used. */
17876 /* Have to add index to base if both exist. */
17880 /* Have to use shift and adds if scale is 2 or greater. */
17882 {
17887 else
17889 }
17891 /* Have to use add instruction with immediate if
17892 disp is non zero. */
17896 /* Subtract the price of lea. */
17898 }
17902 }
17904 /* Emit x86 binary operand CODE in mode MODE, where the first operand
17905 matches destination. RTX includes clobber of FLAGS_REG. */
17907 static void
17910 {
17917 }
17919 /* Return true if regno1 def is nearest to the insn. */
17921 static bool
17923 {
17930 {
17932 {
17935 }
17941 }
17943 /* None of the regs is defined in the bb. */
17945 }
17947 /* Split lea instructions into a sequence of instructions
17948 which are executed on ALU to avoid AGU stalls.
17949 It is assumed that it is allowed to clobber flags register
17950 at lea position. */
17952 void
17954 {
17970 {
17973 }
17976 {
17979 }
17985 {
17986 /* Case r1 = r1 + ... */
17988 {
17989 /* If we have a case r1 = r1 + C * r1 then we
17990 should use multiplication which is very
17991 expensive. Assume cost model is wrong if we
17992 have such case here. */
17997 }
17998 else
17999 {
18000 /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
18004 /* Use shift for scaling. */
18013 }
18014 }
18016 {
18019 }
18020 else
18021 {
18023 {
18026 }
18028 {
18031 }
18032 else
18033 {
18038 else
18039 {
18040 rtx tmp1;
18042 /* Find better operand for SET instruction, depending
18043 on which definition is farther from the insn. */
18046 else
18056 }
18059 }
18063 }
18064 }
18066 /* Return true if it is ok to optimize an ADD operation to LEA
18067 operation to avoid flag register consumation. For most processors,
18068 ADD is faster than LEA. For the processors like ATOM, if the
18069 destination register of LEA holds an actual address which will be
18070 used soon, LEA is better and otherwise ADD is better. */
18072 bool
18074 {
18079 /* If a = b + c, (a!=b && a!=c), must use lea form. */
18087 }
18089 /* Return true if destination reg of SET_BODY is shift count of
18090 USE_BODY. */
18092 static bool
18094 {
18095 rtx set_dest;
18096 rtx shift_rtx;
18099 /* Retrieve destination of SET_BODY. */
18101 {
18110 use_body))
18115 }
18117 /* Retrieve shift count of USE_BODY. */
18119 {
18131 }
18139 {
18142 /* Return true if shift count is dest of SET_BODY. */
18144 {
18145 /* Add check since it can be invoked before register
18146 allocation in pre-reload schedule. */
18152 }
18153 }
18156 }
18158 /* Return true if destination reg of SET_INSN is shift count of
18159 USE_INSN. */
18161 bool
18163 {
18166 }
18168 /* Return TRUE or FALSE depending on whether the unary operator meets the
18169 appropriate constraints. */
18171 bool
18175 {
18176 /* If one of operands is memory, source and destination must match. */
18182 }
18184 /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
18185 are ok, keeping in mind the possible movddup alternative. */
18187 bool
18189 {
18195 }
18197 /* Post-reload splitter for converting an SF or DFmode value in an
18198 SSE register into an unsigned SImode. */
18200 void
18202 {
18213 /* Load up the value into the low element. We must ensure that the other
18214 elements are valid floats -- zero is the easiest such value. */
18216 {
18219 else
18221 }
18222 else
18223 {
18228 else
18230 }
18250 else
18255 }
18257 /* Convert an unsigned DImode value into a DFmode, using only SSE.
18258 Expects the 64-bit DImode to be supplied in a pair of integral
18259 registers. Requires SSE2; will use SSE3 if available. For x86_32,
18260 -mfpmath=sse, !optimize_size only. */
18262 void
18264 {
18268 rtx x;
18274 {
18277 }
18278 else
18279 {
18283 }
18291 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
18294 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
18295 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
18296 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
18297 (0x1.0p84 + double(fp_value_hi_xmm)).
18298 Note these exponents differ by 32. */
18302 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
18303 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
18312 /* Add the upper and lower DFmode values together. */
18315 else
18316 {
18320 }
18323 }
18325 /* Not used, but eases macroization of patterns. */
18326 void
18328 rtx input ATTRIBUTE_UNUSED)
18329 {
18331 }
18333 /* Convert an unsigned SImode value into a DFmode. Only currently used
18334 for SSE, but applicable anywhere. */
18336 void
18338 {
18339 REAL_VALUE_TYPE TWO31r;
18354 }
18356 /* Convert a signed DImode value into a DFmode. Only used for SSE in
18357 32-bit mode; otherwise we have a direct convert instruction. */
18359 void
18361 {
18362 REAL_VALUE_TYPE TWO32r;
18380 }
18382 /* Convert an unsigned SImode value into a SFmode, using only SSE.
18383 For x86_32, -mfpmath=sse, !optimize_size only. */
18384 void
18386 {
18387 REAL_VALUE_TYPE ONE16r;
18406 }
18408 /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
18409 a vector of unsigned ints VAL to vector of floats TARGET. */
18411 void
18413 {
18415 REAL_VALUE_TYPE TWO16r;
18422 else
18427 OPTAB_DIRECT);
18438 OPTAB_DIRECT);
18440 OPTAB_DIRECT);
18443 }
18445 /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
18446 pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
18447 This is done by doing just signed conversion if < 0x1p31, and otherwise by
18448 subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
18450 rtx
18452 {
18453 REAL_VALUE_TYPE TWO31r;
18468 {
18474 }
18483 OPTAB_DIRECT);
18484 else
18485 {
18491 OPTAB_DIRECT);
18492 }
18495 }
18497 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
18498 then replicate the value for all elements of the vector
18499 register. */
18501 rtx
18503 {
18505 rtvec v;
18509 {
18536 }
18537 }
18539 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
18540 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
18541 for an SSE register. If VECT is true, then replicate the mask for
18542 all elements of the vector register. If INVERT is true, then create
18543 a mask excluding the sign bit. */
18545 rtx
18547 {
18551 rtx v;
18552 rtx mask;
18554 /* Find the sign bit, sign extended to 2*HWI. */
18556 {
18576 else
18584 {
18587 }
18588 else
18589 {
18590 rtvec vec;
18596 {
18599 }
18600 else
18610 }
18615 }
18620 /* Force this value into the low part of a fp vector constant. */
18629 }
18631 /* Generate code for floating point ABS or NEG. */
18633 void
18635 rtx operands[])
18636 {
18647 {
18653 }
18655 /* NEG and ABS performed with SSE use bitwise mask operations.
18656 Create the appropriate mask now. */
18659 else
18669 {
18671 rtvec par;
18676 else
18677 {
18680 }
18682 }
18683 else
18685 }
18687 /* Expand a copysign operation. Special case operand 0 being a constant. */
18689 void
18691 {
18705 else
18709 {
18716 {
18719 else
18720 {
18724 }
18725 }
18735 else
18739 }
18740 else
18741 {
18751 else
18755 }
18756 }
18758 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
18759 be a constant, and so has already been expanded into a vector constant. */
18761 void
18763 {
18779 {
18782 }
18783 }
18785 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
18786 so we have to do two masks. */
18788 void
18790 {
18805 {
18806 /* Shouldn't happen often (it's useless, obviously), but when it does
18807 we'd generate incorrect code if we continue below. */
18810 }
18813 {
18824 }
18825 else
18826 {
18828 {
18830 }
18832 {
18836 }
18840 {
18843 }
18845 {
18850 }
18852 }
18856 }
18858 /* Return TRUE or FALSE depending on whether the first SET in INSN
18859 has source and destination with matching CC modes, and that the
18860 CC mode is at least as constrained as REQ_MODE. */
18862 bool
18864 {
18865 rtx set;
18876 {
18886 /* FALLTHRU */
18890 /* FALLTHRU */
18894 /* FALLTHRU */
18908 }
18911 }
18913 /* Generate insn patterns to do an integer compare of OPERANDS. */
18915 static rtx
18917 {
18924 /* This is very simple, but making the interface the same as in the
18925 FP case makes the rest of the code easier. */
18929 /* Return the test that should be put into the flags user, i.e.
18930 the bcc, scc, or cmov instruction. */
18932 }
18934 /* Figure out whether to use ordered or unordered fp comparisons.
18935 Return the appropriate mode to use. */
18937 enum machine_mode
18939 {
18940 /* ??? In order to make all comparisons reversible, we do all comparisons
18941 non-trapping when compiling for IEEE. Once gcc is able to distinguish
18942 all forms trapping and nontrapping comparisons, we can make inequality
18943 comparisons trapping again, since it results in better code when using
18944 FCOM based compares. */
18946 }
18948 enum machine_mode
18950 {
18954 {
18957 }
18960 {
18961 /* Only zero flag is needed. */
18965 /* Codes needing carry flag. */
18968 /* Detect overflow checks. They need just the carry flag. */
18972 else
18977 /* Codes possibly doable only with sign flag when
18978 comparing against zero. */
18983 else
18984 /* For other cases Carry flag is not required. */
18986 /* Codes doable only with sign flag when comparing
18987 against zero, but we miss jump instruction for it
18988 so we need to use relational tests against overflow
18989 that thus needs to be zero. */
18994 else
18996 /* strcmp pattern do (use flags) and combine may ask us for proper
18997 mode. */
19002 }
19003 }
19005 /* Return the fixed registers used for condition codes. */
19007 static bool
19009 {
19013 }
19015 /* If two condition code modes are compatible, return a condition code
19016 mode which is compatible with both. Otherwise, return
19017 VOIDmode. */
19019 static enum machine_mode
19021 {
19038 {
19052 {
19066 }
19070 /* These are only compatible with themselves, which we already
19071 checked above. */
19073 }
19074 }
19077 /* Return a comparison we can do and that it is equivalent to
19078 swap_condition (code) apart possibly from orderedness.
19079 But, never change orderedness if TARGET_IEEE_FP, returning
19080 UNKNOWN in that case if necessary. */
19082 static enum rtx_code
19084 {
19086 {
19097 }
19098 }
19100 /* Return cost of comparison CODE using the best strategy for performance.
19101 All following functions do use number of instructions as a cost metrics.
19102 In future this should be tweaked to compute bytes for optimize_size and
19103 take into account performance of various instructions on various CPUs. */
19105 static int
19107 {
19110 /* The cost of code using bit-twiddling on %ah. */
19112 {
19135 }
19138 {
19145 }
19146 }
19148 /* Return strategy to use for floating-point. We assume that fcomi is always
19149 preferrable where available, since that is also true when looking at size
19150 (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
19152 enum ix86_fpcmp_strategy
19154 {
19155 /* Do fcomi/sahf based test when profitable. */
19164 }
19166 /* Swap, force into registers, or otherwise massage the two operands
19167 to a fp comparison. The operands are updated in place; the new
19168 comparison code is returned. */
19170 static enum rtx_code
19172 {
19178 /* All of the unordered compare instructions only work on registers.
19179 The same is true of the fcomi compare instructions. The XFmode
19180 compare instructions require registers except when comparing
19181 against zero or when converting operand 1 from fixed point to
19182 floating point. */
19191 {
19194 }
19195 else
19196 {
19197 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
19198 things around if they appear profitable, otherwise force op0
19199 into a register. */
19205 {
19208 {
19209 rtx tmp;
19212 }
19213 }
19219 {
19224 {
19227 }
19228 else
19230 }
19231 }
19233 /* Try to rearrange the comparison to make it cheaper. */
19237 {
19238 rtx tmp;
19243 }
19248 }
19250 /* Convert comparison codes we use to represent FP comparison to integer
19251 code that will result in proper branch. Return UNKNOWN if no such code
19252 is available. */
19254 enum rtx_code
19256 {
19258 {
19281 }
19282 }
19284 /* Generate insn patterns to do a floating point compare of OPERANDS. */
19286 static rtx
19288 {
19295 /* Do fcomi/sahf based test when profitable. */
19297 {
19302 tmp);
19310 tmp);
19319 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
19326 /* In the unordered case, we have to check C2 for NaN's, which
19327 doesn't happen to work out to anything nice combination-wise.
19328 So do some bit twiddling on the value we've got in AH to come
19329 up with an appropriate set of condition codes. */
19333 {
19337 {
19340 }
19341 else
19342 {
19348 }
19353 {
19358 }
19359 else
19360 {
19363 }
19368 {
19371 }
19372 else
19373 {
19377 }
19382 {
19388 }
19389 else
19390 {
19393 }
19398 {
19403 }
19404 else
19405 {
19408 }
19413 {
19418 }
19419 else
19420 {
19423 }
19437 }
19442 }
19444 /* Return the test that should be put into the flags user, i.e.
19445 the bcc, scc, or cmov instruction. */
19448 const0_rtx);
19449 }
19451 static rtx
19453 {
19454 rtx ret;
19460 {
19463 }
19464 else
19468 }
19470 void
19472 {
19474 rtx tmp;
19477 {
19484 simple:
19488 pc_rtx);
19496 /* Expand DImode branch into multiple compare+branch. */
19497 {
19503 {
19506 }
19513 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
19514 avoid two branches. This costs one extra insn, so disable when
19515 optimizing for size. */
19520 {
19538 }
19540 /* Otherwise, if we are doing less-than or greater-or-equal-than,
19541 op1 is a constant and the low word is zero, then we can just
19542 examine the high word. Similarly for low word -1 and
19543 less-or-equal-than or greater-than. */
19547 {
19550 {
19553 }
19557 {
19560 }
19564 }
19566 /* Otherwise, we need two or three jumps. */
19575 {
19589 }
19591 /*
19592 * a < b =>
19593 * if (hi(a) < hi(b)) goto true;
19594 * if (hi(a) > hi(b)) goto false;
19595 * if (lo(a) < lo(b)) goto true;
19596 * false:
19597 */
19609 }
19614 }
19615 }
19617 /* Split branch based on floating point condition. */
19618 void
19621 {
19622 rtx condition;
19623 rtx i;
19626 {
19631 }
19634 tmp);
19636 /* Remove pushed operand from stack. */
19646 }
19648 void
19650 {
19651 rtx ret;
19658 }
19660 /* Expand comparison setting or clearing carry flag. Return true when
19661 successful and set pop for the operation. */
19662 static bool
19664 {
19668 /* Do not handle double-mode compares that go through special path. */
19673 {
19678 /* Shortcut: following common codes never translate
19679 into carry flag compares. */
19684 /* These comparisons require zero flag; swap operands so they won't. */
19687 {
19692 }
19694 /* Try to expand the comparison and verify that we end up with
19695 carry flag based comparison. This fails to be true only when
19696 we decide to expand comparison using arithmetic that is not
19697 too common scenario. */
19706 else
19715 }
19721 {
19726 /* Convert a==0 into (unsigned)a<1. */
19735 /* Convert a>b into b<a or a>=b-1. */
19739 {
19741 /* Bail out on overflow. We still can swap operands but that
19742 would force loading of the constant into register. */
19747 }
19748 else
19749 {
19754 }
19757 /* Convert a>=0 into (unsigned)a<0x80000000. */
19775 }
19776 /* Swapping operands may cause constant to appear as first operand. */
19778 {
19782 }
19786 }
19788 bool
19790 {
19814 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
19815 HImode insns, we'd be swallowed in word prefix ops. */
19821 {
19825 HOST_WIDE_INT diff;
19828 /* Sign bit compares are better done using shifts than we do by using
19829 sbb. */
19832 {
19833 /* Detect overlap between destination and compare sources. */
19837 {
19838 rtx flags;
19847 {
19849 compare_code
19851 }
19853 /* To simplify rest of code, restrict to the GEU case. */
19855 {
19861 }
19862 else
19863 {
19866 reverse_condition_maybe_unordered
19868 else
19871 }
19880 else
19883 }
19884 else
19885 {
19888 else
19889 {
19894 }
19896 }
19899 {
19900 /*
19901 * cmpl op0,op1
19902 * sbbl dest,dest
19903 * [addl dest, ct]
19904 *
19905 * Size 5 - 8.
19906 */
19911 }
19913 {
19914 /*
19915 * cmpl op0,op1
19916 * sbbl dest,dest
19917 * orl $ct, dest
19918 *
19919 * Size 8.
19920 */
19924 }
19926 {
19927 /*
19928 * cmpl op0,op1
19929 * sbbl dest,dest
19930 * notl dest
19931 * [addl dest, cf]
19932 *
19933 * Size 8 - 11.
19934 */
19940 }
19941 else
19942 {
19943 /*
19944 * cmpl op0,op1
19945 * sbbl dest,dest
19946 * [notl dest]
19947 * andl cf - ct, dest
19948 * [addl dest, ct]
19949 *
19950 * Size 8 - 11.
19951 */
19954 {
19958 }
19968 }
19974 }
19977 {
19980 HOST_WIDE_INT tmp;
19985 {
19988 /* We may be reversing unordered compare to normal compare, that
19989 is not valid in general (we may convert non-trapping condition
19990 to trapping one), however on i386 we currently emit all
19991 comparisons unordered. */
19994 }
19995 else
19996 {
19999 }
20000 }
20005 {
20010 {
20015 }
20016 }
20018 /* Optimize dest = (op0 < 0) ? -1 : cf. */
20022 {
20023 /* If lea code below could be used, only optimize
20024 if it results in a 2 insn sequence. */
20030 {
20031 /*
20032 * notl op1 (if necessary)
20033 * sarl $31, op1
20034 * orl cf, op1
20035 */
20037 {
20041 }
20052 }
20053 }
20061 {
20062 /*
20063 * xorl dest,dest
20064 * cmpl op1,op2
20065 * setcc dest
20066 * lea cf(dest*(ct-cf)),dest
20067 *
20068 * Size 14.
20069 *
20070 * This also catches the degenerate setcc-only case.
20071 */
20073 rtx tmp;
20079 /* On x86_64 the lea instruction operates on Pmode, so we need
20080 to get arithmetics done in proper mode to match. */
20083 else
20084 {
20085 rtx out1;
20088 nops++;
20090 {
20092 nops++;
20093 }
20094 }
20096 {
20098 nops++;
20099 }
20101 {
20104 else
20106 }
20111 }
20113 /*
20114 * General case: Jumpful:
20115 * xorl dest,dest cmpl op1, op2
20116 * cmpl op1, op2 movl ct, dest
20117 * setcc dest jcc 1f
20118 * decl dest movl cf, dest
20119 * andl (cf-ct),dest 1:
20120 * addl ct,dest
20121 *
20122 * Size 20. Size 14.
20123 *
20124 * This is reasonably steep, but branch mispredict costs are
20125 * high on modern cpus, so consider failing only if optimizing
20126 * for space.
20127 */
20132 {
20134 {
20141 {
20144 /* We may be reversing unordered compare to normal compare,
20145 that is not valid in general (we may convert non-trapping
20146 condition to trapping one), however on i386 we currently
20147 emit all comparisons unordered. */
20149 }
20150 else
20151 {
20155 }
20156 }
20159 {
20160 /* notl op1 (if needed)
20161 sarl $31, op1
20162 andl (cf-ct), op1
20163 addl ct, op1
20165 For x < 0 (resp. x <= -1) there will be no notl,
20166 so if possible swap the constants to get rid of the
20167 complement.
20168 True/false will be -1/0 while code below (store flag
20169 followed by decrement) is 0/-1, so the constants need
20170 to be exchanged once more. */
20173 {
20176 }
20177 else
20178 {
20182 }
20185 }
20186 else
20187 {
20191 constm1_rtx,
20193 }
20205 }
20206 }
20209 {
20210 /* Try a few things more with specific constants and a variable. */
20212 optab op;
20218 /* If one of the two operands is an interesting constant, load a
20219 constant with the above and mask it in with a logical operation. */
20222 {
20228 else
20230 }
20232 {
20238 else
20240 }
20241 else
20248 /* Recurse to get the constant loaded. */
20252 /* Mask in the interesting variable. */
20254 OPTAB_WIDEN);
20259 }
20261 /*
20262 * For comparison with above,
20263 *
20264 * movl cf,dest
20265 * movl ct,tmp
20266 * cmpl op1,op2
20267 * cmovcc tmp,dest
20268 *
20269 * Size 15.
20270 */
20292 }
20294 /* Swap, force into registers, or otherwise massage the two operands
20295 to an sse comparison with a mask result. Thus we differ a bit from
20296 ix86_prepare_fp_compare_args which expects to produce a flags result.
20298 The DEST operand exists to help determine whether to commute commutative
20299 operators. The POP0/POP1 operands are updated in place. The new
20300 comparison code is returned, or UNKNOWN if not implementable. */
20302 static enum rtx_code
20305 {
20306 rtx tmp;
20309 {
20312 /* AVX supports all the needed comparisons. */
20315 /* We have no LTGT as an operator. We could implement it with
20316 NE & ORDERED, but this requires an extra temporary. It's
20317 not clear that it's worth it. */
20324 /* These are supported directly. */
20331 /* AVX has 3 operand comparisons, no need to swap anything. */
20334 /* For commutative operators, try to canonicalize the destination
20335 operand to be first in the comparison - this helps reload to
20336 avoid extra moves. */
20339 /* FALLTHRU */
20345 /* These are not supported directly before AVX, and furthermore
20346 ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
20347 comparison operands to transform into something that is
20348 supported. */
20357 }
20360 }
20362 /* Detect conditional moves that exactly match min/max operational
20363 semantics. Note that this is IEEE safe, as long as we don't
20364 interchange the operands.
20366 Returns FALSE if this conditional move doesn't match a MIN/MAX,
20367 and TRUE if the operation is successful and instructions are emitted. */
20369 static bool
20372 {
20375 rtx tmp;
20378 ;
20380 {
20384 }
20385 else
20392 else
20397 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
20398 but MODE may be a vector mode and thus not appropriate. */
20400 {
20402 rtvec v;
20407 }
20408 else
20409 {
20412 }
20416 }
20418 /* Expand an sse vector comparison. Return the register with the result. */
20420 static rtx
20423 {
20426 rtx x;
20439 {
20442 }
20443 else
20447 }
20449 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
20450 operations. This is used for both scalar and vector conditional moves. */
20452 static void
20454 {
20460 {
20462 }
20464 {
20468 }
20470 {
20475 }
20477 {
20481 }
20483 {
20491 op_true,
20492 op_false)));
20493 }
20494 else
20495 {
20505 {
20519 {
20526 }
20541 {
20548 }
20552 }
20555 {
20559 }
20560 else
20561 {
20567 else
20579 }
20580 }
20581 }
20583 /* Expand a floating-point conditional move. Return true if successful. */
20585 bool
20587 {
20595 {
20598 /* Since we've no cmove for sse registers, don't force bad register
20599 allocation just to gain access to it. Deny movcc when the
20600 comparison mode doesn't match the move mode. */
20619 }
20626 /* The floating point conditional move instructions don't directly
20627 support conditions resulting from a signed integer comparison. */
20631 {
20636 }
20643 }
20645 /* Expand a floating-point vector conditional move; a vcond operation
20646 rather than a movcc operation. */
20648 bool
20650 {
20652 rtx cmp;
20657 {
20658 rtx temp;
20660 {
20677 }
20679 OPTAB_DIRECT);
20682 }
20692 }
20694 /* Expand a signed/unsigned integral vector conditional move. */
20696 bool
20698 {
20708 /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
20709 and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
20718 {
20722 {
20728 }
20731 {
20737 }
20738 }
20747 /* XOP supports all of the comparisons on all 128-bit vector int types. */
20751 ;
20752 else
20753 {
20754 /* Canonicalize the comparison to EQ, GT, GTU. */
20756 {
20773 /* FALLTHRU */
20783 }
20785 /* Only SSE4.1/SSE4.2 supports V2DImode. */
20787 {
20789 {
20791 /* SSE4.1 supports EQ. */
20798 /* SSE4.2 supports GT/GTU. */
20805 }
20806 }
20808 /* Unsigned parallel compare is not supported by the hardware.
20809 Play some tricks to turn this into a signed comparison
20810 against 0. */
20812 {
20816 {
20821 {
20826 {
20833 }
20834 /* Subtract (-(INT MAX) - 1) from both operands to make
20835 them signed. */
20846 }
20853 /* Perform a parallel unsigned saturating subtraction. */
20866 }
20867 }
20868 }
20870 /* Allow the comparison to be done in one mode, but the movcc to
20871 happen in another mode. */
20873 {
20876 }
20877 else
20878 {
20883 }
20888 }
20890 /* Expand a variable vector permutation. */
20892 void
20894 {
20905 /* Number of elements in the vector. */
20911 {
20913 {
20914 /* Unfortunately, the VPERMQ and VPERMPD instructions only support
20915 an constant shuffle operand. With a tiny bit of effort we can
20916 use VPERMD instead. A re-interpretation stall for V4DFmode is
20917 unfortunate but there's no avoiding it.
20918 Similarly for V16HImode we don't have instructions for variable
20919 shuffling, while for V32QImode we can use after preparing suitable
20920 masks vpshufb; vpshufb; vpermq; vpor. */
20923 {
20927 }
20928 else
20929 {
20933 }
20936 /* Replicate the low bits of the V4DImode mask into V8SImode:
20937 mask = { A B C D }
20938 t1 = { A A B B C C D D }. */
20946 else
20949 /* Multiply the shuffle indicies by two. */
20951 OPTAB_DIRECT);
20953 /* Add one to the odd shuffle indicies:
20954 t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
20956 {
20959 }
20963 OPTAB_DIRECT);
20965 /* Continue as if V8SImode (resp. V32QImode) was used initially. */
20970 }
20973 {
20975 /* The VPERMD and VPERMPS instructions already properly ignore
20976 the high bits of the shuffle elements. No need for us to
20977 perform an AND ourselves. */
20979 {
20984 }
20985 else
20986 {
20992 }
20999 else
21000 {
21006 }
21010 /* By combining the two 128-bit input vectors into one 256-bit
21011 input vector, we can use VPERMD and VPERMPS for the full
21012 two-operand shuffle. */
21044 /* From mask create two adjusted masks, which contain the same
21045 bits as mask in the low 7 bits of each vector element.
21046 The first mask will have the most significant bit clear
21047 if it requests element from the same 128-bit lane
21048 and MSB set if it requests element from the other 128-bit lane.
21049 The second mask will have the opposite values of the MSB,
21050 and additionally will have its 128-bit lanes swapped.
21051 E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
21052 t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
21053 t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
21054 stands for other 12 bytes. */
21055 /* The bit whether element is from the same lane or the other
21056 lane is bit 4, so shift it up by 3 to the MSB position. */
21060 /* Clear MSB bits from the mask just in case it had them set. */
21062 /* After this t1 will have MSB set for elements from other lane. */
21064 /* Clear bits other than MSB. */
21066 /* Or in the lower bits from mask into t3. */
21068 /* And invert MSB bits in t1, so MSB is set for elements from the same
21069 lane. */
21071 /* Swap 128-bit lanes in t3. */
21076 /* And or in the lower bits from mask into t1. */
21079 {
21080 /* Each of these shuffles will put 0s in places where
21081 element from the other 128-bit lane is needed, otherwise
21082 will shuffle in the requested value. */
21086 /* For t3 the 128-bit lanes are swapped again. */
21091 /* And oring both together leads to the result. */
21098 }
21101 /* Similarly to the above one_operand_shuffle code,
21102 just for repeated twice for each operand. merge_two:
21103 code will merge the two results together. */
21127 }
21128 }
21131 {
21132 /* The XOP VPPERM insn supports three inputs. By ignoring the
21133 one_operand_shuffle special case, we avoid creating another
21134 set of constant vectors in memory. */
21137 /* mask = mask & {2*w-1, ...} */
21139 }
21140 else
21141 {
21142 /* mask = mask & {w-1, ...} */
21144 }
21152 /* For non-QImode operations, convert the word permutation control
21153 into a byte permutation control. */
21155 {
21160 /* Convert mask to vector of chars. */
21163 /* Replicate each of the input bytes into byte positions:
21164 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
21165 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
21166 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
21173 else
21176 /* Convert it into the byte positions by doing
21177 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
21183 }
21185 /* The actual shuffle operations all operate on V16QImode. */
21190 {
21197 }
21199 {
21206 }
21207 else
21208 {
21212 /* Shuffle the two input vectors independently. */
21218 merge_two:
21219 /* Then merge them together. The key is whether any given control
21220 element contained a bit set that indicates the second word. */
21224 {
21225 /* Without SSE4.1, we don't have V2DImode EQ. Perform one
21226 more shuffle to convert the V2DI input mask into a V4SI
21227 input mask. At which point the masking that expand_int_vcond
21228 will work as desired. */
21236 }
21258 }
21259 }
21261 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
21262 true if we should do zero extension, else sign extension. HIGH_P is
21263 true if we want the N/2 high elements, else the low elements. */
21265 void
21267 {
21269 rtx tmp;
21272 {
21278 {
21282 else
21285 extract
21291 else
21294 extract
21300 else
21303 extract
21309 else
21315 else
21321 else
21326 }
21329 {
21332 }
21334 {
21335 /* Shift higher 8 bytes to lower 8 bytes. */
21340 }
21341 else
21345 }
21346 else
21347 {
21351 {
21355 else
21361 else
21367 else
21372 }
21376 else
21383 }
21384 }
21386 /* Expand conditional increment or decrement using adb/sbb instructions.
21387 The default case using setcc followed by the conditional move can be
21388 done by generic code. */
21389 bool
21391 {
21393 rtx flags;
21395 rtx compare_op;
21413 {
21416 }
21419 {
21423 reverse_condition_maybe_unordered
21425 else
21427 }
21431 /* Construct either adc or sbb insn. */
21433 {
21435 {
21450 }
21451 }
21452 else
21453 {
21455 {
21470 }
21471 }
21475 }
21478 /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
21479 but works for floating pointer parameters and nonoffsetable memories.
21480 For pushes, it returns just stack offsets; the values will be saved
21481 in the right order. Maximally three parts are generated. */
21483 static int
21485 {
21490 else
21496 /* Optimize constant pool reference to immediates. This is used by fp
21497 moves, that force all constants to memory to allow combining. */
21499 {
21503 }
21506 {
21507 /* The only non-offsetable memories we handle are pushes. */
21516 }
21519 {
21521 /* Caution: if we looked through a constant pool memory above,
21522 the operand may actually have a different mode now. That's
21523 ok, since we want to pun this all the way back to an integer. */
21527 }
21530 {
21533 else
21534 {
21538 {
21542 }
21544 {
21549 }
21551 {
21552 REAL_VALUE_TYPE r;
21557 {
21564 /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
21565 long double may not be 80-bit. */
21574 }
21577 }
21578 else
21580 }
21581 }
21582 else
21583 {
21587 {
21590 {
21594 }
21596 {
21600 }
21602 {
21603 REAL_VALUE_TYPE r;
21609 /* Do not use shift by 32 to avoid warning on 32bit systems. */
21612 = gen_int_mode
21615 DImode);
21616 else
21623 = gen_int_mode
21626 DImode);
21627 else
21629 }
21630 else
21632 }
21633 }
21636 }
21638 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
21639 Return false when normal moves are needed; true when all required
21640 insns have been emitted. Operands 2-4 contain the input values
21641 int the correct order; operands 5-7 contain the output values. */
21643 void
21645 {
21653 /* The DFmode expanders may ask us to move double.
21654 For 64bit target this is single move. By hiding the fact
21655 here we simplify i386.md splitters. */
21657 {
21658 /* Optimize constant pool reference to immediates. This is used by
21659 fp moves, that force all constants to memory to allow combining. */
21666 {
21669 }
21670 else
21675 }
21677 /* The only non-offsettable memory we handle is push. */
21680 else
21687 /* When emitting push, take care for source operands on the stack. */
21690 {
21693 /* Compensate for the stack decrement by 4. */
21698 /* src_base refers to the stack pointer and is
21699 automatically decreased by emitted push. */
21703 }
21705 /* We need to do copy in the right order in case an address register
21706 of the source overlaps the destination. */
21708 {
21709 rtx tmp;
21712 {
21716 collisions++;
21717 }
21719 /* Collision in the middle part can be handled by reordering. */
21721 {
21724 }
21728 {
21730 {
21733 }
21734 else
21735 {
21738 }
21739 }
21741 /* If there are more collisions, we can't handle it by reordering.
21742 Do an lea to the last part and use only one colliding move. */
21744 {
21745 rtx base;
21751 /* Handle the case when the last part isn't valid for lea.
21752 Happens in 64-bit mode storing the 12-byte XFmode. */
21759 {
21762 }
21763 }
21764 }
21767 {
21769 {
21771 {
21776 }
21778 {
21781 }
21782 }
21783 else
21784 {
21785 /* In 64bit mode we don't have 32bit push available. In case this is
21786 register, it is OK - we will just use larger counterpart. We also
21787 retype memory - these comes from attempt to avoid REX prefix on
21788 moving of second half of TFmode value. */
21790 {
21792 {
21803 }
21807 }
21808 }
21812 }
21814 /* Choose correct order to not overwrite the source before it is copied. */
21824 {
21826 {
21829 }
21830 }
21831 else
21832 {
21834 {
21837 }
21838 }
21840 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
21842 {
21851 }
21857 }
21859 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
21860 left shift by a constant, either using a single shift or
21861 a sequence of add instructions. */
21863 static void
21865 {
21871 {
21875 }
21876 else
21877 {
21880 }
21881 }
21883 void
21885 {
21894 {
21899 {
21905 }
21906 else
21907 {
21915 }
21917 }
21924 {
21925 /* Assuming we've chosen a QImode capable registers, then 1 << N
21926 can be done with two 32/64-bit shifts, no branches, no cmoves. */
21928 {
21944 }
21946 /* Otherwise, we can get the same results by manually performing
21947 a bit extract operation on bit 5/6, and then performing the two
21948 shifts. The two methods of getting 0/1 into low/high are exactly
21949 the same size. Avoiding the shift in the bit extract case helps
21950 pentium4 a bit; no one else seems to care much either way. */
21951 else
21952 {
21957 HOST_WIDE_INT bits;
21958 rtx x;
21961 {
21967 }
21968 else
21969 {
21975 }
21979 else
21987 }
21992 }
21995 {
21996 /* For -1 << N, we can avoid the shld instruction, because we
21997 know that we're shifting 0...31/63 ones into a -1. */
22001 else
22003 }
22004 else
22005 {
22013 }
22018 {
22024 }
22025 else
22026 {
22031 }
22032 }
22034 void
22036 {
22046 {
22051 {
22057 }
22059 {
22068 }
22069 else
22070 {
22078 }
22079 }
22080 else
22081 {
22093 {
22101 scratch));
22102 }
22103 else
22104 {
22109 }
22110 }
22111 }
22113 void
22115 {
22125 {
22130 {
22137 }
22138 else
22139 {
22147 }
22148 }
22149 else
22150 {
22162 {
22168 scratch));
22169 }
22170 else
22171 {
22176 }
22177 }
22178 }
22180 /* Predict just emitted jump instruction to be taken with probability PROB. */
22181 static void
22183 {
22187 }
22189 /* Helper function for the string operations below. Dest VARIABLE whether
22190 it is aligned to VALUE bytes. If true, jump to the label. */
22191 static rtx
22193 {
22198 else
22204 else
22207 }
22209 /* Adjust COUNTER by the VALUE. */
22210 static void
22212 {
22217 }
22219 /* Zero extend possibly SImode EXP to Pmode register. */
22220 rtx
22222 {
22224 }
22226 /* Divide COUNTREG by SCALE. */
22227 static rtx
22229 {
22230 rtx sc;
22242 }
22244 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
22245 DImode for constant loop counts. */
22247 static enum machine_mode
22249 {
22257 }
22259 /* Copy the address to a Pmode register. This is used for x32 to
22260 truncate DImode TLS address to a SImode register. */
22262 static rtx
22264 {
22267 else
22268 {
22271 }
22272 }
22274 /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
22275 to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
22276 specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
22277 memory by VALUE (supposed to be in MODE).
22279 The size is rounded down to whole number of chunk size moved at once.
22280 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
22283 static void
22288 {
22294 rtx size;
22303 /* Those two should combine. */
22305 {
22309 }
22316 /* This assert could be relaxed - in this case we'll need to compute
22317 smallest power of two, containing in PIECE_SIZE_N and pass it to
22318 offset_address. */
22324 {
22328 /* When unrolling for chips that reorder memory reads and writes,
22329 we can save registers by using single temporary.
22330 Also using 4 temporaries is overkill in 32bit mode. */
22332 {
22334 {
22336 {
22337 destmem =
22339 srcmem =
22341 }
22343 }
22344 }
22345 else
22346 {
22350 {
22353 {
22354 srcmem =
22356 }
22358 }
22360 {
22362 {
22363 destmem =
22365 }
22367 }
22368 }
22369 }
22370 else
22372 {
22374 destmem =
22377 }
22387 {
22393 else
22395 }
22396 else
22404 {
22409 }
22411 }
22413 /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
22414 When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
22415 When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
22416 For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
22417 ORIG_VALUE is the original value passed to memset to fill the memory with.
22418 Other arguments have same meaning as for previous function. */
22420 static void
22423 rtx count,
22425 {
22426 rtx destexp;
22427 rtx srcexp;
22428 rtx countreg;
22429 HOST_WIDE_INT rounded_count;
22431 /* If possible, it is shorter to use rep movs.
22432 TODO: Maybe it is better to move this logic to decide_alg. */
22443 {
22447 }
22448 else
22451 {
22456 }
22461 {
22464 }
22465 else
22466 {
22470 {
22474 }
22475 else
22478 {
22483 }
22484 else
22485 {
22488 }
22491 }
22492 }
22494 /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
22495 DESTMEM.
22496 SRC is passed by pointer to be updated on return.
22497 Return value is updated DST. */
22498 static rtx
22500 HOST_WIDE_INT size_to_move)
22501 {
22507 /* Find the widest mode in which we could perform moves.
22508 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
22509 it until move of such size is supported. */
22514 {
22518 }
22520 /* Find the corresponding vector mode with the same size as MOVE_MODE.
22521 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
22523 {
22528 {
22532 }
22533 }
22539 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
22543 {
22544 /* We move from memory to memory, so we'll need to do it via
22545 a temporary register. */
22556 piece_size);
22558 piece_size);
22559 }
22561 /* Update DST and SRC rtx. */
22564 }
22566 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
22567 static void
22570 {
22573 {
22578 /* For now MAX_SIZE should be a power of 2. This assert could be
22579 relaxed, but it'll require a bit more complicated epilogue
22580 expanding. */
22583 {
22586 }
22588 }
22590 {
22596 }
22598 /* When there are stringops, we can cheaply increase dest and src pointers.
22599 Otherwise we save code size by maintaining offset (zero is readily
22600 available from preceding rep operation) and using x86 addressing modes.
22601 */
22603 {
22605 {
22612 }
22614 {
22621 }
22623 {
22630 }
22631 }
22632 else
22633 {
22635 rtx tmp;
22638 {
22649 }
22651 {
22664 }
22666 {
22675 }
22676 }
22677 }
22679 /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
22680 with value PROMOTED_VAL.
22681 SRC is passed by pointer to be updated on return.
22682 Return value is updated DST. */
22683 static rtx
22685 HOST_WIDE_INT size_to_move)
22686 {
22692 /* Find the widest mode in which we could perform moves.
22693 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
22694 it until move of such size is supported. */
22699 {
22702 }
22709 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
22713 {
22715 {
22718 }
22726 piece_size);
22727 }
22729 /* Update DST rtx. */
22731 }
22732 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
22733 static void
22736 {
22737 count =
22743 }
22745 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
22746 static void
22749 {
22750 rtx dest;
22753 {
22758 /* For now MAX_SIZE should be a power of 2. This assert could be
22759 relaxed, but it'll require a bit more complicated epilogue
22760 expanding. */
22763 {
22765 {
22768 else
22770 }
22771 }
22773 }
22775 {
22778 }
22780 {
22783 {
22787 }
22788 else
22789 {
22795 }
22798 }
22800 {
22803 {
22806 }
22807 else
22808 {
22812 }
22815 }
22817 {
22823 }
22825 {
22831 }
22833 {
22839 }
22840 }
22842 /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
22843 DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
22844 Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
22845 ignored.
22846 Return value is updated DESTMEM. */
22847 static rtx
22852 {
22855 {
22857 {
22860 {
22863 else
22865 }
22866 else
22872 }
22873 }
22875 }
22877 /* Test if COUNT&SIZE is nonzero and if so, expand movme
22878 or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
22879 and jump to DONE_LABEL. */
22880 static void
22886 {
22889 rtx modesize;
22892 /* If we do not have vector value to copy, we must reduce size. */
22894 {
22896 {
22901 }
22902 else
22904 }
22905 else
22906 {
22907 /* Choose appropriate vector mode. */
22913 }
22918 {
22921 else
22922 {
22925 }
22927 }
22934 else
22935 {
22940 }
22946 }
22948 /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
22949 and get ready for the main memcpy loop by copying iniital DESIRED_ALIGN-ALIGN
22950 bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
22951 proceed with an loop copying SIZE bytes at once. Do moves in MODE.
22952 DONE_LABEL is a label after the whole copying sequence. The label is created
22953 on demand if *DONE_LABEL is NULL.
22954 MIN_SIZE is minimal size of block copied. This value gets adjusted for new
22955 bounds after the initial copies.
22957 DESTMEM/SRCMEM are memory expressions pointing to the copies block,
22958 DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
22959 we will dispatch to a library call for large blocks.
22961 In pseudocode we do:
22963 if (COUNT < SIZE)
22964 {
22965 Assume that SIZE is 4. Bigger sizes are handled analogously
22966 if (COUNT & 4)
22967 {
22968 copy 4 bytes from SRCPTR to DESTPTR
22969 copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
22970 goto done_label
22971 }
22972 if (!COUNT)
22973 goto done_label;
22974 copy 1 byte from SRCPTR to DESTPTR
22975 if (COUNT & 2)
22976 {
22977 copy 2 bytes from SRCPTR to DESTPTR
22978 copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
22979 }
22980 }
22981 else
22982 {
22983 copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
22984 copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
22986 OLD_DESPTR = DESTPTR;
22987 Align DESTPTR up to DESIRED_ALIGN
22988 SRCPTR += DESTPTR - OLD_DESTPTR
22989 COUNT -= DEST_PTR - OLD_DESTPTR
22990 if (DYNAMIC_CHECK)
22991 Round COUNT down to multiple of SIZE
22992 << optional caller supplied zero size guard is here >>
22993 << optional caller suppplied dynamic check is here >>
22994 << caller supplied main copy loop is here >>
22995 }
22996 done_label:
22997 */
22998 static void
23011 {
23014 rtx modesize;
23016 rtx mode_value;
23018 /* Chose proper value to copy. */
23021 else
23025 /* See if block is big or small, handle small blocks. */
23027 {
23038 /* Handle sizes > 3. */
23045 /* Nothing to copy? Jump to DONE_LABEL if so */
23049 /* Do a byte copy. */
23053 else
23054 {
23057 }
23059 /* Handle sizes 2 and 3. */
23066 else
23067 {
23072 }
23078 }
23079 else
23083 /* Start memcpy for COUNT >= SIZE. */
23085 {
23088 }
23090 /* Copy first desired_align bytes. */
23096 {
23099 else
23100 {
23103 }
23106 }
23109 /* Copy last SIZE bytes. */
23116 else
23117 {
23123 }
23125 {
23129 else
23130 {
23133 }
23134 }
23136 /* Align destination. */
23138 {
23142 /* Align destptr up, place it to new register. */
23149 /* See how many bytes we skipped. */
23153 /* Adjust srcptr and count. */
23159 /* We copied at most size + prolog_size. */
23162 else
23165 /* Our loops always round down the bock size, but for dispatch to library
23166 we need precise value. */
23170 }
23171 else
23172 {
23174 /* Decrease count, so we won't end up copying last word twice. */
23178 else
23182 }
23183 }
23186 /* This function is like the previous one, except here we know how many bytes
23187 need to be copied. That allows us to update alignment not only of DST, which
23188 is returned, but also of SRC, which is passed as a pointer for that
23189 reason. */
23190 static rtx
23195 {
23203 {
23207 }
23212 {
23214 {
23216 {
23219 else
23221 }
23222 else
23225 }
23226 }
23233 {
23239 {
23242 {
23246 }
23251 }
23255 }
23258 }
23260 /* Return true if ALG can be used in current context.
23261 Assume we expand memset if MEMSET is true. */
23262 static bool
23264 {
23269 /* Algorithms using the rep prefix want at least edi and ecx;
23270 additionally, memset wants eax and memcpy wants esi. Don't
23271 consider such algorithms if the user has appropriated those
23272 registers for their own purposes. */
23279 }
23281 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
23282 static enum stringop_alg
23286 {
23297 /* Even if the string operation call is cold, we still might spend a lot
23298 of time processing large blocks. */
23304 else
23310 else
23313 /* See maximal size for user defined algorithm. */
23315 {
23322 }
23324 /* If expected size is not known but max size is small enough
23325 so inline version is a win, set expected size into
23326 the range. */
23330 /* If user specified the algorithm, honnor it if possible. */
23334 /* rep; movq or rep; movl is the smallest variant. */
23336 {
23341 else
23344 }
23345 /* Very tiny blocks are best handled via the loop, REP is expensive to
23346 setup. */
23350 {
23354 {
23355 /* We get here if the algorithms that were not libcall-based
23356 were rep-prefix based and we are unable to use rep prefixes
23357 based on global register usage. Break out of the loop and
23358 use the heuristic below. */
23362 {
23366 {
23369 }
23370 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
23371 last non-libcall inline algorithm. */
23373 {
23374 /* When the current size is best to be copied by a libcall,
23375 but we are still forced to inline, run the heuristic below
23376 that will pick code for medium sized blocks. */
23378 {
23381 }
23383 }
23385 {
23388 }
23389 }
23390 }
23391 }
23392 /* When asked to inline the call anyway, try to pick meaningful choice.
23393 We look for maximal size of block that is faster to copy by hand and
23394 take blocks of at most of that size guessing that average size will
23395 be roughly half of the block.
23397 If this turns out to be bad, we might simply specify the preferred
23398 choice in ix86_costs. */
23402 {
23405 /* If there aren't any usable algorithms, then recursing on
23406 smaller sizes isn't going to find anything. Just return the
23407 simple byte-at-a-time copy loop. */
23409 {
23410 /* Pick something reasonable. */
23414 }
23424 }
23427 }
23429 /* Decide on alignment. We know that the operand is already aligned to ALIGN
23430 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
23431 static int
23436 {
23447 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
23448 copying whole cacheline at once. */
23461 }
23464 /* Helper function for memcpy. For QImode value 0xXY produce
23465 0xXYXYXYXY of wide specified by MODE. This is essentially
23466 a * 0x10101010, but we can do slightly better than
23467 synth_mult by unwinding the sequence by hand on CPUs with
23468 slow multiply. */
23469 static rtx
23471 {
23473 rtx tmp;
23480 {
23488 }
23497 nops--;
23502 {
23506 OPTAB_DIRECT);
23507 }
23508 else
23509 {
23515 else
23517 else
23518 {
23521 reg =
23523 }
23533 }
23534 }
23536 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
23537 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
23538 alignment from ALIGN to DESIRED_ALIGN. */
23539 static rtx
23542 {
23543 rtx promoted_val;
23552 else
23556 }
23558 /* Expand string move (memcpy) ot store (memset) operation. Use i386 string
23559 operations when profitable. The code depends upon architecture, block size
23560 and alignment, but always has one of the following overall structures:
23562 Aligned move sequence:
23564 1) Prologue guard: Conditional that jumps up to epilogues for small
23565 blocks that can be handled by epilogue alone. This is faster
23566 but also needed for correctness, since prologue assume the block
23567 is larger than the desired alignment.
23569 Optional dynamic check for size and libcall for large
23570 blocks is emitted here too, with -minline-stringops-dynamically.
23572 2) Prologue: copy first few bytes in order to get destination
23573 aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
23574 than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
23575 copied. We emit either a jump tree on power of two sized
23576 blocks, or a byte loop.
23578 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
23579 with specified algorithm.
23581 4) Epilogue: code copying tail of the block that is too small to be
23582 handled by main body (or up to size guarded by prologue guard).
23584 Misaligned move sequence
23586 1) missaligned move prologue/epilogue containing:
23587 a) Prologue handling small memory blocks and jumping to done_label
23588 (skipped if blocks are known to be large enough)
23589 b) Signle move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
23590 needed by single possibly misaligned move
23591 (skipped if alignment is not needed)
23592 c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
23594 2) Zero size guard dispatching to done_label, if needed
23596 3) dispatch to library call, if needed,
23598 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
23599 with specified algorithm. */
23600 static bool
23604 {
23605 rtx destreg;
23608 rtx tmp;
23624 /* TODO: Once vlaue ranges are available, fill in proper data. */
23631 /* i386 can do misaligned access on reasonably increased cost. */
23635 /* ALIGN is the minimum of destination and source alignment, but we care here
23636 just about destination alignment. */
23646 /* Make sure we don't need to care about overflow later on. */
23650 /* Step 0: Decide on preferred algorithm, desired alignment and
23651 size of chunks to be copied by main loop. */
23659 /* For now vector-version of memset is generated only for memory zeroing, as
23660 creating of promoted vector value is very cheap in this case. */
23673 {
23692 /* Find the widest supported mode. */
23698 /* Find the corresponding vector mode with the same size as MOVE_MODE.
23699 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
23701 {
23706 }
23718 }
23726 /* Step 1: Prologue guard. */
23728 /* Alignment code needs count to be in register. */
23730 {
23733 {
23734 align_bytes
23738 else
23740 }
23743 }
23746 /* Misaligned move sequences handles both prologues and epilogues at once.
23747 Default code generation results in smaller code for large alignments and
23748 also avoids redundant job when sizes are known precisely. */
23749 misaligned_prologue_used = (TARGET_MISALIGNED_MOVE_STRING_PROLOGUES
23754 /* Do the cheap promotion to allow better CSE across the
23755 main loop and epilogue (ie one load of the big constant in the
23756 front of all code.
23757 For now the misaligned move sequences do not have fast path
23758 without broadcasting. */
23760 {
23762 {
23768 }
23769 else
23770 {
23773 }
23774 }
23775 /* Misaligned move sequences handles both prologues and epilogues at once.
23776 Default code generation results in smaller code for large alignments and
23777 also avoids redundant job when sizes are known precisely. */
23779 {
23780 /* Misaligned move prologue handled small blocks by itself. */
23782 expand_set_or_movmem_prologue_epilogue_by_misaligned_moves
23787 desired_align < align
23796 {
23797 /* It is possible that we copied enough so the main loop will not
23798 execute. */
23808 else
23810 }
23811 }
23812 /* Ensure that alignment prologue won't copy past end of block. */
23814 {
23816 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
23817 Make sure it is power of 2. */
23820 /* To improve performance of small blocks, we jump around the VAL
23821 promoting mode. This mean that if the promoted VAL is not constant,
23822 we might not use it in the epilogue and have to use byte
23823 loop variant. */
23827 {
23829 {
23830 /* If main algorithm works on QImode, no epilogue is needed.
23831 For small sizes just don't align anything. */
23834 else
23836 }
23837 }
23839 {
23847 else
23849 }
23850 }
23852 /* Emit code to decide on runtime whether library call or inline should be
23853 used. */
23855 {
23857 {
23859 {
23863 }
23864 }
23865 else
23866 {
23874 else
23878 }
23879 }
23881 /* Step 2: Alignment prologue. */
23882 /* Do the expensive promotion once we branched off the small blocks. */
23888 {
23890 {
23891 /* Except for the first move in prologue, we no longer know
23892 constant offset in aliasing info. It don't seems to worth
23893 the pain to maintain it for the first move, so throw away
23894 the info early. */
23901 issetmem);
23902 /* At most desired_align - align bytes are copied. */
23905 else
23907 }
23908 else
23909 {
23910 /* If we know how many bytes need to be stored before dst is
23911 sufficiently aligned, maintain aliasing info accurately. */
23913 srcreg,
23914 promoted_val,
23915 vec_promoted_val,
23916 desired_align,
23917 align_bytes,
23918 issetmem);
23925 }
23927 && !min_size
23932 {
23933 /* It is possible that we copied enough so the main loop will not
23934 execute. */
23944 else
23946 }
23947 }
23949 {
23956 }
23960 /* Step 3: Main loop. */
23963 {
23986 }
23987 /* Adjust properly the offset of src and dest memory for aliasing. */
23989 {
23995 }
23996 else
23997 {
24001 }
24003 /* Step 4: Epilogue to copy the remaining bytes. */
24004 epilogue:
24006 {
24007 /* When the main loop is done, COUNT_EXP might hold original count,
24008 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
24009 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
24010 bytes. Compensate if needed. */
24013 {
24014 tmp =
24017 OPTAB_DIRECT);
24020 }
24023 }
24026 {
24029 epilogue_size_needed);
24030 else
24031 {
24035 epilogue_size_needed);
24036 else
24038 epilogue_size_needed);
24039 }
24040 }
24044 }
24046 /* Wrapper for ix86_expand_set_or_movmem for memcpy case. */
24047 bool
24050 {
24053 }
24055 /* Wrapper for ix86_expand_set_or_movmem for memset case. */
24056 bool
24059 {
24062 }
24065 /* Expand the appropriate insns for doing strlen if not just doing
24066 repnz; scasb
24068 out = result, initialized with the start address
24069 align_rtx = alignment of the address.
24070 scratch = scratch register, initialized with the startaddress when
24071 not aligned, otherwise undefined
24073 This is just the body. It needs the initializations mentioned above and
24074 some address computing at the end. These things are done in i386.md. */
24076 static void
24078 {
24080 rtx tmp;
24085 rtx mem;
24088 rtx cmp;
24094 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
24096 /* Is there a known alignment and is it less than 4? */
24098 {
24101 /* Is there a known alignment and is it not 2? */
24103 {
24107 /* Leave just the 3 lower bits. */
24117 }
24118 else
24119 {
24120 /* Since the alignment is 2, we have to check 2 or 0 bytes;
24121 check if is aligned to 4 - byte. */
24128 }
24132 /* Now compare the bytes. */
24134 /* Compare the first n unaligned byte on a byte per byte basis. */
24138 /* Increment the address. */
24141 /* Not needed with an alignment of 2 */
24143 {
24147 end_0_label);
24152 }
24155 end_0_label);
24158 }
24160 /* Generate loop to check 4 bytes at a time. It is not a good idea to
24161 align this loop. It gives only huge programs, but does not help to
24162 speed up. */
24169 /* This formula yields a nonzero result iff one of the bytes is zero.
24170 This saves three branches inside loop and many cycles. */
24178 align_4_label);
24181 {
24187 /* If zero is not in the first two bytes, move two bytes forward. */
24193 reg,
24194 tmpreg)));
24195 /* Emit lea manually to avoid clobbering of flags. */
24203 reg2,
24204 out)));
24205 }
24206 else
24207 {
24209 /* Is zero in the first two bytes? */
24216 pc_rtx);
24220 /* Not in the first two. Move two bytes forward. */
24226 }
24228 /* Avoid branch in fixing the byte. */
24236 }
24238 /* Expand strlen. */
24240 bool
24242 {
24245 /* The generic case of strlen expander is long. Avoid it's
24246 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
24249 && !TARGET_INLINE_ALL_STRINGOPS
24259 {
24260 /* Well it seems that some optimizer does not combine a call like
24261 foo(strlen(bar), strlen(bar));
24262 when the move and the subtraction is done here. It does calculate
24263 the length just once when these instructions are done inside of
24264 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
24265 often used and I use one fewer register for the lifetime of
24266 output_strlen_unroll() this is better. */
24272 /* strlensi_unroll_1 returns the address of the zero at the end of
24273 the string, like memchr(), so compute the length by subtracting
24274 the start address. */
24276 }
24277 else
24278 {
24279 rtx unspec;
24281 /* Can't use this if the user has appropriated eax, ecx, or edi. */
24294 /* If .md starts supporting :P, this can be done in .md. */
24300 }
24302 }
24304 /* For given symbol (function) construct code to compute address of it's PLT
24305 entry in large x86-64 PIC model. */
24306 static rtx
24308 {
24321 }
24323 rtx
24325 rtx callarg2,
24327 {
24328 unsigned int const cregs_size
24339 {
24340 #if TARGET_MACHO
24343 #endif
24344 }
24345 else
24346 {
24347 /* Static functions and indirect calls don't need the pic register. */
24349 && (!TARGET_64BIT
24355 }
24358 {
24362 }
24365 && !TARGET_PECOFF
24373 {
24376 }
24384 {
24388 }
24392 {
24396 UNSPEC_MS_TO_SYSV_CALL);
24399 {
24405 }
24406 }
24415 }
24417 /* Output the assembly for a call instruction. */
24421 {
24427 {
24430 /* SEH epilogue detection requires the indirect branch case
24431 to include REX.W. */
24434 else
24439 }
24441 /* SEH unwinding can require an extra nop to be emitted in several
24442 circumstances. Determine if we have one of those. */
24444 {
24445 rtx i;
24448 {
24449 /* If we get to another real insn, we don't need the nop. */
24453 /* If we get to the epilogue note, prevent a catch region from
24454 being adjacent to the standard epilogue sequence. If non-
24455 call-exceptions, we'll have done this during epilogue emission. */
24457 && !flag_non_call_exceptions
24459 {
24462 }
24463 }
24465 /* If we didn't find a real insn following the call, prevent the
24466 unwinder from looking into the next function. */
24469 }
24473 else
24482 }
24483 \f
24484 /* Clear stack slot assignments remembered from previous functions.
24485 This is called from INIT_EXPANDERS once before RTL is emitted for each
24486 function. */
24490 {
24498 }
24500 /* Return a MEM corresponding to a stack slot with mode MODE.
24501 Allocate a new slot if necessary.
24503 The RTL for a function can have several slots available: N is
24504 which slot to use. */
24506 rtx
24508 {
24525 }
24527 static void
24529 {
24535 }
24536 \f
24537 /* Calculate the length of the memory address in the instruction encoding.
24538 Includes addr32 prefix, does not include the one-byte modrm, opcode,
24539 or other prefixes. We never generate addr32 prefix for LEA insn. */
24541 int
24543 {
24560 /* If this is not LEA instruction, add the length of addr32 prefix. */
24565 len++;
24579 /* Rule of thumb:
24580 - esp as the base always wants an index,
24581 - ebp as the base always wants a displacement,
24582 - r12 as the base always wants an index,
24583 - r13 as the base always wants a displacement. */
24585 /* Register Indirect. */
24587 {
24588 /* esp (for its index) and ebp (for its displacement) need
24589 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
24590 code. */
24597 len++;
24598 }
24600 /* Direct Addressing. In 64-bit mode mod 00 r/m 5
24601 is not disp32, but disp32(%rip), so for disp32
24602 SIB byte is needed, unless print_operand_address
24603 optimizes it into disp32(%rip) or (%rip) is implied
24604 by UNSPEC. */
24606 {
24609 {
24625 len++;
24626 }
24627 }
24628 else
24629 {
24630 /* Find the length of the displacement constant. */
24632 {
24635 else
24637 }
24638 /* ebp always wants a displacement. Similarly r13. */
24640 len++;
24642 /* An index requires the two-byte modrm form.... */
24644 /* ...like esp (or r12), which always wants an index. */
24648 len++;
24649 }
24652 }
24654 /* Compute default value for "length_immediate" attribute. When SHORTFORM
24655 is set, expect that insn have 8bit immediate alternative. */
24656 int
24658 {
24664 {
24669 {
24672 {
24684 }
24686 {
24689 }
24690 }
24692 {
24702 /* Immediates for DImode instructions are encoded
24703 as 32bit sign extended values. */
24709 }
24710 }
24712 }
24714 /* Compute default value for "length_address" attribute. */
24715 int
24717 {
24721 {
24732 }
24737 {
24740 {
24745 constraints++;
24748 ;
24749 /* Skip ignored operands. */
24752 }
24754 }
24756 }
24758 /* Compute default value for "length_vex" attribute. It includes
24759 2 or 3 byte VEX prefix and 1 opcode byte. */
24761 int
24763 {
24766 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
24767 byte VEX prefix. */
24771 /* We can always use 2 byte VEX prefix in 32bit. */
24779 {
24780 /* REX.W bit uses 3 byte VEX prefix. */
24784 }
24785 else
24786 {
24787 /* REX.X or REX.B bits use 3 byte VEX prefix. */
24791 }
24794 }
24795 \f
24796 /* Return the maximum number of instructions a cpu can issue. */
24798 static int
24800 {
24802 {
24830 }
24831 }
24833 /* A subroutine of ix86_adjust_cost -- return TRUE iff INSN reads flags set
24834 by DEP_INSN and nothing set by DEP_INSN. */
24836 static bool
24838 {
24841 /* Simplify the test for uninteresting insns. */
24849 {
24852 }
24857 {
24860 }
24861 else
24867 /* This test is true if the dependent insn reads the flags but
24868 not any other potentially set register. */
24876 }
24878 /* Return true iff USE_INSN has a memory address with operands set by
24879 SET_INSN. */
24881 bool
24883 {
24888 {
24891 }
24893 }
24895 /* Helper function for exact_store_load_dependency.
24896 Return true if addr is found in insn. */
24897 static bool
24899 {
24906 {
24912 CASE_CONST_ANY:
24921 }
24925 {
24927 {
24937 }
24938 }
24940 }
24942 /* Return true if there exists exact dependency for store & load, i.e.
24943 the same memory address is used in them. */
24944 static bool
24946 {
24960 }
24962 static int
24964 {
24970 /* Anti and output dependencies have zero cost on all CPUs. */
24976 /* If we can't recognize the insns, we can't really do anything. */
24984 {
24986 /* Address Generation Interlock adds a cycle of latency. */
24988 {
24999 }
25003 /* ??? Compares pair with jump/setcc. */
25007 /* Floating point stores require value to be ready one cycle earlier. */
25017 /* INT->FP conversion is expensive. */
25021 /* There is one cycle extra latency between an FP op and a store. */
25029 /* Show ability of reorder buffer to hide latency of load by executing
25030 in parallel with previous instruction in case
25031 previous instruction is not needed to compute the address. */
25034 {
25035 /* Claim moves to take one cycle, as core can issue one load
25036 at time and the next load can start cycle later. */
25041 cost--;
25042 }
25048 /* The esp dependency is resolved before the instruction is really
25049 finished. */
25054 /* INT->FP conversion is expensive. */
25058 /* Show ability of reorder buffer to hide latency of load by executing
25059 in parallel with previous instruction in case
25060 previous instruction is not needed to compute the address. */
25063 {
25064 /* Claim moves to take one cycle, as core can issue one load
25065 at time and the next load can start cycle later. */
25071 else
25073 }
25087 /* Stack engine allows to execute push&pop instructions in parall. */
25093 /* Show ability of reorder buffer to hide latency of load by executing
25094 in parallel with previous instruction in case
25095 previous instruction is not needed to compute the address. */
25098 {
25102 /* Because of the difference between the length of integer and
25103 floating unit pipeline preparation stages, the memory operands
25104 for floating point are cheaper.
25106 ??? For Athlon it the difference is most probably 2. */
25109 else
25114 else
25116 }
25124 /* Stack engine allows to execute push&pop instructions in parall. */
25129 /* Show ability of reorder buffer to hide latency of load by executing
25130 in parallel with previous instruction in case
25131 previous instruction is not needed to compute the address. */
25134 {
25137 else
25139 }
25146 /* Increase cost of integer loads. */
25149 {
25152 {
25155 else
25156 {
25157 /* Increase cost of ld/st for short int types only
25158 because of store forwarding issue. */
25162 {
25163 /* Increase cost of store/load insn if exact
25164 dependence exists and it is load insn. */
25169 }
25170 }
25171 }
25172 }
25176 }
25179 }
25181 /* How many alternative schedules to try. This should be as wide as the
25182 scheduling freedom in the DFA, but no wider. Making this value too
25183 large results extra work for the scheduler. */
25185 static int
25187 {
25189 {
25202 /* Generally, we want haifa-sched:max_issue() to look ahead as far
25203 as many instructions can be executed on a cycle, i.e.,
25204 issue_rate. I wonder why tuning for many CPUs does not do this. */
25207 /* Don't use lookahead for pre-reload schedule to save compile time. */
25212 }
25213 }
25215 /* Try to reorder ready list to take advantage of Atom pipelined IMUL
25216 execution. It is applied if
25217 (1) IMUL instruction is on the top of list;
25218 (2) There exists the only producer of independent IMUL instruction in
25219 ready list.
25220 Return index of IMUL producer if it was found and -1 otherwise. */
25221 static int
25223 {
25225 sd_iterator_def sd_it;
25226 dep_t dep;
25233 /* Check that IMUL instruction is on the top of ready list. */
25242 /* Search for producer of independent IMUL instruction. */
25244 {
25248 /* Skip IMUL instruction. */
25258 {
25259 rtx con;
25270 {
25271 sd_iterator_def sd_it1;
25272 dep_t dep1;
25273 /* Check if there is no other dependee for IMUL. */
25276 {
25277 rtx pro;
25283 }
25286 }
25287 }
25290 }
25292 }
25294 /* Try to find the best candidate on the top of ready list if two insns
25295 have the same priority - candidate is best if its dependees were
25296 scheduled earlier. Applied for Silvermont only.
25297 Return true if top 2 insns must be interchanged. */
25298 static bool
25300 {
25303 rtx set;
25304 sd_iterator_def sd_it;
25305 dep_t dep;
25308 #define INSN_TICK(INSN) (HID (INSN)->tick)
25329 {
25332 /* Determine winner more precise. */
25334 {
25335 rtx pro;
25341 }
25343 {
25344 rtx pro;
25350 }
25353 {
25354 /* Determine winner - load must win. */
25360 }
25362 }
25364 #undef INSN_TICK
25365 }
25367 /* Perform possible reodering of ready list for Atom/Silvermont only.
25368 Return issue rate. */
25369 static int
25372 {
25376 rtx insn;
25379 /* Set up issue rate. */
25382 /* Do reodering for Atom/SLM only. */
25386 /* Nothing to do if ready list contains only 1 instruction. */
25390 /* Do reodering for post-reload scheduler only. */
25395 {
25400 /* Put IMUL producer (ready[index]) at the top of ready list. */
25406 }
25408 {
25412 /* Swap 2 top elements of ready list. */
25416 }
25418 }
25420 static bool
25423 /* Returns true if lhs of insn is HW function argument register and set up
25424 is_spilled to true if it is likely spilled HW register. */
25425 static bool
25427 {
25428 rtx dst;
25432 /* Call instructions are not movable, ignore it. */
25443 {
25444 /* Is it likely spilled HW register? */
25449 }
25451 }
25453 /* Add output dependencies for chain of function adjacent arguments if only
25454 there is a move to likely spilled HW register. Return first argument
25455 if at least one dependence was added or NULL otherwise. */
25456 static rtx
25458 {
25459 rtx insn;
25466 /* Find nearest to call argument passing instruction. */
25468 {
25477 }
25481 {
25488 {
25491 }
25493 {
25494 /* Add output depdendence between two function arguments if chain
25495 of output arguments contains likely spilled HW registers. */
25499 }
25500 else
25502 }
25506 }
25508 /* Add output or anti dependency from insn to first_arg to restrict its code
25509 motion. */
25510 static void
25512 {
25513 rtx set;
25514 rtx tmp;
25521 {
25522 /* Add output dependency to the first function argument. */
25525 }
25526 /* Add anti dependency. */
25528 }
25530 /* Avoid cross block motion of function argument through adding dependency
25531 from the first non-jump instruction in bb. */
25532 static void
25534 {
25538 {
25540 {
25543 {
25546 }
25547 }
25551 }
25552 }
25554 /* Hook for pre-reload schedule - avoid motion of function arguments
25555 passed in likely spilled HW registers. */
25556 static void
25558 {
25559 rtx insn;
25567 {
25570 {
25571 /* Add dependee for first argument to predecessors if only
25572 region contains more than one block. */
25576 /* Skip trivial regions and region head blocks that can have
25577 predecessors outside of region. */
25579 {
25580 edge e;
25581 edge_iterator ei;
25582 /* Assume that region is SCC, i.e. all immediate predecessors
25583 of non-head block are in the same region. */
25585 {
25586 /* Avoid creating of loop-carried dependencies through
25587 using topological odering in region. */
25590 }
25591 }
25595 }
25596 }
25599 }
25601 /* Hook for pre-reload schedule - set priority of moves from likely spilled
25602 HW registers to maximum, to schedule them at soon as possible. These are
25603 moves from function argument registers at the top of the function entry
25604 and moves from function return value registers after call. */
25605 static int
25607 {
25608 rtx set;
25618 {
25625 }
25628 }
25630 /* Model decoder of Core 2/i7.
25631 Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
25632 track the instruction fetch block boundaries and make sure that long
25633 (9+ bytes) instructions are assigned to D0. */
25635 /* Maximum length of an insn that can be handled by
25636 a secondary decoder unit. '8' for Core 2/i7. */
25639 /* Ifetch block size, i.e., number of bytes decoder reads per cycle.
25640 '16' for Core 2/i7. */
25643 /* Maximum number of instructions decoder can handle per cycle.
25644 '6' for Core 2/i7. */
25648 ix86_first_cycle_multipass_data_t;
25650 const_ix86_first_cycle_multipass_data_t;
25652 /* A variable to store target state across calls to max_issue within
25653 one cycle. */
25657 /* Initialize DATA. */
25658 static void
25660 {
25661 ix86_first_cycle_multipass_data_t data
25668 }
25670 /* Advancing the cycle; reset ifetch block counts. */
25671 static void
25673 {
25680 }
25684 /* Filter out insns from ready_try that the core will not be able to issue
25685 on current cycle due to decoder. */
25686 static void
25687 core2i7_first_cycle_multipass_filter_ready_try
25690 {
25692 {
25693 rtx insn;
25703 (!first_cycle_insn_p
25705 /* ... or it would not fit into the ifetch block ... */
25707 /* ... or the decoder is full already ... */
25709 /* ... mask the insn out. */
25710 {
25715 }
25716 }
25717 }
25719 /* Prepare for a new round of multipass lookahead scheduling. */
25720 static void
25723 {
25724 ix86_first_cycle_multipass_data_t data
25726 const_ix86_first_cycle_multipass_data_t prev_data
25729 /* Restore the state from the end of the previous round. */
25733 /* Filter instructions that cannot be issued on current cycle due to
25734 decoder restrictions. */
25736 first_cycle_insn_p);
25737 }
25739 /* INSN is being issued in current solution. Account for its impact on
25740 the decoder model. */
25741 static void
25744 {
25745 ix86_first_cycle_multipass_data_t data
25747 const_ix86_first_cycle_multipass_data_t prev_data
25757 /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
25759 {
25762 }
25764 {
25768 }
25771 /* Filter out insns from ready_try that the core will not be able to issue
25772 on current cycle due to decoder. */
25775 }
25777 /* Revert the effect on ready_try. */
25778 static void
25782 {
25783 const_ix86_first_cycle_multipass_data_t data
25786 sbitmap_iterator sbi;
25790 {
25792 }
25793 }
25795 /* Save the result of multipass lookahead scheduling for the next round. */
25796 static void
25798 {
25799 const_ix86_first_cycle_multipass_data_t data
25801 ix86_first_cycle_multipass_data_t next_data
25805 {
25808 }
25809 }
25811 /* Deallocate target data. */
25812 static void
25814 {
25815 ix86_first_cycle_multipass_data_t data
25819 {
25823 }
25824 }
25826 /* Prepare for scheduling pass. */
25827 static void
25831 {
25832 /* Install scheduling hooks for current CPU. Some of these hooks are used
25833 in time-critical parts of the scheduler, so we only set them up when
25834 they are actually used. */
25836 {
25840 /* Do not perform multipass scheduling for pre-reload schedule
25841 to save compile time. */
25843 {
25859 /* Set decoder parameters. */
25864 }
25865 /* ... Fall through ... */
25875 }
25876 }
25878 \f
25879 /* Compute the alignment given to a constant that is being placed in memory.
25880 EXP is the constant and ALIGN is the alignment that the object would
25881 ordinarily have.
25882 The value of this function is used instead of that alignment to align
25883 the object. */
25885 int
25887 {
25890 {
25895 }
25901 }
25903 /* Compute the alignment for a static variable.
25904 TYPE is the data type, and ALIGN is the alignment that
25905 the object would ordinarily have. The value of this function is used
25906 instead of that alignment to align the object. */
25908 int
25910 {
25922 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
25923 to 16byte boundary. */
25925 {
25932 }
25938 {
25943 }
25945 {
25952 }
25957 {
25962 }
25965 {
25970 }
25973 }
25975 /* Compute the alignment for a local variable or a stack slot. EXP is
25976 the data type or decl itself, MODE is the widest mode available and
25977 ALIGN is the alignment that the object would ordinarily have. The
25978 value of this macro is used instead of that alignment to align the
25979 object. */
25981 unsigned int
25984 {
25988 {
25991 }
25992 else
25993 {
25996 }
25998 /* Don't do dynamic stack realignment for long long objects with
25999 -mpreferred-stack-boundary=2. */
26008 /* If TYPE is NULL, we are allocating a stack slot for caller-save
26009 register in MODE. We will return the largest alignment of XF
26010 and DF. */
26012 {
26016 }
26018 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
26019 to 16byte boundary. Exact wording is:
26021 An array uses the same alignment as its elements, except that a local or
26022 global array variable of length at least 16 bytes or
26023 a C99 variable-length array variable always has alignment of at least 16 bytes.
26025 This was added to allow use of aligned SSE instructions at arrays. This
26026 rule is meant for static storage (where compiler can not do the analysis
26027 by itself). We follow it for automatic variables only when convenient.
26028 We fully control everything in the function compiled and functions from
26029 other unit can not rely on the alignment.
26031 Exclude va_list type. It is the common case of local array where
26032 we can not benefit from the alignment.
26034 TODO: Probably one should optimize for size only when var is not escaping. */
26037 {
26047 }
26049 {
26054 }
26056 {
26062 }
26067 {
26072 }
26075 {
26081 }
26083 }
26085 /* Compute the minimum required alignment for dynamic stack realignment
26086 purposes for a local variable, parameter or a stack slot. EXP is
26087 the data type or decl itself, MODE is its mode and ALIGN is the
26088 alignment that the object would ordinarily have. */
26090 unsigned int
26093 {
26097 {
26100 }
26101 else
26102 {
26105 }
26110 /* Don't do dynamic stack realignment for long long objects with
26111 -mpreferred-stack-boundary=2. */
26118 }
26119 \f
26120 /* Find a location for the static chain incoming to a nested function.
26121 This is a register, unless all free registers are used by arguments. */
26123 static rtx
26125 {
26132 {
26133 /* We always use R10 in 64-bit mode. */
26135 }
26136 else
26137 {
26138 tree fntype;
26141 /* By default in 32-bit mode we use ECX to pass the static chain. */
26147 {
26148 /* Fastcall functions use ecx/edx for arguments, which leaves
26149 us with EAX for the static chain.
26150 Thiscall functions use ecx for arguments, which also
26151 leaves us with EAX for the static chain. */
26153 }
26155 {
26156 /* Thiscall functions use ecx for arguments, which leaves
26157 us with EAX and EDX for the static chain.
26158 We are using for abi-compatibility EAX. */
26160 }
26162 {
26163 /* For regparm 3, we have no free call-clobbered registers in
26164 which to store the static chain. In order to implement this,
26165 we have the trampoline push the static chain to the stack.
26166 However, we can't push a value below the return address when
26167 we call the nested function directly, so we have to use an
26168 alternate entry point. For this we use ESI, and have the
26169 alternate entry point push ESI, so that things appear the
26170 same once we're executing the nested function. */
26172 {
26178 }
26180 }
26181 }
26184 }
26186 /* Emit RTL insns to initialize the variable parts of a trampoline.
26187 FNDECL is the decl of the target address; M_TRAMP is a MEM for
26188 the trampoline, and CHAIN_VALUE is an RTX for the static chain
26189 to be passed to the target function. */
26191 static void
26193 {
26201 {
26204 /* Load the function address to r11. Try to load address using
26205 the shorter movl instead of movabs. We may want to support
26206 movq for kernel mode, but kernel does not use trampolines at
26207 the moment. FNADDR is a 32bit address and may not be in
26208 DImode when ptr_mode == SImode. Always use movl in this
26209 case. */
26212 {
26221 }
26222 else
26223 {
26230 }
26232 /* Load static chain using movabs to r10. Use the shorter movl
26233 instead of movabs when ptr_mode == SImode. */
26235 {
26238 }
26239 else
26240 {
26243 }
26252 /* Jump to r11; the last (unused) byte is a nop, only there to
26253 pad the write out to a single 32-bit store. */
26257 }
26258 else
26259 {
26262 /* Depending on the static chain location, either load a register
26263 with a constant, or push the constant to the stack. All of the
26264 instructions are the same size. */
26267 {
26269 {
26276 }
26277 }
26278 else
26293 /* Compute offset from the end of the jmp to the target function.
26294 In the case in which the trampoline stores the static chain on
26295 the stack, we need to skip the first insn which pushes the
26296 (call-saved) register static chain; this push is 1 byte. */
26303 }
26307 #ifdef HAVE_ENABLE_EXECUTE_STACK
26308 #ifdef CHECK_EXECUTE_STACK_ENABLED
26310 #endif
26313 #endif
26314 }
26315 \f
26316 /* The following file contains several enumerations and data structures
26317 built from the definitions in i386-builtin-types.def. */
26321 /* Table for the ix86 builtin non-function types. */
26324 /* Retrieve an element from the above table, building some of
26325 the types lazily. */
26327 static tree
26329 {
26341 {
26349 }
26350 else
26351 {
26357 else
26365 }
26369 }
26371 /* Table for the ix86 builtin function types. */
26374 /* Retrieve an element from the above table, building some of
26375 the types lazily. */
26377 static tree
26379 {
26380 tree type;
26389 {
26397 {
26400 }
26403 }
26404 else
26405 {
26411 }
26415 }
26418 /* Codes for all the SSE/MMX builtins. */
26419 enum ix86_builtins
26420 {
26421 IX86_BUILTIN_ADDPS,
26422 IX86_BUILTIN_ADDSS,
26423 IX86_BUILTIN_DIVPS,
26424 IX86_BUILTIN_DIVSS,
26425 IX86_BUILTIN_MULPS,
26426 IX86_BUILTIN_MULSS,
26427 IX86_BUILTIN_SUBPS,
26428 IX86_BUILTIN_SUBSS,
26430 IX86_BUILTIN_CMPEQPS,
26431 IX86_BUILTIN_CMPLTPS,
26432 IX86_BUILTIN_CMPLEPS,
26433 IX86_BUILTIN_CMPGTPS,
26434 IX86_BUILTIN_CMPGEPS,
26435 IX86_BUILTIN_CMPNEQPS,
26436 IX86_BUILTIN_CMPNLTPS,
26437 IX86_BUILTIN_CMPNLEPS,
26438 IX86_BUILTIN_CMPNGTPS,
26439 IX86_BUILTIN_CMPNGEPS,
26440 IX86_BUILTIN_CMPORDPS,
26441 IX86_BUILTIN_CMPUNORDPS,
26442 IX86_BUILTIN_CMPEQSS,
26443 IX86_BUILTIN_CMPLTSS,
26444 IX86_BUILTIN_CMPLESS,
26445 IX86_BUILTIN_CMPNEQSS,
26446 IX86_BUILTIN_CMPNLTSS,
26447 IX86_BUILTIN_CMPNLESS,
26448 IX86_BUILTIN_CMPORDSS,
26449 IX86_BUILTIN_CMPUNORDSS,
26451 IX86_BUILTIN_COMIEQSS,
26452 IX86_BUILTIN_COMILTSS,
26453 IX86_BUILTIN_COMILESS,
26454 IX86_BUILTIN_COMIGTSS,
26455 IX86_BUILTIN_COMIGESS,
26456 IX86_BUILTIN_COMINEQSS,
26457 IX86_BUILTIN_UCOMIEQSS,
26458 IX86_BUILTIN_UCOMILTSS,
26459 IX86_BUILTIN_UCOMILESS,
26460 IX86_BUILTIN_UCOMIGTSS,
26461 IX86_BUILTIN_UCOMIGESS,
26462 IX86_BUILTIN_UCOMINEQSS,
26464 IX86_BUILTIN_CVTPI2PS,
26465 IX86_BUILTIN_CVTPS2PI,
26466 IX86_BUILTIN_CVTSI2SS,
26467 IX86_BUILTIN_CVTSI642SS,
26468 IX86_BUILTIN_CVTSS2SI,
26469 IX86_BUILTIN_CVTSS2SI64,
26470 IX86_BUILTIN_CVTTPS2PI,
26471 IX86_BUILTIN_CVTTSS2SI,
26472 IX86_BUILTIN_CVTTSS2SI64,
26474 IX86_BUILTIN_MAXPS,
26475 IX86_BUILTIN_MAXSS,
26476 IX86_BUILTIN_MINPS,
26477 IX86_BUILTIN_MINSS,
26479 IX86_BUILTIN_LOADUPS,
26480 IX86_BUILTIN_STOREUPS,
26481 IX86_BUILTIN_MOVSS,
26483 IX86_BUILTIN_MOVHLPS,
26484 IX86_BUILTIN_MOVLHPS,
26485 IX86_BUILTIN_LOADHPS,
26486 IX86_BUILTIN_LOADLPS,
26487 IX86_BUILTIN_STOREHPS,
26488 IX86_BUILTIN_STORELPS,
26490 IX86_BUILTIN_MASKMOVQ,
26491 IX86_BUILTIN_MOVMSKPS,
26492 IX86_BUILTIN_PMOVMSKB,
26494 IX86_BUILTIN_MOVNTPS,
26495 IX86_BUILTIN_MOVNTQ,
26497 IX86_BUILTIN_LOADDQU,
26498 IX86_BUILTIN_STOREDQU,
26500 IX86_BUILTIN_PACKSSWB,
26501 IX86_BUILTIN_PACKSSDW,
26502 IX86_BUILTIN_PACKUSWB,
26504 IX86_BUILTIN_PADDB,
26505 IX86_BUILTIN_PADDW,
26506 IX86_BUILTIN_PADDD,
26507 IX86_BUILTIN_PADDQ,
26508 IX86_BUILTIN_PADDSB,
26509 IX86_BUILTIN_PADDSW,
26510 IX86_BUILTIN_PADDUSB,
26511 IX86_BUILTIN_PADDUSW,
26512 IX86_BUILTIN_PSUBB,
26513 IX86_BUILTIN_PSUBW,
26514 IX86_BUILTIN_PSUBD,
26515 IX86_BUILTIN_PSUBQ,
26516 IX86_BUILTIN_PSUBSB,
26517 IX86_BUILTIN_PSUBSW,
26518 IX86_BUILTIN_PSUBUSB,
26519 IX86_BUILTIN_PSUBUSW,
26521 IX86_BUILTIN_PAND,
26522 IX86_BUILTIN_PANDN,
26523 IX86_BUILTIN_POR,
26524 IX86_BUILTIN_PXOR,
26526 IX86_BUILTIN_PAVGB,
26527 IX86_BUILTIN_PAVGW,
26529 IX86_BUILTIN_PCMPEQB,
26530 IX86_BUILTIN_PCMPEQW,
26531 IX86_BUILTIN_PCMPEQD,
26532 IX86_BUILTIN_PCMPGTB,
26533 IX86_BUILTIN_PCMPGTW,
26534 IX86_BUILTIN_PCMPGTD,
26536 IX86_BUILTIN_PMADDWD,
26538 IX86_BUILTIN_PMAXSW,
26539 IX86_BUILTIN_PMAXUB,
26540 IX86_BUILTIN_PMINSW,
26541 IX86_BUILTIN_PMINUB,
26543 IX86_BUILTIN_PMULHUW,
26544 IX86_BUILTIN_PMULHW,
26545 IX86_BUILTIN_PMULLW,
26547 IX86_BUILTIN_PSADBW,
26548 IX86_BUILTIN_PSHUFW,
26550 IX86_BUILTIN_PSLLW,
26551 IX86_BUILTIN_PSLLD,
26552 IX86_BUILTIN_PSLLQ,
26553 IX86_BUILTIN_PSRAW,
26554 IX86_BUILTIN_PSRAD,
26555 IX86_BUILTIN_PSRLW,
26556 IX86_BUILTIN_PSRLD,
26557 IX86_BUILTIN_PSRLQ,
26558 IX86_BUILTIN_PSLLWI,
26559 IX86_BUILTIN_PSLLDI,
26560 IX86_BUILTIN_PSLLQI,
26561 IX86_BUILTIN_PSRAWI,
26562 IX86_BUILTIN_PSRADI,
26563 IX86_BUILTIN_PSRLWI,
26564 IX86_BUILTIN_PSRLDI,
26565 IX86_BUILTIN_PSRLQI,
26567 IX86_BUILTIN_PUNPCKHBW,
26568 IX86_BUILTIN_PUNPCKHWD,
26569 IX86_BUILTIN_PUNPCKHDQ,
26570 IX86_BUILTIN_PUNPCKLBW,
26571 IX86_BUILTIN_PUNPCKLWD,
26572 IX86_BUILTIN_PUNPCKLDQ,
26574 IX86_BUILTIN_SHUFPS,
26576 IX86_BUILTIN_RCPPS,
26577 IX86_BUILTIN_RCPSS,
26578 IX86_BUILTIN_RSQRTPS,
26579 IX86_BUILTIN_RSQRTPS_NR,
26580 IX86_BUILTIN_RSQRTSS,
26581 IX86_BUILTIN_RSQRTF,
26582 IX86_BUILTIN_SQRTPS,
26583 IX86_BUILTIN_SQRTPS_NR,
26584 IX86_BUILTIN_SQRTSS,
26586 IX86_BUILTIN_UNPCKHPS,
26587 IX86_BUILTIN_UNPCKLPS,
26589 IX86_BUILTIN_ANDPS,
26590 IX86_BUILTIN_ANDNPS,
26591 IX86_BUILTIN_ORPS,
26592 IX86_BUILTIN_XORPS,
26594 IX86_BUILTIN_EMMS,
26595 IX86_BUILTIN_LDMXCSR,
26596 IX86_BUILTIN_STMXCSR,
26597 IX86_BUILTIN_SFENCE,
26599 IX86_BUILTIN_FXSAVE,
26600 IX86_BUILTIN_FXRSTOR,
26601 IX86_BUILTIN_FXSAVE64,
26602 IX86_BUILTIN_FXRSTOR64,
26604 IX86_BUILTIN_XSAVE,
26605 IX86_BUILTIN_XRSTOR,
26606 IX86_BUILTIN_XSAVE64,
26607 IX86_BUILTIN_XRSTOR64,
26609 IX86_BUILTIN_XSAVEOPT,
26610 IX86_BUILTIN_XSAVEOPT64,
26612 /* 3DNow! Original */
26613 IX86_BUILTIN_FEMMS,
26614 IX86_BUILTIN_PAVGUSB,
26615 IX86_BUILTIN_PF2ID,
26616 IX86_BUILTIN_PFACC,
26617 IX86_BUILTIN_PFADD,
26618 IX86_BUILTIN_PFCMPEQ,
26619 IX86_BUILTIN_PFCMPGE,
26620 IX86_BUILTIN_PFCMPGT,
26621 IX86_BUILTIN_PFMAX,
26622 IX86_BUILTIN_PFMIN,
26623 IX86_BUILTIN_PFMUL,
26624 IX86_BUILTIN_PFRCP,
26625 IX86_BUILTIN_PFRCPIT1,
26626 IX86_BUILTIN_PFRCPIT2,
26627 IX86_BUILTIN_PFRSQIT1,
26628 IX86_BUILTIN_PFRSQRT,
26629 IX86_BUILTIN_PFSUB,
26630 IX86_BUILTIN_PFSUBR,
26631 IX86_BUILTIN_PI2FD,
26632 IX86_BUILTIN_PMULHRW,
26634 /* 3DNow! Athlon Extensions */
26635 IX86_BUILTIN_PF2IW,
26636 IX86_BUILTIN_PFNACC,
26637 IX86_BUILTIN_PFPNACC,
26638 IX86_BUILTIN_PI2FW,
26639 IX86_BUILTIN_PSWAPDSI,
26640 IX86_BUILTIN_PSWAPDSF,
26642 /* SSE2 */
26643 IX86_BUILTIN_ADDPD,
26644 IX86_BUILTIN_ADDSD,
26645 IX86_BUILTIN_DIVPD,
26646 IX86_BUILTIN_DIVSD,
26647 IX86_BUILTIN_MULPD,
26648 IX86_BUILTIN_MULSD,
26649 IX86_BUILTIN_SUBPD,
26650 IX86_BUILTIN_SUBSD,
26652 IX86_BUILTIN_CMPEQPD,
26653 IX86_BUILTIN_CMPLTPD,
26654 IX86_BUILTIN_CMPLEPD,
26655 IX86_BUILTIN_CMPGTPD,
26656 IX86_BUILTIN_CMPGEPD,
26657 IX86_BUILTIN_CMPNEQPD,
26658 IX86_BUILTIN_CMPNLTPD,
26659 IX86_BUILTIN_CMPNLEPD,
26660 IX86_BUILTIN_CMPNGTPD,
26661 IX86_BUILTIN_CMPNGEPD,
26662 IX86_BUILTIN_CMPORDPD,
26663 IX86_BUILTIN_CMPUNORDPD,
26664 IX86_BUILTIN_CMPEQSD,
26665 IX86_BUILTIN_CMPLTSD,
26666 IX86_BUILTIN_CMPLESD,
26667 IX86_BUILTIN_CMPNEQSD,
26668 IX86_BUILTIN_CMPNLTSD,
26669 IX86_BUILTIN_CMPNLESD,
26670 IX86_BUILTIN_CMPORDSD,
26671 IX86_BUILTIN_CMPUNORDSD,
26673 IX86_BUILTIN_COMIEQSD,
26674 IX86_BUILTIN_COMILTSD,
26675 IX86_BUILTIN_COMILESD,
26676 IX86_BUILTIN_COMIGTSD,
26677 IX86_BUILTIN_COMIGESD,
26678 IX86_BUILTIN_COMINEQSD,
26679 IX86_BUILTIN_UCOMIEQSD,
26680 IX86_BUILTIN_UCOMILTSD,
26681 IX86_BUILTIN_UCOMILESD,
26682 IX86_BUILTIN_UCOMIGTSD,
26683 IX86_BUILTIN_UCOMIGESD,
26684 IX86_BUILTIN_UCOMINEQSD,
26686 IX86_BUILTIN_MAXPD,
26687 IX86_BUILTIN_MAXSD,
26688 IX86_BUILTIN_MINPD,
26689 IX86_BUILTIN_MINSD,
26691 IX86_BUILTIN_ANDPD,
26692 IX86_BUILTIN_ANDNPD,
26693 IX86_BUILTIN_ORPD,
26694 IX86_BUILTIN_XORPD,
26696 IX86_BUILTIN_SQRTPD,
26697 IX86_BUILTIN_SQRTSD,
26699 IX86_BUILTIN_UNPCKHPD,
26700 IX86_BUILTIN_UNPCKLPD,
26702 IX86_BUILTIN_SHUFPD,
26704 IX86_BUILTIN_LOADUPD,
26705 IX86_BUILTIN_STOREUPD,
26706 IX86_BUILTIN_MOVSD,
26708 IX86_BUILTIN_LOADHPD,
26709 IX86_BUILTIN_LOADLPD,
26711 IX86_BUILTIN_CVTDQ2PD,
26712 IX86_BUILTIN_CVTDQ2PS,
26714 IX86_BUILTIN_CVTPD2DQ,
26715 IX86_BUILTIN_CVTPD2PI,
26716 IX86_BUILTIN_CVTPD2PS,
26717 IX86_BUILTIN_CVTTPD2DQ,
26718 IX86_BUILTIN_CVTTPD2PI,
26720 IX86_BUILTIN_CVTPI2PD,
26721 IX86_BUILTIN_CVTSI2SD,
26722 IX86_BUILTIN_CVTSI642SD,
26724 IX86_BUILTIN_CVTSD2SI,
26725 IX86_BUILTIN_CVTSD2SI64,
26726 IX86_BUILTIN_CVTSD2SS,
26727 IX86_BUILTIN_CVTSS2SD,
26728 IX86_BUILTIN_CVTTSD2SI,
26729 IX86_BUILTIN_CVTTSD2SI64,
26731 IX86_BUILTIN_CVTPS2DQ,
26732 IX86_BUILTIN_CVTPS2PD,
26733 IX86_BUILTIN_CVTTPS2DQ,
26735 IX86_BUILTIN_MOVNTI,
26736 IX86_BUILTIN_MOVNTI64,
26737 IX86_BUILTIN_MOVNTPD,
26738 IX86_BUILTIN_MOVNTDQ,
26740 IX86_BUILTIN_MOVQ128,
26742 /* SSE2 MMX */
26743 IX86_BUILTIN_MASKMOVDQU,
26744 IX86_BUILTIN_MOVMSKPD,
26745 IX86_BUILTIN_PMOVMSKB128,
26747 IX86_BUILTIN_PACKSSWB128,
26748 IX86_BUILTIN_PACKSSDW128,
26749 IX86_BUILTIN_PACKUSWB128,
26751 IX86_BUILTIN_PADDB128,
26752 IX86_BUILTIN_PADDW128,
26753 IX86_BUILTIN_PADDD128,
26754 IX86_BUILTIN_PADDQ128,
26755 IX86_BUILTIN_PADDSB128,
26756 IX86_BUILTIN_PADDSW128,
26757 IX86_BUILTIN_PADDUSB128,
26758 IX86_BUILTIN_PADDUSW128,
26759 IX86_BUILTIN_PSUBB128,
26760 IX86_BUILTIN_PSUBW128,
26761 IX86_BUILTIN_PSUBD128,
26762 IX86_BUILTIN_PSUBQ128,
26763 IX86_BUILTIN_PSUBSB128,
26764 IX86_BUILTIN_PSUBSW128,
26765 IX86_BUILTIN_PSUBUSB128,
26766 IX86_BUILTIN_PSUBUSW128,
26768 IX86_BUILTIN_PAND128,
26769 IX86_BUILTIN_PANDN128,
26770 IX86_BUILTIN_POR128,
26771 IX86_BUILTIN_PXOR128,
26773 IX86_BUILTIN_PAVGB128,
26774 IX86_BUILTIN_PAVGW128,
26776 IX86_BUILTIN_PCMPEQB128,
26777 IX86_BUILTIN_PCMPEQW128,
26778 IX86_BUILTIN_PCMPEQD128,
26779 IX86_BUILTIN_PCMPGTB128,
26780 IX86_BUILTIN_PCMPGTW128,
26781 IX86_BUILTIN_PCMPGTD128,
26783 IX86_BUILTIN_PMADDWD128,
26785 IX86_BUILTIN_PMAXSW128,
26786 IX86_BUILTIN_PMAXUB128,
26787 IX86_BUILTIN_PMINSW128,
26788 IX86_BUILTIN_PMINUB128,
26790 IX86_BUILTIN_PMULUDQ,
26791 IX86_BUILTIN_PMULUDQ128,
26792 IX86_BUILTIN_PMULHUW128,
26793 IX86_BUILTIN_PMULHW128,
26794 IX86_BUILTIN_PMULLW128,
26796 IX86_BUILTIN_PSADBW128,
26797 IX86_BUILTIN_PSHUFHW,
26798 IX86_BUILTIN_PSHUFLW,
26799 IX86_BUILTIN_PSHUFD,
26801 IX86_BUILTIN_PSLLDQI128,
26802 IX86_BUILTIN_PSLLWI128,
26803 IX86_BUILTIN_PSLLDI128,
26804 IX86_BUILTIN_PSLLQI128,
26805 IX86_BUILTIN_PSRAWI128,
26806 IX86_BUILTIN_PSRADI128,
26807 IX86_BUILTIN_PSRLDQI128,
26808 IX86_BUILTIN_PSRLWI128,
26809 IX86_BUILTIN_PSRLDI128,
26810 IX86_BUILTIN_PSRLQI128,
26812 IX86_BUILTIN_PSLLDQ128,
26813 IX86_BUILTIN_PSLLW128,
26814 IX86_BUILTIN_PSLLD128,
26815 IX86_BUILTIN_PSLLQ128,
26816 IX86_BUILTIN_PSRAW128,
26817 IX86_BUILTIN_PSRAD128,
26818 IX86_BUILTIN_PSRLW128,
26819 IX86_BUILTIN_PSRLD128,
26820 IX86_BUILTIN_PSRLQ128,
26822 IX86_BUILTIN_PUNPCKHBW128,
26823 IX86_BUILTIN_PUNPCKHWD128,
26824 IX86_BUILTIN_PUNPCKHDQ128,
26825 IX86_BUILTIN_PUNPCKHQDQ128,
26826 IX86_BUILTIN_PUNPCKLBW128,
26827 IX86_BUILTIN_PUNPCKLWD128,
26828 IX86_BUILTIN_PUNPCKLDQ128,
26829 IX86_BUILTIN_PUNPCKLQDQ128,
26831 IX86_BUILTIN_CLFLUSH,
26832 IX86_BUILTIN_MFENCE,
26833 IX86_BUILTIN_LFENCE,
26834 IX86_BUILTIN_PAUSE,
26836 IX86_BUILTIN_BSRSI,
26837 IX86_BUILTIN_BSRDI,
26838 IX86_BUILTIN_RDPMC,
26839 IX86_BUILTIN_RDTSC,
26840 IX86_BUILTIN_RDTSCP,
26841 IX86_BUILTIN_ROLQI,
26842 IX86_BUILTIN_ROLHI,
26843 IX86_BUILTIN_RORQI,
26844 IX86_BUILTIN_RORHI,
26846 /* SSE3. */
26847 IX86_BUILTIN_ADDSUBPS,
26848 IX86_BUILTIN_HADDPS,
26849 IX86_BUILTIN_HSUBPS,
26850 IX86_BUILTIN_MOVSHDUP,
26851 IX86_BUILTIN_MOVSLDUP,
26852 IX86_BUILTIN_ADDSUBPD,
26853 IX86_BUILTIN_HADDPD,
26854 IX86_BUILTIN_HSUBPD,
26855 IX86_BUILTIN_LDDQU,
26857 IX86_BUILTIN_MONITOR,
26858 IX86_BUILTIN_MWAIT,
26860 /* SSSE3. */
26861 IX86_BUILTIN_PHADDW,
26862 IX86_BUILTIN_PHADDD,
26863 IX86_BUILTIN_PHADDSW,
26864 IX86_BUILTIN_PHSUBW,
26865 IX86_BUILTIN_PHSUBD,
26866 IX86_BUILTIN_PHSUBSW,
26867 IX86_BUILTIN_PMADDUBSW,
26868 IX86_BUILTIN_PMULHRSW,
26869 IX86_BUILTIN_PSHUFB,
26870 IX86_BUILTIN_PSIGNB,
26871 IX86_BUILTIN_PSIGNW,
26872 IX86_BUILTIN_PSIGND,
26873 IX86_BUILTIN_PALIGNR,
26874 IX86_BUILTIN_PABSB,
26875 IX86_BUILTIN_PABSW,
26876 IX86_BUILTIN_PABSD,
26878 IX86_BUILTIN_PHADDW128,
26879 IX86_BUILTIN_PHADDD128,
26880 IX86_BUILTIN_PHADDSW128,
26881 IX86_BUILTIN_PHSUBW128,
26882 IX86_BUILTIN_PHSUBD128,
26883 IX86_BUILTIN_PHSUBSW128,
26884 IX86_BUILTIN_PMADDUBSW128,
26885 IX86_BUILTIN_PMULHRSW128,
26886 IX86_BUILTIN_PSHUFB128,
26887 IX86_BUILTIN_PSIGNB128,
26888 IX86_BUILTIN_PSIGNW128,
26889 IX86_BUILTIN_PSIGND128,
26890 IX86_BUILTIN_PALIGNR128,
26891 IX86_BUILTIN_PABSB128,
26892 IX86_BUILTIN_PABSW128,
26893 IX86_BUILTIN_PABSD128,
26895 /* AMDFAM10 - SSE4A New Instructions. */
26896 IX86_BUILTIN_MOVNTSD,
26897 IX86_BUILTIN_MOVNTSS,
26898 IX86_BUILTIN_EXTRQI,
26899 IX86_BUILTIN_EXTRQ,
26900 IX86_BUILTIN_INSERTQI,
26901 IX86_BUILTIN_INSERTQ,
26903 /* SSE4.1. */
26904 IX86_BUILTIN_BLENDPD,
26905 IX86_BUILTIN_BLENDPS,
26906 IX86_BUILTIN_BLENDVPD,
26907 IX86_BUILTIN_BLENDVPS,
26908 IX86_BUILTIN_PBLENDVB128,
26909 IX86_BUILTIN_PBLENDW128,
26911 IX86_BUILTIN_DPPD,
26912 IX86_BUILTIN_DPPS,
26914 IX86_BUILTIN_INSERTPS128,
26916 IX86_BUILTIN_MOVNTDQA,
26917 IX86_BUILTIN_MPSADBW128,
26918 IX86_BUILTIN_PACKUSDW128,
26919 IX86_BUILTIN_PCMPEQQ,
26920 IX86_BUILTIN_PHMINPOSUW128,
26922 IX86_BUILTIN_PMAXSB128,
26923 IX86_BUILTIN_PMAXSD128,
26924 IX86_BUILTIN_PMAXUD128,
26925 IX86_BUILTIN_PMAXUW128,
26927 IX86_BUILTIN_PMINSB128,
26928 IX86_BUILTIN_PMINSD128,
26929 IX86_BUILTIN_PMINUD128,
26930 IX86_BUILTIN_PMINUW128,
26932 IX86_BUILTIN_PMOVSXBW128,
26933 IX86_BUILTIN_PMOVSXBD128,
26934 IX86_BUILTIN_PMOVSXBQ128,
26935 IX86_BUILTIN_PMOVSXWD128,
26936 IX86_BUILTIN_PMOVSXWQ128,
26937 IX86_BUILTIN_PMOVSXDQ128,
26939 IX86_BUILTIN_PMOVZXBW128,
26940 IX86_BUILTIN_PMOVZXBD128,
26941 IX86_BUILTIN_PMOVZXBQ128,
26942 IX86_BUILTIN_PMOVZXWD128,
26943 IX86_BUILTIN_PMOVZXWQ128,
26944 IX86_BUILTIN_PMOVZXDQ128,
26946 IX86_BUILTIN_PMULDQ128,
26947 IX86_BUILTIN_PMULLD128,
26949 IX86_BUILTIN_ROUNDSD,
26950 IX86_BUILTIN_ROUNDSS,
26952 IX86_BUILTIN_ROUNDPD,
26953 IX86_BUILTIN_ROUNDPS,
26955 IX86_BUILTIN_FLOORPD,
26956 IX86_BUILTIN_CEILPD,
26957 IX86_BUILTIN_TRUNCPD,
26958 IX86_BUILTIN_RINTPD,
26959 IX86_BUILTIN_ROUNDPD_AZ,
26961 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
26962 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
26963 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
26965 IX86_BUILTIN_FLOORPS,
26966 IX86_BUILTIN_CEILPS,
26967 IX86_BUILTIN_TRUNCPS,
26968 IX86_BUILTIN_RINTPS,
26969 IX86_BUILTIN_ROUNDPS_AZ,
26971 IX86_BUILTIN_FLOORPS_SFIX,
26972 IX86_BUILTIN_CEILPS_SFIX,
26973 IX86_BUILTIN_ROUNDPS_AZ_SFIX,
26975 IX86_BUILTIN_PTESTZ,
26976 IX86_BUILTIN_PTESTC,
26977 IX86_BUILTIN_PTESTNZC,
26979 IX86_BUILTIN_VEC_INIT_V2SI,
26980 IX86_BUILTIN_VEC_INIT_V4HI,
26981 IX86_BUILTIN_VEC_INIT_V8QI,
26982 IX86_BUILTIN_VEC_EXT_V2DF,
26983 IX86_BUILTIN_VEC_EXT_V2DI,
26984 IX86_BUILTIN_VEC_EXT_V4SF,
26985 IX86_BUILTIN_VEC_EXT_V4SI,
26986 IX86_BUILTIN_VEC_EXT_V8HI,
26987 IX86_BUILTIN_VEC_EXT_V2SI,
26988 IX86_BUILTIN_VEC_EXT_V4HI,
26989 IX86_BUILTIN_VEC_EXT_V16QI,
26990 IX86_BUILTIN_VEC_SET_V2DI,
26991 IX86_BUILTIN_VEC_SET_V4SF,
26992 IX86_BUILTIN_VEC_SET_V4SI,
26993 IX86_BUILTIN_VEC_SET_V8HI,
26994 IX86_BUILTIN_VEC_SET_V4HI,
26995 IX86_BUILTIN_VEC_SET_V16QI,
26997 IX86_BUILTIN_VEC_PACK_SFIX,
26998 IX86_BUILTIN_VEC_PACK_SFIX256,
27000 /* SSE4.2. */
27001 IX86_BUILTIN_CRC32QI,
27002 IX86_BUILTIN_CRC32HI,
27003 IX86_BUILTIN_CRC32SI,
27004 IX86_BUILTIN_CRC32DI,
27006 IX86_BUILTIN_PCMPESTRI128,
27007 IX86_BUILTIN_PCMPESTRM128,
27008 IX86_BUILTIN_PCMPESTRA128,
27009 IX86_BUILTIN_PCMPESTRC128,
27010 IX86_BUILTIN_PCMPESTRO128,
27011 IX86_BUILTIN_PCMPESTRS128,
27012 IX86_BUILTIN_PCMPESTRZ128,
27013 IX86_BUILTIN_PCMPISTRI128,
27014 IX86_BUILTIN_PCMPISTRM128,
27015 IX86_BUILTIN_PCMPISTRA128,
27016 IX86_BUILTIN_PCMPISTRC128,
27017 IX86_BUILTIN_PCMPISTRO128,
27018 IX86_BUILTIN_PCMPISTRS128,
27019 IX86_BUILTIN_PCMPISTRZ128,
27021 IX86_BUILTIN_PCMPGTQ,
27023 /* AES instructions */
27024 IX86_BUILTIN_AESENC128,
27025 IX86_BUILTIN_AESENCLAST128,
27026 IX86_BUILTIN_AESDEC128,
27027 IX86_BUILTIN_AESDECLAST128,
27028 IX86_BUILTIN_AESIMC128,
27029 IX86_BUILTIN_AESKEYGENASSIST128,
27031 /* PCLMUL instruction */
27032 IX86_BUILTIN_PCLMULQDQ128,
27034 /* AVX */
27035 IX86_BUILTIN_ADDPD256,
27036 IX86_BUILTIN_ADDPS256,
27037 IX86_BUILTIN_ADDSUBPD256,
27038 IX86_BUILTIN_ADDSUBPS256,
27039 IX86_BUILTIN_ANDPD256,
27040 IX86_BUILTIN_ANDPS256,
27041 IX86_BUILTIN_ANDNPD256,
27042 IX86_BUILTIN_ANDNPS256,
27043 IX86_BUILTIN_BLENDPD256,
27044 IX86_BUILTIN_BLENDPS256,
27045 IX86_BUILTIN_BLENDVPD256,
27046 IX86_BUILTIN_BLENDVPS256,
27047 IX86_BUILTIN_DIVPD256,
27048 IX86_BUILTIN_DIVPS256,
27049 IX86_BUILTIN_DPPS256,
27050 IX86_BUILTIN_HADDPD256,
27051 IX86_BUILTIN_HADDPS256,
27052 IX86_BUILTIN_HSUBPD256,
27053 IX86_BUILTIN_HSUBPS256,
27054 IX86_BUILTIN_MAXPD256,
27055 IX86_BUILTIN_MAXPS256,
27056 IX86_BUILTIN_MINPD256,
27057 IX86_BUILTIN_MINPS256,
27058 IX86_BUILTIN_MULPD256,
27059 IX86_BUILTIN_MULPS256,
27060 IX86_BUILTIN_ORPD256,
27061 IX86_BUILTIN_ORPS256,
27062 IX86_BUILTIN_SHUFPD256,
27063 IX86_BUILTIN_SHUFPS256,
27064 IX86_BUILTIN_SUBPD256,
27065 IX86_BUILTIN_SUBPS256,
27066 IX86_BUILTIN_XORPD256,
27067 IX86_BUILTIN_XORPS256,
27068 IX86_BUILTIN_CMPSD,
27069 IX86_BUILTIN_CMPSS,
27070 IX86_BUILTIN_CMPPD,
27071 IX86_BUILTIN_CMPPS,
27072 IX86_BUILTIN_CMPPD256,
27073 IX86_BUILTIN_CMPPS256,
27074 IX86_BUILTIN_CVTDQ2PD256,
27075 IX86_BUILTIN_CVTDQ2PS256,
27076 IX86_BUILTIN_CVTPD2PS256,
27077 IX86_BUILTIN_CVTPS2DQ256,
27078 IX86_BUILTIN_CVTPS2PD256,
27079 IX86_BUILTIN_CVTTPD2DQ256,
27080 IX86_BUILTIN_CVTPD2DQ256,
27081 IX86_BUILTIN_CVTTPS2DQ256,
27082 IX86_BUILTIN_EXTRACTF128PD256,
27083 IX86_BUILTIN_EXTRACTF128PS256,
27084 IX86_BUILTIN_EXTRACTF128SI256,
27085 IX86_BUILTIN_VZEROALL,
27086 IX86_BUILTIN_VZEROUPPER,
27087 IX86_BUILTIN_VPERMILVARPD,
27088 IX86_BUILTIN_VPERMILVARPS,
27089 IX86_BUILTIN_VPERMILVARPD256,
27090 IX86_BUILTIN_VPERMILVARPS256,
27091 IX86_BUILTIN_VPERMILPD,
27092 IX86_BUILTIN_VPERMILPS,
27093 IX86_BUILTIN_VPERMILPD256,
27094 IX86_BUILTIN_VPERMILPS256,
27095 IX86_BUILTIN_VPERMIL2PD,
27096 IX86_BUILTIN_VPERMIL2PS,
27097 IX86_BUILTIN_VPERMIL2PD256,
27098 IX86_BUILTIN_VPERMIL2PS256,
27099 IX86_BUILTIN_VPERM2F128PD256,
27100 IX86_BUILTIN_VPERM2F128PS256,
27101 IX86_BUILTIN_VPERM2F128SI256,
27102 IX86_BUILTIN_VBROADCASTSS,
27103 IX86_BUILTIN_VBROADCASTSD256,
27104 IX86_BUILTIN_VBROADCASTSS256,
27105 IX86_BUILTIN_VBROADCASTPD256,
27106 IX86_BUILTIN_VBROADCASTPS256,
27107 IX86_BUILTIN_VINSERTF128PD256,
27108 IX86_BUILTIN_VINSERTF128PS256,
27109 IX86_BUILTIN_VINSERTF128SI256,
27110 IX86_BUILTIN_LOADUPD256,
27111 IX86_BUILTIN_LOADUPS256,
27112 IX86_BUILTIN_STOREUPD256,
27113 IX86_BUILTIN_STOREUPS256,
27114 IX86_BUILTIN_LDDQU256,
27115 IX86_BUILTIN_MOVNTDQ256,
27116 IX86_BUILTIN_MOVNTPD256,
27117 IX86_BUILTIN_MOVNTPS256,
27118 IX86_BUILTIN_LOADDQU256,
27119 IX86_BUILTIN_STOREDQU256,
27120 IX86_BUILTIN_MASKLOADPD,
27121 IX86_BUILTIN_MASKLOADPS,
27122 IX86_BUILTIN_MASKSTOREPD,
27123 IX86_BUILTIN_MASKSTOREPS,
27124 IX86_BUILTIN_MASKLOADPD256,
27125 IX86_BUILTIN_MASKLOADPS256,
27126 IX86_BUILTIN_MASKSTOREPD256,
27127 IX86_BUILTIN_MASKSTOREPS256,
27128 IX86_BUILTIN_MOVSHDUP256,
27129 IX86_BUILTIN_MOVSLDUP256,
27130 IX86_BUILTIN_MOVDDUP256,
27132 IX86_BUILTIN_SQRTPD256,
27133 IX86_BUILTIN_SQRTPS256,
27134 IX86_BUILTIN_SQRTPS_NR256,
27135 IX86_BUILTIN_RSQRTPS256,
27136 IX86_BUILTIN_RSQRTPS_NR256,
27138 IX86_BUILTIN_RCPPS256,
27140 IX86_BUILTIN_ROUNDPD256,
27141 IX86_BUILTIN_ROUNDPS256,
27143 IX86_BUILTIN_FLOORPD256,
27144 IX86_BUILTIN_CEILPD256,
27145 IX86_BUILTIN_TRUNCPD256,
27146 IX86_BUILTIN_RINTPD256,
27147 IX86_BUILTIN_ROUNDPD_AZ256,
27149 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
27150 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
27151 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
27153 IX86_BUILTIN_FLOORPS256,
27154 IX86_BUILTIN_CEILPS256,
27155 IX86_BUILTIN_TRUNCPS256,
27156 IX86_BUILTIN_RINTPS256,
27157 IX86_BUILTIN_ROUNDPS_AZ256,
27159 IX86_BUILTIN_FLOORPS_SFIX256,
27160 IX86_BUILTIN_CEILPS_SFIX256,
27161 IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
27163 IX86_BUILTIN_UNPCKHPD256,
27164 IX86_BUILTIN_UNPCKLPD256,
27165 IX86_BUILTIN_UNPCKHPS256,
27166 IX86_BUILTIN_UNPCKLPS256,
27168 IX86_BUILTIN_SI256_SI,
27169 IX86_BUILTIN_PS256_PS,
27170 IX86_BUILTIN_PD256_PD,
27171 IX86_BUILTIN_SI_SI256,
27172 IX86_BUILTIN_PS_PS256,
27173 IX86_BUILTIN_PD_PD256,
27175 IX86_BUILTIN_VTESTZPD,
27176 IX86_BUILTIN_VTESTCPD,
27177 IX86_BUILTIN_VTESTNZCPD,
27178 IX86_BUILTIN_VTESTZPS,
27179 IX86_BUILTIN_VTESTCPS,
27180 IX86_BUILTIN_VTESTNZCPS,
27181 IX86_BUILTIN_VTESTZPD256,
27182 IX86_BUILTIN_VTESTCPD256,
27183 IX86_BUILTIN_VTESTNZCPD256,
27184 IX86_BUILTIN_VTESTZPS256,
27185 IX86_BUILTIN_VTESTCPS256,
27186 IX86_BUILTIN_VTESTNZCPS256,
27187 IX86_BUILTIN_PTESTZ256,
27188 IX86_BUILTIN_PTESTC256,
27189 IX86_BUILTIN_PTESTNZC256,
27191 IX86_BUILTIN_MOVMSKPD256,
27192 IX86_BUILTIN_MOVMSKPS256,
27194 /* AVX2 */
27195 IX86_BUILTIN_MPSADBW256,
27196 IX86_BUILTIN_PABSB256,
27197 IX86_BUILTIN_PABSW256,
27198 IX86_BUILTIN_PABSD256,
27199 IX86_BUILTIN_PACKSSDW256,
27200 IX86_BUILTIN_PACKSSWB256,
27201 IX86_BUILTIN_PACKUSDW256,
27202 IX86_BUILTIN_PACKUSWB256,
27203 IX86_BUILTIN_PADDB256,
27204 IX86_BUILTIN_PADDW256,
27205 IX86_BUILTIN_PADDD256,
27206 IX86_BUILTIN_PADDQ256,
27207 IX86_BUILTIN_PADDSB256,
27208 IX86_BUILTIN_PADDSW256,
27209 IX86_BUILTIN_PADDUSB256,
27210 IX86_BUILTIN_PADDUSW256,
27211 IX86_BUILTIN_PALIGNR256,
27212 IX86_BUILTIN_AND256I,
27213 IX86_BUILTIN_ANDNOT256I,
27214 IX86_BUILTIN_PAVGB256,
27215 IX86_BUILTIN_PAVGW256,
27216 IX86_BUILTIN_PBLENDVB256,
27217 IX86_BUILTIN_PBLENDVW256,
27218 IX86_BUILTIN_PCMPEQB256,
27219 IX86_BUILTIN_PCMPEQW256,
27220 IX86_BUILTIN_PCMPEQD256,
27221 IX86_BUILTIN_PCMPEQQ256,
27222 IX86_BUILTIN_PCMPGTB256,
27223 IX86_BUILTIN_PCMPGTW256,
27224 IX86_BUILTIN_PCMPGTD256,
27225 IX86_BUILTIN_PCMPGTQ256,
27226 IX86_BUILTIN_PHADDW256,
27227 IX86_BUILTIN_PHADDD256,
27228 IX86_BUILTIN_PHADDSW256,
27229 IX86_BUILTIN_PHSUBW256,
27230 IX86_BUILTIN_PHSUBD256,
27231 IX86_BUILTIN_PHSUBSW256,
27232 IX86_BUILTIN_PMADDUBSW256,
27233 IX86_BUILTIN_PMADDWD256,
27234 IX86_BUILTIN_PMAXSB256,
27235 IX86_BUILTIN_PMAXSW256,
27236 IX86_BUILTIN_PMAXSD256,
27237 IX86_BUILTIN_PMAXUB256,
27238 IX86_BUILTIN_PMAXUW256,
27239 IX86_BUILTIN_PMAXUD256,
27240 IX86_BUILTIN_PMINSB256,
27241 IX86_BUILTIN_PMINSW256,
27242 IX86_BUILTIN_PMINSD256,
27243 IX86_BUILTIN_PMINUB256,
27244 IX86_BUILTIN_PMINUW256,
27245 IX86_BUILTIN_PMINUD256,
27246 IX86_BUILTIN_PMOVMSKB256,
27247 IX86_BUILTIN_PMOVSXBW256,
27248 IX86_BUILTIN_PMOVSXBD256,
27249 IX86_BUILTIN_PMOVSXBQ256,
27250 IX86_BUILTIN_PMOVSXWD256,
27251 IX86_BUILTIN_PMOVSXWQ256,
27252 IX86_BUILTIN_PMOVSXDQ256,
27253 IX86_BUILTIN_PMOVZXBW256,
27254 IX86_BUILTIN_PMOVZXBD256,
27255 IX86_BUILTIN_PMOVZXBQ256,
27256 IX86_BUILTIN_PMOVZXWD256,
27257 IX86_BUILTIN_PMOVZXWQ256,
27258 IX86_BUILTIN_PMOVZXDQ256,
27259 IX86_BUILTIN_PMULDQ256,
27260 IX86_BUILTIN_PMULHRSW256,
27261 IX86_BUILTIN_PMULHUW256,
27262 IX86_BUILTIN_PMULHW256,
27263 IX86_BUILTIN_PMULLW256,
27264 IX86_BUILTIN_PMULLD256,
27265 IX86_BUILTIN_PMULUDQ256,
27266 IX86_BUILTIN_POR256,
27267 IX86_BUILTIN_PSADBW256,
27268 IX86_BUILTIN_PSHUFB256,
27269 IX86_BUILTIN_PSHUFD256,
27270 IX86_BUILTIN_PSHUFHW256,
27271 IX86_BUILTIN_PSHUFLW256,
27272 IX86_BUILTIN_PSIGNB256,
27273 IX86_BUILTIN_PSIGNW256,
27274 IX86_BUILTIN_PSIGND256,
27275 IX86_BUILTIN_PSLLDQI256,
27276 IX86_BUILTIN_PSLLWI256,
27277 IX86_BUILTIN_PSLLW256,
27278 IX86_BUILTIN_PSLLDI256,
27279 IX86_BUILTIN_PSLLD256,
27280 IX86_BUILTIN_PSLLQI256,
27281 IX86_BUILTIN_PSLLQ256,
27282 IX86_BUILTIN_PSRAWI256,
27283 IX86_BUILTIN_PSRAW256,
27284 IX86_BUILTIN_PSRADI256,
27285 IX86_BUILTIN_PSRAD256,
27286 IX86_BUILTIN_PSRLDQI256,
27287 IX86_BUILTIN_PSRLWI256,
27288 IX86_BUILTIN_PSRLW256,
27289 IX86_BUILTIN_PSRLDI256,
27290 IX86_BUILTIN_PSRLD256,
27291 IX86_BUILTIN_PSRLQI256,
27292 IX86_BUILTIN_PSRLQ256,
27293 IX86_BUILTIN_PSUBB256,
27294 IX86_BUILTIN_PSUBW256,
27295 IX86_BUILTIN_PSUBD256,
27296 IX86_BUILTIN_PSUBQ256,
27297 IX86_BUILTIN_PSUBSB256,
27298 IX86_BUILTIN_PSUBSW256,
27299 IX86_BUILTIN_PSUBUSB256,
27300 IX86_BUILTIN_PSUBUSW256,
27301 IX86_BUILTIN_PUNPCKHBW256,
27302 IX86_BUILTIN_PUNPCKHWD256,
27303 IX86_BUILTIN_PUNPCKHDQ256,
27304 IX86_BUILTIN_PUNPCKHQDQ256,
27305 IX86_BUILTIN_PUNPCKLBW256,
27306 IX86_BUILTIN_PUNPCKLWD256,
27307 IX86_BUILTIN_PUNPCKLDQ256,
27308 IX86_BUILTIN_PUNPCKLQDQ256,
27309 IX86_BUILTIN_PXOR256,
27310 IX86_BUILTIN_MOVNTDQA256,
27311 IX86_BUILTIN_VBROADCASTSS_PS,
27312 IX86_BUILTIN_VBROADCASTSS_PS256,
27313 IX86_BUILTIN_VBROADCASTSD_PD256,
27314 IX86_BUILTIN_VBROADCASTSI256,
27315 IX86_BUILTIN_PBLENDD256,
27316 IX86_BUILTIN_PBLENDD128,
27317 IX86_BUILTIN_PBROADCASTB256,
27318 IX86_BUILTIN_PBROADCASTW256,
27319 IX86_BUILTIN_PBROADCASTD256,
27320 IX86_BUILTIN_PBROADCASTQ256,
27321 IX86_BUILTIN_PBROADCASTB128,
27322 IX86_BUILTIN_PBROADCASTW128,
27323 IX86_BUILTIN_PBROADCASTD128,
27324 IX86_BUILTIN_PBROADCASTQ128,
27325 IX86_BUILTIN_VPERMVARSI256,
27326 IX86_BUILTIN_VPERMDF256,
27327 IX86_BUILTIN_VPERMVARSF256,
27328 IX86_BUILTIN_VPERMDI256,
27329 IX86_BUILTIN_VPERMTI256,
27330 IX86_BUILTIN_VEXTRACT128I256,
27331 IX86_BUILTIN_VINSERT128I256,
27332 IX86_BUILTIN_MASKLOADD,
27333 IX86_BUILTIN_MASKLOADQ,
27334 IX86_BUILTIN_MASKLOADD256,
27335 IX86_BUILTIN_MASKLOADQ256,
27336 IX86_BUILTIN_MASKSTORED,
27337 IX86_BUILTIN_MASKSTOREQ,
27338 IX86_BUILTIN_MASKSTORED256,
27339 IX86_BUILTIN_MASKSTOREQ256,
27340 IX86_BUILTIN_PSLLVV4DI,
27341 IX86_BUILTIN_PSLLVV2DI,
27342 IX86_BUILTIN_PSLLVV8SI,
27343 IX86_BUILTIN_PSLLVV4SI,
27344 IX86_BUILTIN_PSRAVV8SI,
27345 IX86_BUILTIN_PSRAVV4SI,
27346 IX86_BUILTIN_PSRLVV4DI,
27347 IX86_BUILTIN_PSRLVV2DI,
27348 IX86_BUILTIN_PSRLVV8SI,
27349 IX86_BUILTIN_PSRLVV4SI,
27351 IX86_BUILTIN_GATHERSIV2DF,
27352 IX86_BUILTIN_GATHERSIV4DF,
27353 IX86_BUILTIN_GATHERDIV2DF,
27354 IX86_BUILTIN_GATHERDIV4DF,
27355 IX86_BUILTIN_GATHERSIV4SF,
27356 IX86_BUILTIN_GATHERSIV8SF,
27357 IX86_BUILTIN_GATHERDIV4SF,
27358 IX86_BUILTIN_GATHERDIV8SF,
27359 IX86_BUILTIN_GATHERSIV2DI,
27360 IX86_BUILTIN_GATHERSIV4DI,
27361 IX86_BUILTIN_GATHERDIV2DI,
27362 IX86_BUILTIN_GATHERDIV4DI,
27363 IX86_BUILTIN_GATHERSIV4SI,
27364 IX86_BUILTIN_GATHERSIV8SI,
27365 IX86_BUILTIN_GATHERDIV4SI,
27366 IX86_BUILTIN_GATHERDIV8SI,
27368 /* Alternate 4 element gather for the vectorizer where
27369 all operands are 32-byte wide. */
27370 IX86_BUILTIN_GATHERALTSIV4DF,
27371 IX86_BUILTIN_GATHERALTDIV8SF,
27372 IX86_BUILTIN_GATHERALTSIV4DI,
27373 IX86_BUILTIN_GATHERALTDIV8SI,
27375 /* TFmode support builtins. */
27376 IX86_BUILTIN_INFQ,
27377 IX86_BUILTIN_HUGE_VALQ,
27378 IX86_BUILTIN_FABSQ,
27379 IX86_BUILTIN_COPYSIGNQ,
27381 /* Vectorizer support builtins. */
27382 IX86_BUILTIN_CPYSGNPS,
27383 IX86_BUILTIN_CPYSGNPD,
27384 IX86_BUILTIN_CPYSGNPS256,
27385 IX86_BUILTIN_CPYSGNPD256,
27387 /* FMA4 instructions. */
27388 IX86_BUILTIN_VFMADDSS,
27389 IX86_BUILTIN_VFMADDSD,
27390 IX86_BUILTIN_VFMADDPS,
27391 IX86_BUILTIN_VFMADDPD,
27392 IX86_BUILTIN_VFMADDPS256,
27393 IX86_BUILTIN_VFMADDPD256,
27394 IX86_BUILTIN_VFMADDSUBPS,
27395 IX86_BUILTIN_VFMADDSUBPD,
27396 IX86_BUILTIN_VFMADDSUBPS256,
27397 IX86_BUILTIN_VFMADDSUBPD256,
27399 /* FMA3 instructions. */
27400 IX86_BUILTIN_VFMADDSS3,
27401 IX86_BUILTIN_VFMADDSD3,
27403 /* XOP instructions. */
27404 IX86_BUILTIN_VPCMOV,
27405 IX86_BUILTIN_VPCMOV_V2DI,
27406 IX86_BUILTIN_VPCMOV_V4SI,
27407 IX86_BUILTIN_VPCMOV_V8HI,
27408 IX86_BUILTIN_VPCMOV_V16QI,
27409 IX86_BUILTIN_VPCMOV_V4SF,
27410 IX86_BUILTIN_VPCMOV_V2DF,
27411 IX86_BUILTIN_VPCMOV256,
27412 IX86_BUILTIN_VPCMOV_V4DI256,
27413 IX86_BUILTIN_VPCMOV_V8SI256,
27414 IX86_BUILTIN_VPCMOV_V16HI256,
27415 IX86_BUILTIN_VPCMOV_V32QI256,
27416 IX86_BUILTIN_VPCMOV_V8SF256,
27417 IX86_BUILTIN_VPCMOV_V4DF256,
27419 IX86_BUILTIN_VPPERM,
27421 IX86_BUILTIN_VPMACSSWW,
27422 IX86_BUILTIN_VPMACSWW,
27423 IX86_BUILTIN_VPMACSSWD,
27424 IX86_BUILTIN_VPMACSWD,
27425 IX86_BUILTIN_VPMACSSDD,
27426 IX86_BUILTIN_VPMACSDD,
27427 IX86_BUILTIN_VPMACSSDQL,
27428 IX86_BUILTIN_VPMACSSDQH,
27429 IX86_BUILTIN_VPMACSDQL,
27430 IX86_BUILTIN_VPMACSDQH,
27431 IX86_BUILTIN_VPMADCSSWD,
27432 IX86_BUILTIN_VPMADCSWD,
27434 IX86_BUILTIN_VPHADDBW,
27435 IX86_BUILTIN_VPHADDBD,
27436 IX86_BUILTIN_VPHADDBQ,
27437 IX86_BUILTIN_VPHADDWD,
27438 IX86_BUILTIN_VPHADDWQ,
27439 IX86_BUILTIN_VPHADDDQ,
27440 IX86_BUILTIN_VPHADDUBW,
27441 IX86_BUILTIN_VPHADDUBD,
27442 IX86_BUILTIN_VPHADDUBQ,
27443 IX86_BUILTIN_VPHADDUWD,
27444 IX86_BUILTIN_VPHADDUWQ,
27445 IX86_BUILTIN_VPHADDUDQ,
27446 IX86_BUILTIN_VPHSUBBW,
27447 IX86_BUILTIN_VPHSUBWD,
27448 IX86_BUILTIN_VPHSUBDQ,
27450 IX86_BUILTIN_VPROTB,
27451 IX86_BUILTIN_VPROTW,
27452 IX86_BUILTIN_VPROTD,
27453 IX86_BUILTIN_VPROTQ,
27454 IX86_BUILTIN_VPROTB_IMM,
27455 IX86_BUILTIN_VPROTW_IMM,
27456 IX86_BUILTIN_VPROTD_IMM,
27457 IX86_BUILTIN_VPROTQ_IMM,
27459 IX86_BUILTIN_VPSHLB,
27460 IX86_BUILTIN_VPSHLW,
27461 IX86_BUILTIN_VPSHLD,
27462 IX86_BUILTIN_VPSHLQ,
27463 IX86_BUILTIN_VPSHAB,
27464 IX86_BUILTIN_VPSHAW,
27465 IX86_BUILTIN_VPSHAD,
27466 IX86_BUILTIN_VPSHAQ,
27468 IX86_BUILTIN_VFRCZSS,
27469 IX86_BUILTIN_VFRCZSD,
27470 IX86_BUILTIN_VFRCZPS,
27471 IX86_BUILTIN_VFRCZPD,
27472 IX86_BUILTIN_VFRCZPS256,
27473 IX86_BUILTIN_VFRCZPD256,
27475 IX86_BUILTIN_VPCOMEQUB,
27476 IX86_BUILTIN_VPCOMNEUB,
27477 IX86_BUILTIN_VPCOMLTUB,
27478 IX86_BUILTIN_VPCOMLEUB,
27479 IX86_BUILTIN_VPCOMGTUB,
27480 IX86_BUILTIN_VPCOMGEUB,
27481 IX86_BUILTIN_VPCOMFALSEUB,
27482 IX86_BUILTIN_VPCOMTRUEUB,
27484 IX86_BUILTIN_VPCOMEQUW,
27485 IX86_BUILTIN_VPCOMNEUW,
27486 IX86_BUILTIN_VPCOMLTUW,
27487 IX86_BUILTIN_VPCOMLEUW,
27488 IX86_BUILTIN_VPCOMGTUW,
27489 IX86_BUILTIN_VPCOMGEUW,
27490 IX86_BUILTIN_VPCOMFALSEUW,
27491 IX86_BUILTIN_VPCOMTRUEUW,
27493 IX86_BUILTIN_VPCOMEQUD,
27494 IX86_BUILTIN_VPCOMNEUD,
27495 IX86_BUILTIN_VPCOMLTUD,
27496 IX86_BUILTIN_VPCOMLEUD,
27497 IX86_BUILTIN_VPCOMGTUD,
27498 IX86_BUILTIN_VPCOMGEUD,
27499 IX86_BUILTIN_VPCOMFALSEUD,
27500 IX86_BUILTIN_VPCOMTRUEUD,
27502 IX86_BUILTIN_VPCOMEQUQ,
27503 IX86_BUILTIN_VPCOMNEUQ,
27504 IX86_BUILTIN_VPCOMLTUQ,
27505 IX86_BUILTIN_VPCOMLEUQ,
27506 IX86_BUILTIN_VPCOMGTUQ,
27507 IX86_BUILTIN_VPCOMGEUQ,
27508 IX86_BUILTIN_VPCOMFALSEUQ,
27509 IX86_BUILTIN_VPCOMTRUEUQ,
27511 IX86_BUILTIN_VPCOMEQB,
27512 IX86_BUILTIN_VPCOMNEB,
27513 IX86_BUILTIN_VPCOMLTB,
27514 IX86_BUILTIN_VPCOMLEB,
27515 IX86_BUILTIN_VPCOMGTB,
27516 IX86_BUILTIN_VPCOMGEB,
27517 IX86_BUILTIN_VPCOMFALSEB,
27518 IX86_BUILTIN_VPCOMTRUEB,
27520 IX86_BUILTIN_VPCOMEQW,
27521 IX86_BUILTIN_VPCOMNEW,
27522 IX86_BUILTIN_VPCOMLTW,
27523 IX86_BUILTIN_VPCOMLEW,
27524 IX86_BUILTIN_VPCOMGTW,
27525 IX86_BUILTIN_VPCOMGEW,
27526 IX86_BUILTIN_VPCOMFALSEW,
27527 IX86_BUILTIN_VPCOMTRUEW,
27529 IX86_BUILTIN_VPCOMEQD,
27530 IX86_BUILTIN_VPCOMNED,
27531 IX86_BUILTIN_VPCOMLTD,
27532 IX86_BUILTIN_VPCOMLED,
27533 IX86_BUILTIN_VPCOMGTD,
27534 IX86_BUILTIN_VPCOMGED,
27535 IX86_BUILTIN_VPCOMFALSED,
27536 IX86_BUILTIN_VPCOMTRUED,
27538 IX86_BUILTIN_VPCOMEQQ,
27539 IX86_BUILTIN_VPCOMNEQ,
27540 IX86_BUILTIN_VPCOMLTQ,
27541 IX86_BUILTIN_VPCOMLEQ,
27542 IX86_BUILTIN_VPCOMGTQ,
27543 IX86_BUILTIN_VPCOMGEQ,
27544 IX86_BUILTIN_VPCOMFALSEQ,
27545 IX86_BUILTIN_VPCOMTRUEQ,
27547 /* LWP instructions. */
27548 IX86_BUILTIN_LLWPCB,
27549 IX86_BUILTIN_SLWPCB,
27550 IX86_BUILTIN_LWPVAL32,
27551 IX86_BUILTIN_LWPVAL64,
27552 IX86_BUILTIN_LWPINS32,
27553 IX86_BUILTIN_LWPINS64,
27555 IX86_BUILTIN_CLZS,
27557 /* RTM */
27558 IX86_BUILTIN_XBEGIN,
27559 IX86_BUILTIN_XEND,
27560 IX86_BUILTIN_XABORT,
27561 IX86_BUILTIN_XTEST,
27563 /* BMI instructions. */
27564 IX86_BUILTIN_BEXTR32,
27565 IX86_BUILTIN_BEXTR64,
27566 IX86_BUILTIN_CTZS,
27568 /* TBM instructions. */
27569 IX86_BUILTIN_BEXTRI32,
27570 IX86_BUILTIN_BEXTRI64,
27572 /* BMI2 instructions. */
27573 IX86_BUILTIN_BZHI32,
27574 IX86_BUILTIN_BZHI64,
27575 IX86_BUILTIN_PDEP32,
27576 IX86_BUILTIN_PDEP64,
27577 IX86_BUILTIN_PEXT32,
27578 IX86_BUILTIN_PEXT64,
27580 /* ADX instructions. */
27581 IX86_BUILTIN_ADDCARRYX32,
27582 IX86_BUILTIN_ADDCARRYX64,
27584 /* FSGSBASE instructions. */
27585 IX86_BUILTIN_RDFSBASE32,
27586 IX86_BUILTIN_RDFSBASE64,
27587 IX86_BUILTIN_RDGSBASE32,
27588 IX86_BUILTIN_RDGSBASE64,
27589 IX86_BUILTIN_WRFSBASE32,
27590 IX86_BUILTIN_WRFSBASE64,
27591 IX86_BUILTIN_WRGSBASE32,
27592 IX86_BUILTIN_WRGSBASE64,
27594 /* RDRND instructions. */
27595 IX86_BUILTIN_RDRAND16_STEP,
27596 IX86_BUILTIN_RDRAND32_STEP,
27597 IX86_BUILTIN_RDRAND64_STEP,
27599 /* RDSEED instructions. */
27600 IX86_BUILTIN_RDSEED16_STEP,
27601 IX86_BUILTIN_RDSEED32_STEP,
27602 IX86_BUILTIN_RDSEED64_STEP,
27604 /* F16C instructions. */
27605 IX86_BUILTIN_CVTPH2PS,
27606 IX86_BUILTIN_CVTPH2PS256,
27607 IX86_BUILTIN_CVTPS2PH,
27608 IX86_BUILTIN_CVTPS2PH256,
27610 /* CFString built-in for darwin */
27611 IX86_BUILTIN_CFSTRING,
27613 /* Builtins to get CPU type and supported features. */
27614 IX86_BUILTIN_CPU_INIT,
27615 IX86_BUILTIN_CPU_IS,
27616 IX86_BUILTIN_CPU_SUPPORTS,
27618 IX86_BUILTIN_MAX
27619 };
27621 /* Table for the ix86 builtin decls. */
27624 /* Table of all of the builtin functions that are possible with different ISA's
27625 but are waiting to be built until a function is declared to use that
27626 ISA. */
27633 };
27638 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
27639 of which isa_flags to use in the ix86_builtins_isa array. Stores the
27640 function decl in the ix86_builtins array. Returns the function decl or
27641 NULL_TREE, if the builtin was not added.
27643 If the front end has a special hook for builtin functions, delay adding
27644 builtin functions that aren't in the current ISA until the ISA is changed
27645 with function specific optimization. Doing so, can save about 300K for the
27646 default compiler. When the builtin is expanded, check at that time whether
27647 it is valid.
27649 If the front end doesn't have a special hook, record all builtins, even if
27650 it isn't an instruction set in the current ISA in case the user uses
27651 function specific options for a different ISA, so that we don't get scope
27652 errors if a builtin is added in the middle of a function scope. */
27658 {
27662 {
27671 {
27677 }
27678 else
27679 {
27685 }
27686 }
27689 }
27691 /* Like def_builtin, but also marks the function decl "const". */
27696 {
27700 else
27704 }
27706 /* Add any new builtin functions for a given ISA that may not have been
27707 declared. This saves a bit of space compared to adding all of the
27708 declarations to the tree, even if we didn't use them. */
27710 static void
27712 {
27716 {
27719 {
27722 /* Don't define the builtin again. */
27728 NULL_TREE);
27733 }
27734 }
27735 }
27737 /* Bits for builtin_description.flag. */
27739 /* Set when we don't support the comparison natively, and should
27740 swap_comparison in order to support it. */
27741 #define BUILTIN_DESC_SWAP_OPERANDS 1
27743 struct builtin_description
27744 {
27751 };
27754 {
27755 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
27756 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
27757 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
27758 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
27759 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
27760 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
27761 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
27762 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
27763 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
27764 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
27765 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
27766 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
27767 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
27768 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
27769 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
27770 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
27771 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
27772 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
27773 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
27774 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
27775 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
27776 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
27777 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
27778 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
27779 };
27782 {
27783 /* SSE4.2 */
27784 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
27785 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
27786 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
27787 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
27788 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
27789 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
27790 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
27791 };
27794 {
27795 /* SSE4.2 */
27796 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
27797 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
27798 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
27799 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
27800 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
27801 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
27802 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
27803 };
27805 /* Special builtins with variable number of arguments. */
27807 {
27808 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtsc", IX86_BUILTIN_RDTSC, UNKNOWN, (int) UINT64_FTYPE_VOID },
27809 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
27810 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
27812 /* MMX */
27813 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
27815 /* 3DNow! */
27816 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
27818 /* FXSR, XSAVE and XSAVEOPT */
27819 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxsave", IX86_BUILTIN_FXSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID },
27820 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxrstor", IX86_BUILTIN_FXRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID },
27821 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xsave", IX86_BUILTIN_XSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27822 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xrstor", IX86_BUILTIN_XRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27823 { OPTION_MASK_ISA_XSAVEOPT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt", IX86_BUILTIN_XSAVEOPT, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27825 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxsave64", IX86_BUILTIN_FXSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID },
27826 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxrstor64", IX86_BUILTIN_FXRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID },
27827 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsave64", IX86_BUILTIN_XSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27828 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xrstor64", IX86_BUILTIN_XRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27829 { OPTION_MASK_ISA_XSAVEOPT | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt64", IX86_BUILTIN_XSAVEOPT64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27831 /* SSE */
27832 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storeups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
27833 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
27834 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
27836 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
27837 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
27838 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
27839 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
27841 /* SSE or 3DNow!A */
27842 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
27843 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntq, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PULONGLONG_ULONGLONG },
27845 /* SSE2 */
27846 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
27847 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
27848 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storeupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
27849 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storedquv16qi, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
27850 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
27851 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
27852 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
27853 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
27854 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
27855 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loaddquv16qi, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
27857 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
27858 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
27860 /* SSE3 */
27861 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
27863 /* SSE4.1 */
27864 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
27866 /* SSE4A */
27867 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
27868 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
27870 /* AVX */
27871 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
27872 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, "__builtin_ia32_vzeroupper", IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
27874 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4sf, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
27875 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4df, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
27876 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv8sf, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
27877 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v4df, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
27878 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v8sf, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
27880 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
27881 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
27882 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
27883 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
27884 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loaddquv32qi, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
27885 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storedquv32qi, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
27886 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
27888 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
27889 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
27890 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
27892 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI },
27893 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI },
27894 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI },
27895 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI },
27896 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF },
27897 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF },
27898 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF },
27899 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF },
27901 /* AVX2 */
27902 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_movntdqa, "__builtin_ia32_movntdqa256", IX86_BUILTIN_MOVNTDQA256, UNKNOWN, (int) V4DI_FTYPE_PV4DI },
27903 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd, "__builtin_ia32_maskloadd", IX86_BUILTIN_MASKLOADD, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI },
27904 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq, "__builtin_ia32_maskloadq", IX86_BUILTIN_MASKLOADQ, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI },
27905 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd256, "__builtin_ia32_maskloadd256", IX86_BUILTIN_MASKLOADD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI },
27906 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq256, "__builtin_ia32_maskloadq256", IX86_BUILTIN_MASKLOADQ256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI },
27907 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored, "__builtin_ia32_maskstored", IX86_BUILTIN_MASKSTORED, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_V4SI },
27908 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq, "__builtin_ia32_maskstoreq", IX86_BUILTIN_MASKSTOREQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_V2DI },
27909 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored256, "__builtin_ia32_maskstored256", IX86_BUILTIN_MASKSTORED256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_V8SI },
27910 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq256, "__builtin_ia32_maskstoreq256", IX86_BUILTIN_MASKSTOREQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_V4DI },
27912 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID },
27913 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID },
27914 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvalsi3, "__builtin_ia32_lwpval32", IX86_BUILTIN_LWPVAL32, UNKNOWN, (int) VOID_FTYPE_UINT_UINT_UINT },
27915 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvaldi3, "__builtin_ia32_lwpval64", IX86_BUILTIN_LWPVAL64, UNKNOWN, (int) VOID_FTYPE_UINT64_UINT_UINT },
27916 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT },
27917 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT },
27919 /* FSGSBASE */
27920 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
27921 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
27922 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
27923 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
27924 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
27925 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
27926 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
27927 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
27929 /* RTM */
27930 { OPTION_MASK_ISA_RTM, CODE_FOR_xbegin, "__builtin_ia32_xbegin", IX86_BUILTIN_XBEGIN, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
27931 { OPTION_MASK_ISA_RTM, CODE_FOR_xend, "__builtin_ia32_xend", IX86_BUILTIN_XEND, UNKNOWN, (int) VOID_FTYPE_VOID },
27932 { OPTION_MASK_ISA_RTM, CODE_FOR_xtest, "__builtin_ia32_xtest", IX86_BUILTIN_XTEST, UNKNOWN, (int) INT_FTYPE_VOID },
27933 };
27935 /* Builtins with variable number of arguments. */
27937 {
27938 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_bsr, "__builtin_ia32_bsrsi", IX86_BUILTIN_BSRSI, UNKNOWN, (int) INT_FTYPE_INT },
27939 { OPTION_MASK_ISA_64BIT, CODE_FOR_bsr_rex64, "__builtin_ia32_bsrdi", IX86_BUILTIN_BSRDI, UNKNOWN, (int) INT64_FTYPE_INT64 },
27940 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdpmc", IX86_BUILTIN_RDPMC, UNKNOWN, (int) UINT64_FTYPE_INT },
27941 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlqi3, "__builtin_ia32_rolqi", IX86_BUILTIN_ROLQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
27942 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlhi3, "__builtin_ia32_rolhi", IX86_BUILTIN_ROLHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
27943 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrqi3, "__builtin_ia32_rorqi", IX86_BUILTIN_RORQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
27944 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrhi3, "__builtin_ia32_rorhi", IX86_BUILTIN_RORHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
27946 /* MMX */
27947 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27948 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27949 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27950 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27951 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27952 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27954 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27955 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27956 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27957 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27958 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27959 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27960 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27961 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27963 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27964 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27966 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27967 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27968 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27969 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27971 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27972 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27973 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27974 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27975 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27976 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27978 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27979 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27980 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27981 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27982 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
27983 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
27985 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
27986 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
27987 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
27989 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
27991 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
27992 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
27993 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
27994 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
27995 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
27996 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
27998 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
27999 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
28000 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
28001 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
28002 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
28003 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
28005 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
28006 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
28007 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
28008 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
28010 /* 3DNow! */
28011 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
28012 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
28013 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28014 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28016 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28017 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28018 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28019 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28020 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28021 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28022 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28023 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28024 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28025 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28026 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28027 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28028 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28029 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28030 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28032 /* 3DNow!A */
28033 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
28034 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
28035 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
28036 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28037 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28038 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28040 /* SSE */
28041 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
28042 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28043 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28044 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28045 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28046 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28047 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
28048 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
28049 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
28050 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
28051 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
28052 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
28054 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28056 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28057 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28058 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28059 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28060 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28061 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28062 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28063 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28065 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
28066 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
28067 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
28068 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28069 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28070 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28071 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
28072 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
28073 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
28074 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28075 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
28076 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28077 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
28078 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
28079 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
28080 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28081 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
28082 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
28083 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
28084 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28086 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28087 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28088 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28089 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28091 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28092 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28093 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28094 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28096 { OPTION_MASK_ISA_SSE, CODE_FOR_copysignv4sf3, "__builtin_ia32_copysignps", IX86_BUILTIN_CPYSGNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28098 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28099 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28100 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28101 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_highv4sf, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28102 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_lowv4sf, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28104 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
28105 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
28106 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
28108 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
28110 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28111 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28112 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28114 { OPTION_MASK_ISA_SSE, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
28115 { OPTION_MASK_ISA_SSE, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
28117 /* SSE MMX or 3Dnow!A */
28118 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28119 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28120 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28122 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28123 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28124 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28125 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28127 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
28128 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
28130 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
28132 /* SSE2 */
28133 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28135 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
28136 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
28137 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
28138 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
28139 { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
28141 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
28142 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
28143 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
28144 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
28145 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
28147 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
28149 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
28150 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
28151 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
28152 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
28154 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_fix_notruncv4sfv4si, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28155 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
28156 { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28158 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28159 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28160 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28161 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28162 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28163 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28164 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28165 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28167 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
28168 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
28169 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
28170 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28171 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
28172 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28173 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
28174 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
28175 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
28176 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28177 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28178 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28179 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
28180 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
28181 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
28182 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28183 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
28184 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
28185 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
28186 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28188 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28189 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28190 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28191 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28193 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28194 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28195 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28196 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28198 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysignv2df3, "__builtin_ia32_copysignpd", IX86_BUILTIN_CPYSGNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28200 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28201 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2df, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28202 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2df, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28204 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
28206 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28207 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28208 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28209 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28210 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28211 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28212 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28213 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28215 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28216 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28217 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28218 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28219 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28220 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28221 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28222 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28224 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28225 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
28227 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28228 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28229 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28230 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28232 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28233 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28235 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28236 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28237 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28238 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28239 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28240 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28242 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28243 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28244 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28245 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28247 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv16qi, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28248 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv8hi, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28249 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv4si, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28250 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2di, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28251 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv16qi, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28252 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv8hi, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28253 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv4si, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28254 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2di, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28256 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
28257 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
28258 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
28260 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28261 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
28263 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
28264 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_widen_umult_even_v4si, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
28266 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
28268 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
28269 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
28270 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
28271 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
28273 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlv1ti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
28274 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28275 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28276 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
28277 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28278 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28279 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
28281 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrv1ti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
28282 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28283 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28284 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
28285 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28286 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28287 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
28289 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28290 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28291 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28292 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28294 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
28295 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
28296 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
28298 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
28300 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28302 /* SSE2 MMX */
28303 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
28304 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
28306 /* SSE3 */
28307 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
28308 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28310 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28311 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28312 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28313 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28314 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28315 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28317 /* SSSE3 */
28318 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
28319 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
28320 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28321 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
28322 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
28323 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
28325 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28326 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28327 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28328 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28329 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28330 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28331 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28332 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28333 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28334 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28335 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28336 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28337 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
28338 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
28339 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28340 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28341 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28342 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28343 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28344 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28345 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28346 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28347 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28348 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28350 /* SSSE3. */
28351 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_CONVERT },
28352 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_INT_CONVERT },
28354 /* SSE4.1 */
28355 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28356 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28357 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
28358 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
28359 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28360 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28361 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28362 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
28363 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
28364 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
28366 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
28367 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
28368 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
28369 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
28370 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
28371 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
28372 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
28373 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
28374 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
28375 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
28376 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
28377 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
28378 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28380 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
28381 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28382 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28383 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28384 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28385 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28386 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28387 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28388 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28389 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28390 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
28391 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28393 /* SSE4.1 */
28394 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
28395 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
28396 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28397 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28399 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_floorpd", IX86_BUILTIN_FLOORPD, (enum rtx_code) ROUND_FLOOR, (int) V2DF_FTYPE_V2DF_ROUND },
28400 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_ceilpd", IX86_BUILTIN_CEILPD, (enum rtx_code) ROUND_CEIL, (int) V2DF_FTYPE_V2DF_ROUND },
28401 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
28402 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
28404 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
28405 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
28407 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
28408 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
28410 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
28411 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
28412 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
28413 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
28415 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
28416 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
28418 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28419 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28421 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28422 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28423 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28425 /* SSE4.2 */
28426 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28427 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
28428 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
28429 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28430 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28432 /* SSE4A */
28433 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
28434 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
28435 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
28436 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28438 /* AES */
28439 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
28440 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28442 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28443 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28444 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28445 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28447 /* PCLMUL */
28448 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
28450 /* AVX */
28451 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28452 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28453 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28454 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28455 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28456 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28457 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28458 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28459 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28460 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28461 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28462 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28463 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28464 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28465 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28466 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28467 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28468 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28469 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28470 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28471 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28472 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28473 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28474 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28475 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28476 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28478 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
28479 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
28480 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
28481 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
28483 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28484 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28485 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
28486 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
28487 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28488 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28489 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28490 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28491 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28492 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28493 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28494 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28495 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28496 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
28497 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
28498 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
28499 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
28500 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
28501 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
28502 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_fix_notruncv8sfv8si, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28503 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
28504 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
28505 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
28506 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28507 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28508 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28509 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
28510 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
28511 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
28512 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28513 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
28514 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
28515 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
28516 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
28518 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28519 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28520 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28522 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28523 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28524 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28525 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28526 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28528 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28530 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28531 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
28533 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_floorpd256", IX86_BUILTIN_FLOORPD256, (enum rtx_code) ROUND_FLOOR, (int) V4DF_FTYPE_V4DF_ROUND },
28534 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_ceilpd256", IX86_BUILTIN_CEILPD256, (enum rtx_code) ROUND_CEIL, (int) V4DF_FTYPE_V4DF_ROUND },
28535 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_truncpd256", IX86_BUILTIN_TRUNCPD256, (enum rtx_code) ROUND_TRUNC, (int) V4DF_FTYPE_V4DF_ROUND },
28536 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
28538 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28539 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
28541 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
28542 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
28544 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
28545 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
28546 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
28547 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
28549 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
28550 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
28552 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28553 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28555 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28556 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28557 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28558 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28560 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
28561 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
28562 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
28563 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
28564 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
28565 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
28567 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28568 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28569 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28570 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28571 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28572 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28573 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28574 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28575 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28576 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28577 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28578 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28579 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28580 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28581 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28583 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
28584 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
28586 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28587 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28589 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
28591 /* AVX2 */
28592 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
28593 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
28594 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv16hi2, "__builtin_ia32_pabsw256", IX86_BUILTIN_PABSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI },
28595 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv8si2, "__builtin_ia32_pabsd256", IX86_BUILTIN_PABSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI },
28596 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packssdw, "__builtin_ia32_packssdw256", IX86_BUILTIN_PACKSSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
28597 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packsswb, "__builtin_ia32_packsswb256", IX86_BUILTIN_PACKSSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
28598 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packusdw, "__builtin_ia32_packusdw256", IX86_BUILTIN_PACKUSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
28599 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packuswb, "__builtin_ia32_packuswb256", IX86_BUILTIN_PACKUSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
28600 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv32qi3, "__builtin_ia32_paddb256", IX86_BUILTIN_PADDB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28601 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv16hi3, "__builtin_ia32_paddw256", IX86_BUILTIN_PADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28602 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv8si3, "__builtin_ia32_paddd256", IX86_BUILTIN_PADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28603 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv4di3, "__builtin_ia32_paddq256", IX86_BUILTIN_PADDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28604 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv32qi3, "__builtin_ia32_paddsb256", IX86_BUILTIN_PADDSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28605 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28606 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28607 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28608 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
28609 { OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28610 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28611 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28612 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv16hi3, "__builtin_ia32_pavgw256", IX86_BUILTIN_PAVGW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28613 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendvb, "__builtin_ia32_pblendvb256", IX86_BUILTIN_PBLENDVB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI },
28614 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendw, "__builtin_ia32_pblendw256", IX86_BUILTIN_PBLENDVW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_INT },
28615 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv32qi3, "__builtin_ia32_pcmpeqb256", IX86_BUILTIN_PCMPEQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28616 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv16hi3, "__builtin_ia32_pcmpeqw256", IX86_BUILTIN_PCMPEQW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28617 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv8si3, "__builtin_ia32_pcmpeqd256", IX86_BUILTIN_PCMPEQD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28618 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv4di3, "__builtin_ia32_pcmpeqq256", IX86_BUILTIN_PCMPEQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28619 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv32qi3, "__builtin_ia32_pcmpgtb256", IX86_BUILTIN_PCMPGTB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28620 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv16hi3, "__builtin_ia32_pcmpgtw256", IX86_BUILTIN_PCMPGTW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28621 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv8si3, "__builtin_ia32_pcmpgtd256", IX86_BUILTIN_PCMPGTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28622 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv4di3, "__builtin_ia32_pcmpgtq256", IX86_BUILTIN_PCMPGTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28623 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddwv16hi3, "__builtin_ia32_phaddw256", IX86_BUILTIN_PHADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28624 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phadddv8si3, "__builtin_ia32_phaddd256", IX86_BUILTIN_PHADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28625 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddswv16hi3, "__builtin_ia32_phaddsw256", IX86_BUILTIN_PHADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28626 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubwv16hi3, "__builtin_ia32_phsubw256", IX86_BUILTIN_PHSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28627 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubdv8si3, "__builtin_ia32_phsubd256", IX86_BUILTIN_PHSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28628 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubswv16hi3, "__builtin_ia32_phsubsw256", IX86_BUILTIN_PHSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28629 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddubsw256, "__builtin_ia32_pmaddubsw256", IX86_BUILTIN_PMADDUBSW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
28630 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddwd, "__builtin_ia32_pmaddwd256", IX86_BUILTIN_PMADDWD256, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI },
28631 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv32qi3, "__builtin_ia32_pmaxsb256", IX86_BUILTIN_PMAXSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28632 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv16hi3, "__builtin_ia32_pmaxsw256", IX86_BUILTIN_PMAXSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28633 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv8si3 , "__builtin_ia32_pmaxsd256", IX86_BUILTIN_PMAXSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28634 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv32qi3, "__builtin_ia32_pmaxub256", IX86_BUILTIN_PMAXUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28635 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv16hi3, "__builtin_ia32_pmaxuw256", IX86_BUILTIN_PMAXUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28636 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv8si3 , "__builtin_ia32_pmaxud256", IX86_BUILTIN_PMAXUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28637 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv32qi3, "__builtin_ia32_pminsb256", IX86_BUILTIN_PMINSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28638 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv16hi3, "__builtin_ia32_pminsw256", IX86_BUILTIN_PMINSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28639 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv8si3 , "__builtin_ia32_pminsd256", IX86_BUILTIN_PMINSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28640 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv32qi3, "__builtin_ia32_pminub256", IX86_BUILTIN_PMINUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28641 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv16hi3, "__builtin_ia32_pminuw256", IX86_BUILTIN_PMINUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28642 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv8si3 , "__builtin_ia32_pminud256", IX86_BUILTIN_PMINUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28643 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmovmskb, "__builtin_ia32_pmovmskb256", IX86_BUILTIN_PMOVMSKB256, UNKNOWN, (int) INT_FTYPE_V32QI },
28644 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv16qiv16hi2, "__builtin_ia32_pmovsxbw256", IX86_BUILTIN_PMOVSXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
28645 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8qiv8si2 , "__builtin_ia32_pmovsxbd256", IX86_BUILTIN_PMOVSXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
28646 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4qiv4di2 , "__builtin_ia32_pmovsxbq256", IX86_BUILTIN_PMOVSXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
28647 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8hiv8si2 , "__builtin_ia32_pmovsxwd256", IX86_BUILTIN_PMOVSXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
28648 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4hiv4di2 , "__builtin_ia32_pmovsxwq256", IX86_BUILTIN_PMOVSXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
28649 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4siv4di2 , "__builtin_ia32_pmovsxdq256", IX86_BUILTIN_PMOVSXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
28650 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv16qiv16hi2, "__builtin_ia32_pmovzxbw256", IX86_BUILTIN_PMOVZXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
28651 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8qiv8si2 , "__builtin_ia32_pmovzxbd256", IX86_BUILTIN_PMOVZXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
28652 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4qiv4di2 , "__builtin_ia32_pmovzxbq256", IX86_BUILTIN_PMOVZXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
28653 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
28654 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
28655 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
28656 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_smult_even_v8si, "__builtin_ia32_pmuldq256", IX86_BUILTIN_PMULDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
28657 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28658 { OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28659 { OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28660 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28661 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28662 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_umult_even_v8si, "__builtin_ia32_pmuludq256", IX86_BUILTIN_PMULUDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
28663 { OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28664 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
28665 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28666 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufdv3, "__builtin_ia32_pshufd256", IX86_BUILTIN_PSHUFD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT },
28667 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufhwv3, "__builtin_ia32_pshufhw256", IX86_BUILTIN_PSHUFHW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
28668 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshuflwv3, "__builtin_ia32_pshuflw256", IX86_BUILTIN_PSHUFLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
28669 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28670 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28671 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28672 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
28673 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28674 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28675 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28676 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28677 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
28678 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
28679 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28680 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28681 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28682 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28683 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
28684 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28685 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28686 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28687 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrld256", IX86_BUILTIN_PSRLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28688 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlqi256", IX86_BUILTIN_PSRLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
28689 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlq256", IX86_BUILTIN_PSRLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
28690 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv32qi3, "__builtin_ia32_psubb256", IX86_BUILTIN_PSUBB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28691 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv16hi3, "__builtin_ia32_psubw256", IX86_BUILTIN_PSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28692 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv8si3, "__builtin_ia32_psubd256", IX86_BUILTIN_PSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28693 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv4di3, "__builtin_ia32_psubq256", IX86_BUILTIN_PSUBQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28694 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv32qi3, "__builtin_ia32_psubsb256", IX86_BUILTIN_PSUBSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28695 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv16hi3, "__builtin_ia32_psubsw256", IX86_BUILTIN_PSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28696 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv32qi3, "__builtin_ia32_psubusb256", IX86_BUILTIN_PSUBUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28697 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv16hi3, "__builtin_ia32_psubusw256", IX86_BUILTIN_PSUBUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28698 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv32qi, "__builtin_ia32_punpckhbw256", IX86_BUILTIN_PUNPCKHBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28699 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv16hi, "__builtin_ia32_punpckhwd256", IX86_BUILTIN_PUNPCKHWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28700 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv8si, "__builtin_ia32_punpckhdq256", IX86_BUILTIN_PUNPCKHDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28701 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv4di, "__builtin_ia32_punpckhqdq256", IX86_BUILTIN_PUNPCKHQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28702 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv32qi, "__builtin_ia32_punpcklbw256", IX86_BUILTIN_PUNPCKLBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28703 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv16hi, "__builtin_ia32_punpcklwd256", IX86_BUILTIN_PUNPCKLWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28704 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv8si, "__builtin_ia32_punpckldq256", IX86_BUILTIN_PUNPCKLDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28705 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv4di, "__builtin_ia32_punpcklqdq256", IX86_BUILTIN_PUNPCKLQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28706 { OPTION_MASK_ISA_AVX2, CODE_FOR_xorv4di3, "__builtin_ia32_pxor256", IX86_BUILTIN_PXOR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28707 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4sf, "__builtin_ia32_vbroadcastss_ps", IX86_BUILTIN_VBROADCASTSS_PS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28708 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv8sf, "__builtin_ia32_vbroadcastss_ps256", IX86_BUILTIN_VBROADCASTSS_PS256, UNKNOWN, (int) V8SF_FTYPE_V4SF },
28709 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4df, "__builtin_ia32_vbroadcastsd_pd256", IX86_BUILTIN_VBROADCASTSD_PD256, UNKNOWN, (int) V4DF_FTYPE_V2DF },
28710 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vbroadcasti128_v4di, "__builtin_ia32_vbroadcastsi256", IX86_BUILTIN_VBROADCASTSI256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
28711 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv4si, "__builtin_ia32_pblendd128", IX86_BUILTIN_PBLENDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
28712 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv8si, "__builtin_ia32_pblendd256", IX86_BUILTIN_PBLENDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
28713 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv32qi, "__builtin_ia32_pbroadcastb256", IX86_BUILTIN_PBROADCASTB256, UNKNOWN, (int) V32QI_FTYPE_V16QI },
28714 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16hi, "__builtin_ia32_pbroadcastw256", IX86_BUILTIN_PBROADCASTW256, UNKNOWN, (int) V16HI_FTYPE_V8HI },
28715 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8si, "__builtin_ia32_pbroadcastd256", IX86_BUILTIN_PBROADCASTD256, UNKNOWN, (int) V8SI_FTYPE_V4SI },
28716 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4di, "__builtin_ia32_pbroadcastq256", IX86_BUILTIN_PBROADCASTQ256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
28717 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16qi, "__builtin_ia32_pbroadcastb128", IX86_BUILTIN_PBROADCASTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
28718 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8hi, "__builtin_ia32_pbroadcastw128", IX86_BUILTIN_PBROADCASTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28719 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
28720 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28721 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28722 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
28723 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28724 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
28725 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
28726 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_extracti128, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
28727 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_inserti128, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
28728 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28729 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28730 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28731 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28732 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28733 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28734 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28735 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv2di, "__builtin_ia32_psrlv2di", IX86_BUILTIN_PSRLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28736 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv8si, "__builtin_ia32_psrlv8si", IX86_BUILTIN_PSRLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28737 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4si, "__builtin_ia32_psrlv4si", IX86_BUILTIN_PSRLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28739 { OPTION_MASK_ISA_LZCNT, CODE_FOR_clzhi2_lzcnt, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
28741 /* BMI */
28742 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28743 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28744 { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
28746 /* TBM */
28747 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28748 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28750 /* F16C */
28751 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI },
28752 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI },
28753 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT },
28754 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT },
28756 /* BMI2 */
28757 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_si3, "__builtin_ia32_bzhi_si", IX86_BUILTIN_BZHI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28758 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_di3, "__builtin_ia32_bzhi_di", IX86_BUILTIN_BZHI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28759 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_si3, "__builtin_ia32_pdep_si", IX86_BUILTIN_PDEP32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28760 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_di3, "__builtin_ia32_pdep_di", IX86_BUILTIN_PDEP64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28761 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_si3, "__builtin_ia32_pext_si", IX86_BUILTIN_PEXT32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28762 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_di3, "__builtin_ia32_pext_di", IX86_BUILTIN_PEXT64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28763 };
28765 /* FMA4 and XOP. */
28766 #define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT
28767 #define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT
28768 #define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT
28769 #define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT
28770 #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF
28771 #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF
28772 #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF
28773 #define MULTI_ARG_3_DF2 V4DF_FTYPE_V4DF_V4DF_V4DF
28774 #define MULTI_ARG_3_DI V2DI_FTYPE_V2DI_V2DI_V2DI
28775 #define MULTI_ARG_3_SI V4SI_FTYPE_V4SI_V4SI_V4SI
28776 #define MULTI_ARG_3_SI_DI V4SI_FTYPE_V4SI_V4SI_V2DI
28777 #define MULTI_ARG_3_HI V8HI_FTYPE_V8HI_V8HI_V8HI
28778 #define MULTI_ARG_3_HI_SI V8HI_FTYPE_V8HI_V8HI_V4SI
28779 #define MULTI_ARG_3_QI V16QI_FTYPE_V16QI_V16QI_V16QI
28780 #define MULTI_ARG_3_DI2 V4DI_FTYPE_V4DI_V4DI_V4DI
28781 #define MULTI_ARG_3_SI2 V8SI_FTYPE_V8SI_V8SI_V8SI
28782 #define MULTI_ARG_3_HI2 V16HI_FTYPE_V16HI_V16HI_V16HI
28783 #define MULTI_ARG_3_QI2 V32QI_FTYPE_V32QI_V32QI_V32QI
28784 #define MULTI_ARG_2_SF V4SF_FTYPE_V4SF_V4SF
28785 #define MULTI_ARG_2_DF V2DF_FTYPE_V2DF_V2DF
28786 #define MULTI_ARG_2_DI V2DI_FTYPE_V2DI_V2DI
28787 #define MULTI_ARG_2_SI V4SI_FTYPE_V4SI_V4SI
28788 #define MULTI_ARG_2_HI V8HI_FTYPE_V8HI_V8HI
28789 #define MULTI_ARG_2_QI V16QI_FTYPE_V16QI_V16QI
28790 #define MULTI_ARG_2_DI_IMM V2DI_FTYPE_V2DI_SI
28791 #define MULTI_ARG_2_SI_IMM V4SI_FTYPE_V4SI_SI
28792 #define MULTI_ARG_2_HI_IMM V8HI_FTYPE_V8HI_SI
28793 #define MULTI_ARG_2_QI_IMM V16QI_FTYPE_V16QI_SI
28794 #define MULTI_ARG_2_DI_CMP V2DI_FTYPE_V2DI_V2DI_CMP
28795 #define MULTI_ARG_2_SI_CMP V4SI_FTYPE_V4SI_V4SI_CMP
28796 #define MULTI_ARG_2_HI_CMP V8HI_FTYPE_V8HI_V8HI_CMP
28797 #define MULTI_ARG_2_QI_CMP V16QI_FTYPE_V16QI_V16QI_CMP
28798 #define MULTI_ARG_2_SF_TF V4SF_FTYPE_V4SF_V4SF_TF
28799 #define MULTI_ARG_2_DF_TF V2DF_FTYPE_V2DF_V2DF_TF
28800 #define MULTI_ARG_2_DI_TF V2DI_FTYPE_V2DI_V2DI_TF
28801 #define MULTI_ARG_2_SI_TF V4SI_FTYPE_V4SI_V4SI_TF
28802 #define MULTI_ARG_2_HI_TF V8HI_FTYPE_V8HI_V8HI_TF
28803 #define MULTI_ARG_2_QI_TF V16QI_FTYPE_V16QI_V16QI_TF
28804 #define MULTI_ARG_1_SF V4SF_FTYPE_V4SF
28805 #define MULTI_ARG_1_DF V2DF_FTYPE_V2DF
28806 #define MULTI_ARG_1_SF2 V8SF_FTYPE_V8SF
28807 #define MULTI_ARG_1_DF2 V4DF_FTYPE_V4DF
28808 #define MULTI_ARG_1_DI V2DI_FTYPE_V2DI
28809 #define MULTI_ARG_1_SI V4SI_FTYPE_V4SI
28810 #define MULTI_ARG_1_HI V8HI_FTYPE_V8HI
28811 #define MULTI_ARG_1_QI V16QI_FTYPE_V16QI
28812 #define MULTI_ARG_1_SI_DI V2DI_FTYPE_V4SI
28813 #define MULTI_ARG_1_HI_DI V2DI_FTYPE_V8HI
28814 #define MULTI_ARG_1_HI_SI V4SI_FTYPE_V8HI
28815 #define MULTI_ARG_1_QI_DI V2DI_FTYPE_V16QI
28816 #define MULTI_ARG_1_QI_SI V4SI_FTYPE_V16QI
28817 #define MULTI_ARG_1_QI_HI V8HI_FTYPE_V16QI
28820 {
28861 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
28862 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
28863 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4si, "__builtin_ia32_vpcmov_v4si", IX86_BUILTIN_VPCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
28864 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8hi, "__builtin_ia32_vpcmov_v8hi", IX86_BUILTIN_VPCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
28865 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16qi, "__builtin_ia32_vpcmov_v16qi",IX86_BUILTIN_VPCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
28866 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2df, "__builtin_ia32_vpcmov_v2df", IX86_BUILTIN_VPCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
28867 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4sf, "__builtin_ia32_vpcmov_v4sf", IX86_BUILTIN_VPCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
28869 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov256", IX86_BUILTIN_VPCMOV256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
28870 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov_v4di256", IX86_BUILTIN_VPCMOV_V4DI256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
28871 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8si256, "__builtin_ia32_vpcmov_v8si256", IX86_BUILTIN_VPCMOV_V8SI256, UNKNOWN, (int)MULTI_ARG_3_SI2 },
28872 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16hi256, "__builtin_ia32_vpcmov_v16hi256", IX86_BUILTIN_VPCMOV_V16HI256, UNKNOWN, (int)MULTI_ARG_3_HI2 },
28873 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v32qi256, "__builtin_ia32_vpcmov_v32qi256", IX86_BUILTIN_VPCMOV_V32QI256, UNKNOWN, (int)MULTI_ARG_3_QI2 },
28874 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4df256, "__builtin_ia32_vpcmov_v4df256", IX86_BUILTIN_VPCMOV_V4DF256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
28875 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8sf256, "__builtin_ia32_vpcmov_v8sf256", IX86_BUILTIN_VPCMOV_V8SF256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
28877 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pperm, "__builtin_ia32_vpperm", IX86_BUILTIN_VPPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
28879 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssww, "__builtin_ia32_vpmacssww", IX86_BUILTIN_VPMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
28880 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsww, "__builtin_ia32_vpmacsww", IX86_BUILTIN_VPMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
28881 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsswd, "__builtin_ia32_vpmacsswd", IX86_BUILTIN_VPMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
28882 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacswd, "__builtin_ia32_vpmacswd", IX86_BUILTIN_VPMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
28883 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdd, "__builtin_ia32_vpmacssdd", IX86_BUILTIN_VPMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
28884 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdd, "__builtin_ia32_vpmacsdd", IX86_BUILTIN_VPMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
28885 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdql, "__builtin_ia32_vpmacssdql", IX86_BUILTIN_VPMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
28886 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdqh, "__builtin_ia32_vpmacssdqh", IX86_BUILTIN_VPMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
28887 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdql, "__builtin_ia32_vpmacsdql", IX86_BUILTIN_VPMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
28888 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdqh, "__builtin_ia32_vpmacsdqh", IX86_BUILTIN_VPMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
28889 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcsswd, "__builtin_ia32_vpmadcsswd", IX86_BUILTIN_VPMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
28890 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcswd, "__builtin_ia32_vpmadcswd", IX86_BUILTIN_VPMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
28892 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv2di3, "__builtin_ia32_vprotq", IX86_BUILTIN_VPROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
28893 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv4si3, "__builtin_ia32_vprotd", IX86_BUILTIN_VPROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
28894 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv8hi3, "__builtin_ia32_vprotw", IX86_BUILTIN_VPROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
28895 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv16qi3, "__builtin_ia32_vprotb", IX86_BUILTIN_VPROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
28896 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv2di3, "__builtin_ia32_vprotqi", IX86_BUILTIN_VPROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
28897 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
28898 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
28899 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
28900 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
28901 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
28902 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
28903 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
28904 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
28905 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
28906 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
28907 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
28909 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_2_SF },
28910 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF },
28911 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
28912 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
28913 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
28914 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
28916 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
28917 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
28918 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbq, "__builtin_ia32_vphaddbq", IX86_BUILTIN_VPHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
28919 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwd, "__builtin_ia32_vphaddwd", IX86_BUILTIN_VPHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
28920 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwq, "__builtin_ia32_vphaddwq", IX86_BUILTIN_VPHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
28921 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadddq, "__builtin_ia32_vphadddq", IX86_BUILTIN_VPHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
28922 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubw, "__builtin_ia32_vphaddubw", IX86_BUILTIN_VPHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
28923 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubd, "__builtin_ia32_vphaddubd", IX86_BUILTIN_VPHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
28924 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubq, "__builtin_ia32_vphaddubq", IX86_BUILTIN_VPHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
28925 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwd, "__builtin_ia32_vphadduwd", IX86_BUILTIN_VPHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
28926 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwq, "__builtin_ia32_vphadduwq", IX86_BUILTIN_VPHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
28927 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddudq, "__builtin_ia32_vphaddudq", IX86_BUILTIN_VPHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
28928 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubbw, "__builtin_ia32_vphsubbw", IX86_BUILTIN_VPHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
28929 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubwd, "__builtin_ia32_vphsubwd", IX86_BUILTIN_VPHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
28930 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubdq, "__builtin_ia32_vphsubdq", IX86_BUILTIN_VPHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
28932 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomeqb", IX86_BUILTIN_VPCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
28933 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
28934 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneqb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
28935 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomltb", IX86_BUILTIN_VPCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
28936 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomleb", IX86_BUILTIN_VPCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
28937 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgtb", IX86_BUILTIN_VPCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
28938 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgeb", IX86_BUILTIN_VPCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
28940 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomeqw", IX86_BUILTIN_VPCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
28941 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomnew", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
28942 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomneqw", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
28943 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomltw", IX86_BUILTIN_VPCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
28944 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomlew", IX86_BUILTIN_VPCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
28945 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgtw", IX86_BUILTIN_VPCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
28946 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgew", IX86_BUILTIN_VPCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
28948 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomeqd", IX86_BUILTIN_VPCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
28949 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomned", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
28950 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomneqd", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
28951 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomltd", IX86_BUILTIN_VPCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
28952 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomled", IX86_BUILTIN_VPCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
28953 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomgtd", IX86_BUILTIN_VPCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
28954 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomged", IX86_BUILTIN_VPCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
28956 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomeqq", IX86_BUILTIN_VPCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
28957 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
28958 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneqq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
28959 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomltq", IX86_BUILTIN_VPCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
28960 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomleq", IX86_BUILTIN_VPCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
28961 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgtq", IX86_BUILTIN_VPCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
28962 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgeq", IX86_BUILTIN_VPCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
28964 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomequb", IX86_BUILTIN_VPCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
28965 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomneub", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
28966 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomnequb", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
28967 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomltub", IX86_BUILTIN_VPCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
28968 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomleub", IX86_BUILTIN_VPCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
28969 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgtub", IX86_BUILTIN_VPCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
28970 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgeub", IX86_BUILTIN_VPCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
28972 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomequw", IX86_BUILTIN_VPCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
28973 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomneuw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
28974 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomnequw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
28975 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomltuw", IX86_BUILTIN_VPCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
28976 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomleuw", IX86_BUILTIN_VPCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
28977 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgtuw", IX86_BUILTIN_VPCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
28978 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgeuw", IX86_BUILTIN_VPCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
28980 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomequd", IX86_BUILTIN_VPCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
28981 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomneud", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
28982 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomnequd", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
28983 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomltud", IX86_BUILTIN_VPCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
28984 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomleud", IX86_BUILTIN_VPCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
28985 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgtud", IX86_BUILTIN_VPCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
28986 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgeud", IX86_BUILTIN_VPCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
28988 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomequq", IX86_BUILTIN_VPCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
28989 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomneuq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
28990 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomnequq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
28991 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomltuq", IX86_BUILTIN_VPCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
28992 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomleuq", IX86_BUILTIN_VPCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
28993 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgtuq", IX86_BUILTIN_VPCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
28994 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgeuq", IX86_BUILTIN_VPCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
28996 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseb", IX86_BUILTIN_VPCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
28997 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalsew", IX86_BUILTIN_VPCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
28998 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalsed", IX86_BUILTIN_VPCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
28999 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseq", IX86_BUILTIN_VPCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
29000 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseub",IX86_BUILTIN_VPCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
29001 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalseuw",IX86_BUILTIN_VPCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
29002 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalseud",IX86_BUILTIN_VPCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
29003 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseuq",IX86_BUILTIN_VPCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
29005 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueb", IX86_BUILTIN_VPCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
29006 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtruew", IX86_BUILTIN_VPCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
29007 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrued", IX86_BUILTIN_VPCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
29008 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueq", IX86_BUILTIN_VPCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
29009 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueub", IX86_BUILTIN_VPCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
29010 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtrueuw", IX86_BUILTIN_VPCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
29011 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
29012 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
29014 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I },
29015 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I },
29016 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 },
29017 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
29019 };
29020 \f
29021 /* TM vector builtins. */
29023 /* Reuse the existing x86-specific `struct builtin_description' cause
29024 we're lazy. Add casts to make them fit. */
29026 {
29027 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29028 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29029 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29030 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29031 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29032 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29033 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29035 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29036 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29037 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29038 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29039 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29040 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29041 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29043 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29044 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29045 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29046 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29047 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29048 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29049 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29051 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
29052 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
29053 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
29054 };
29056 /* TM callbacks. */
29058 /* Return the builtin decl needed to load a vector of TYPE. */
29060 static tree
29062 {
29064 {
29066 {
29073 }
29074 }
29076 }
29078 /* Return the builtin decl needed to store a vector of TYPE. */
29080 static tree
29082 {
29084 {
29086 {
29093 }
29094 }
29096 }
29097 \f
29098 /* Initialize the transactional memory vector load/store builtins. */
29100 static void
29102 {
29106 tree decl;
29113 /* If there are no builtins defined, we must be compiling in a
29114 language without trans-mem support. */
29118 /* Use whatever attributes a normal TM load has. */
29122 /* Use whatever attributes a normal TM store has. */
29126 /* Use whatever attributes a normal TM log has. */
29134 {
29138 {
29146 {
29149 }
29151 {
29154 }
29155 else
29156 {
29159 }
29161 /* The builtin without the prefix for
29162 calling it directly. */
29164 attrs);
29165 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
29166 set the TYPE_ATTRIBUTES. */
29170 }
29171 }
29172 }
29174 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
29175 in the current target ISA to allow the user to compile particular modules
29176 with different target specific options that differ from the command line
29177 options. */
29178 static void
29180 {
29185 /* Add all special builtins with variable number of operands. */
29189 {
29195 }
29197 /* Add all builtins with variable number of operands. */
29201 {
29207 }
29209 /* pcmpestr[im] insns. */
29213 {
29216 else
29219 }
29221 /* pcmpistr[im] insns. */
29225 {
29228 else
29231 }
29233 /* comi/ucomi insns. */
29235 {
29238 else
29241 }
29243 /* SSE */
29249 /* SSE or 3DNow!A */
29252 IX86_BUILTIN_MASKMOVQ);
29254 /* SSE2 */
29263 /* SSE3. */
29269 /* AES */
29283 /* PCLMUL */
29287 /* RDRND */
29294 IX86_BUILTIN_RDRAND64_STEP);
29296 /* AVX2 */
29298 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
29299 IX86_BUILTIN_GATHERSIV2DF);
29302 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
29303 IX86_BUILTIN_GATHERSIV4DF);
29306 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
29307 IX86_BUILTIN_GATHERDIV2DF);
29310 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
29311 IX86_BUILTIN_GATHERDIV4DF);
29314 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
29315 IX86_BUILTIN_GATHERSIV4SF);
29318 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
29319 IX86_BUILTIN_GATHERSIV8SF);
29322 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
29323 IX86_BUILTIN_GATHERDIV4SF);
29326 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
29327 IX86_BUILTIN_GATHERDIV8SF);
29330 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
29331 IX86_BUILTIN_GATHERSIV2DI);
29334 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
29335 IX86_BUILTIN_GATHERSIV4DI);
29338 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
29339 IX86_BUILTIN_GATHERDIV2DI);
29342 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
29343 IX86_BUILTIN_GATHERDIV4DI);
29346 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
29347 IX86_BUILTIN_GATHERSIV4SI);
29350 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
29351 IX86_BUILTIN_GATHERSIV8SI);
29354 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
29355 IX86_BUILTIN_GATHERDIV4SI);
29358 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
29359 IX86_BUILTIN_GATHERDIV8SI);
29362 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
29363 IX86_BUILTIN_GATHERALTSIV4DF);
29366 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
29367 IX86_BUILTIN_GATHERALTDIV8SF);
29370 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
29371 IX86_BUILTIN_GATHERALTSIV4DI);
29374 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
29375 IX86_BUILTIN_GATHERALTDIV8SI);
29377 /* RTM. */
29381 /* MMX access to the vec_init patterns. */
29386 V4HI_FTYPE_HI_HI_HI_HI,
29387 IX86_BUILTIN_VEC_INIT_V4HI);
29390 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
29391 IX86_BUILTIN_VEC_INIT_V8QI);
29393 /* Access to the vec_extract patterns. */
29415 /* Access to the vec_set patterns. */
29436 /* RDSEED */
29445 /* ADCX */
29450 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
29451 IX86_BUILTIN_ADDCARRYX64);
29453 /* Add FMA4 multi-arg argument instructions */
29455 {
29461 }
29462 }
29464 /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
29465 to return a pointer to VERSION_DECL if the outcome of the expression
29466 formed by PREDICATE_CHAIN is true. This function will be called during
29467 version dispatch to decide which function version to execute. It returns
29468 the basic block at the end, to which more conditions can be added. */
29470 static basic_block
29473 {
29474 gimple return_stmt;
29476 gimple convert_stmt;
29477 gimple call_cond_stmt;
29478 gimple if_else_stmt;
29484 gimple_seq gseq;
29501 {
29509 }
29512 {
29527 else
29528 {
29529 gimple assign_stmt;
29530 /* Use MIN_EXPR to check if any integer is zero?.
29531 and_expr_var = min_expr <cond_var, and_expr_var> */
29539 }
29540 }
29543 integer_zero_node,
29573 }
29575 /* This parses the attribute arguments to target in DECL and determines
29576 the right builtin to use to match the platform specification.
29577 It returns the priority value for this version decl. If PREDICATE_LIST
29578 is not NULL, it stores the list of cpu features that need to be checked
29579 before dispatching this function. */
29581 static unsigned int
29583 {
29584 tree attrs;
29586 tree target_node;
29593 /* Priority of i386 features, greater value is higher priority. This is
29594 used to decide the order in which function dispatch must happen. For
29595 instance, a version specialized for SSE4.2 should be checked for dispatch
29596 before a version for SSE3, as SSE4.2 implies SSE3. */
29597 enum feature_priority
29598 {
29600 P_MMX,
29601 P_SSE,
29602 P_SSE2,
29603 P_SSE3,
29604 P_SSSE3,
29605 P_PROC_SSSE3,
29606 P_SSE4_a,
29607 P_PROC_SSE4_a,
29608 P_SSE4_1,
29609 P_SSE4_2,
29610 P_PROC_SSE4_2,
29611 P_POPCNT,
29612 P_AVX,
29613 P_AVX2,
29614 P_FMA,
29615 P_PROC_FMA
29616 };
29620 /* These are the target attribute strings for which a dispatcher is
29621 available, from fold_builtin_cpu. */
29623 static struct _feature_list
29624 {
29627 }
29629 {
29640 };
29643 static unsigned int NUM_FEATURES
29659 /* Return priority zero for default function. */
29663 /* Handle arch= if specified. For priority, set it to be 1 more than
29664 the best instruction set the processor can handle. For instance, if
29665 there is a version for atom and a version for ssse3 (the highest ISA
29666 priority for atom), the atom version must be checked for dispatch
29667 before the ssse3 version. */
29669 {
29679 {
29681 {
29706 }
29707 }
29712 {
29716 }
29719 {
29721 /* For a C string literal the length includes the trailing NULL. */
29724 predicate_chain);
29725 }
29726 }
29728 /* Process feature name. */
29735 {
29736 /* Do not process "arch=" */
29738 {
29741 }
29743 {
29745 {
29747 {
29752 predicate_chain);
29753 }
29754 /* Find the maximum priority feature. */
29759 }
29760 }
29762 {
29766 }
29768 }
29772 {
29775 attrs_str);
29777 }
29779 {
29782 }
29785 }
29787 /* This compares the priority of target features in function DECL1
29788 and DECL2. It returns positive value if DECL1 is higher priority,
29789 negative value if DECL2 is higher priority and 0 if they are the
29790 same. */
29792 static int
29794 {
29799 }
29801 /* V1 and V2 point to function versions with different priorities
29802 based on the target ISA. This function compares their priorities. */
29804 static int
29806 {
29808 {
29809 tree version_decl;
29810 tree predicate_chain;
29817 }
29819 /* This function generates the dispatch function for
29820 multi-versioned functions. DISPATCH_DECL is the function which will
29821 contain the dispatch logic. FNDECLS are the function choices for
29822 dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
29823 in DISPATCH_DECL in which the dispatch code is generated. */
29825 static int
29829 {
29830 tree default_decl;
29831 gimple ifunc_cpu_init_stmt;
29832 gimple_seq gseq;
29834 tree ele;
29840 struct _function_version_info
29841 {
29842 tree version_decl;
29843 tree predicate_chain;
29851 /*fndecls_p is actually a vector. */
29854 /* At least one more version other than the default. */
29861 /* The first version in the vector is the default decl. */
29867 /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
29868 constructors, so explicity call __builtin_cpu_init here. */
29879 {
29883 /* Get attribute string, parse it and find the right predicate decl.
29884 The predicate function could be a lengthy combination of many
29885 features, like arch-type and various isa-variants. */
29896 actual_versions++;
29897 }
29899 /* Sort the versions according to descending order of dispatch priority. The
29900 priority is based on the ISA. This is not a perfect solution. There
29901 could still be ambiguity. If more than one function version is suitable
29902 to execute, which one should be dispatched? In future, allow the user
29903 to specify a dispatch priority next to the version. */
29913 /* dispatch default version at the end. */
29919 }
29921 /* Comparator function to be used in qsort routine to sort attribute
29922 specification strings to "target". */
29924 static int
29926 {
29930 }
29932 /* ARGLIST is the argument to target attribute. This function tokenizes
29933 the comma separated arguments, sorts them and returns a string which
29934 is a unique identifier for the comma separated arguments. It also
29935 replaces non-identifier characters "=,-" with "_". */
29939 {
29940 tree arg;
29949 {
29954 argnum++;
29957 argnum++;
29958 }
29963 {
29969 }
29971 /* Replace "=,-" with "_". */
29984 {
29986 i++;
29988 }
29995 {
30000 }
30005 }
30007 /* This function changes the assembler name for functions that are
30008 versions. If DECL is a function version and has a "target"
30009 attribute, it appends the attribute string to its assembler name. */
30011 static tree
30013 {
30014 tree version_attr;
30022 "Function versions cannot be marked as gnu_inline,"
30031 /* target attribute string cannot be NULL. */
30035 version_string
30046 /* Allow assembler name to be modified if already set. */
30054 }
30056 /* This function returns true if FN1 and FN2 are versions of the same function,
30057 that is, the target strings of the function decls are different. This assumes
30058 that FN1 and FN2 have the same signature. */
30060 static bool
30062 {
30074 /* At least one function decl should have the target attribute specified. */
30078 /* Diagnose missing target attribute if one of the decls is already
30079 multi-versioned. */
30081 {
30083 {
30085 {
30090 }
30093 fn2);
30096 /* Prevent diagnosing of the same error multiple times. */
30101 }
30103 }
30108 /* The sorted target strings must be different for fn1 and fn2
30109 to be versions. */
30112 else
30119 }
30121 static tree
30123 {
30124 /* For function version, add the target suffix to the assembler name. */
30128 #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
30130 #endif
30133 }
30135 /* Return a new name by appending SUFFIX to the DECL name. If make_unique
30136 is true, append the full path name of the source file. */
30140 {
30148 /* Get a unique name that can be used globally without any chances
30149 of collision at link time. */
30159 /* Use '.' to concatenate names as it is demangler friendly. */
30162 suffix);
30163 else
30167 }
30169 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
30171 /* Make a dispatcher declaration for the multi-versioned function DECL.
30172 Calls to DECL function will be replaced with calls to the dispatcher
30173 by the front-end. Return the decl created. */
30175 static tree
30177 {
30178 tree func_decl;
30198 /* Mark this func as external, the resolver will flip it again if
30199 it gets generated. */
30201 /* This will be of type IFUNCs have to be externally visible. */
30205 }
30207 #endif
30209 /* Returns true if decl is multi-versioned and DECL is the default function,
30210 that is it is not tagged with target specific optimization. */
30212 static bool
30214 {
30223 }
30225 /* Make a dispatcher declaration for the multi-versioned function DECL.
30226 Calls to DECL function will be replaced with calls to the dispatcher
30227 by the front-end. Returns the decl of the dispatcher function. */
30229 static tree
30231 {
30253 /* Find the default version and make it the first node. */
30255 /* Go to the beginning of the chain. */
30260 {
30265 }
30267 /* If there is no default node, just return NULL. */
30271 /* Make default info the first node. */
30273 {
30280 }
30284 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
30286 {
30291 /* Right now, the dispatching is done via ifunc. */
30297 dispatcher_version_info
30302 /* Set the dispatcher for all the versions. */
30305 {
30308 }
30309 }
30310 else
30311 #endif
30312 {
30314 "multiversioning needs ifunc which is not supported "
30316 }
30319 }
30321 /* Makes a function attribute of the form NAME(ARG_NAME) and chains
30322 it to CHAIN. */
30324 static tree
30326 {
30327 tree attr_name;
30328 tree attr_arg_name;
30329 tree attr_args;
30330 tree attr;
30337 }
30339 /* Make the resolver function decl to dispatch the versions of
30340 a multi-versioned function, DEFAULT_DECL. Create an
30341 empty basic block in the resolver and store the pointer in
30342 EMPTY_BB. Return the decl of the resolver function. */
30344 static tree
30348 {
30353 /* IFUNC's have to be globally visible. So, if the default_decl is
30354 not, then the name of the IFUNC should be made unique. */
30358 /* Append the filename to the resolver function if the versions are
30359 not externally visible. This is because the resolver function has
30360 to be externally visible for the loader to find it. So, appending
30361 the filename will prevent conflicts with a resolver function from
30362 another module which is based on the same version name. */
30365 /* The resolver function should return a (void *). */
30376 /* IFUNC resolvers have to be externally visible. */
30380 /* Resolver is not external, body is generated. */
30390 {
30391 /* In this case, each translation unit with a call to this
30392 versioned function will put out a resolver. Ensure it
30393 is comdat to keep just one copy. */
30396 }
30397 /* Build result decl and add to function_decl. */
30413 /* Mark dispatch_decl as "ifunc" with resolver as resolver_name. */
30417 /* Create the alias for dispatch to resolver here. */
30418 /*cgraph_create_function_alias (dispatch_decl, decl);*/
30422 }
30424 /* Generate the dispatching code body to dispatch multi-versioned function
30425 DECL. The target hook is called to process the "target" attributes and
30426 provide the code to dispatch the right function at run-time. NODE points
30427 to the dispatcher decl whose body will be created. */
30429 static tree
30431 {
30432 tree resolver_decl;
30433 basic_block empty_bb;
30435 tree default_ver_decl;
30451 /* The first version in the chain corresponds to the default version. */
30454 /* node is going to be an alias, so remove the finalized bit. */
30468 {
30470 /* Check for virtual functions here again, as by this time it should
30471 have been determined if this function needs a vtable index or
30472 not. This happens for methods in derived classes that override
30473 virtual methods in base classes but are not explicitly marked as
30474 virtual. */
30479 }
30486 }
30487 /* This builds the processor_model struct type defined in
30488 libgcc/config/i386/cpuinfo.c */
30490 static tree
30492 {
30499 /* The first 3 fields are unsigned int. */
30501 {
30507 }
30509 /* The last field is an array of unsigned integers of size one. */
30520 }
30522 /* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
30524 static tree
30526 {
30527 tree new_decl;
30530 VAR_DECL,
30532 type);
30545 }
30547 /* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
30548 into an integer defined in libgcc/config/i386/cpuinfo.c */
30550 static tree
30552 {
30558 /* This is the order of bit-fields in __processor_features in cpuinfo.c */
30559 enum processor_features
30560 {
30562 F_MMX,
30563 F_POPCNT,
30564 F_SSE,
30565 F_SSE2,
30566 F_SSE3,
30567 F_SSSE3,
30568 F_SSE4_1,
30569 F_SSE4_2,
30570 F_AVX,
30571 F_AVX2,
30572 F_MAX
30573 };
30575 /* These are the values for vendor types and cpu types and subtypes
30576 in cpuinfo.c. Cpu types and subtypes should be subtracted by
30577 the corresponding start value. */
30578 enum processor_model
30579 {
30581 M_AMD,
30582 M_CPU_TYPE_START,
30583 M_INTEL_ATOM,
30584 M_INTEL_CORE2,
30585 M_INTEL_COREI7,
30586 M_AMDFAM10H,
30587 M_AMDFAM15H,
30588 M_INTEL_SLM,
30589 M_CPU_SUBTYPE_START,
30590 M_INTEL_COREI7_NEHALEM,
30591 M_INTEL_COREI7_WESTMERE,
30592 M_INTEL_COREI7_SANDYBRIDGE,
30593 M_AMDFAM10H_BARCELONA,
30594 M_AMDFAM10H_SHANGHAI,
30595 M_AMDFAM10H_ISTANBUL,
30596 M_AMDFAM15H_BDVER1,
30597 M_AMDFAM15H_BDVER2,
30598 M_AMDFAM15H_BDVER3
30599 };
30601 static struct _arch_names_table
30602 {
30605 }
30607 {
30625 };
30627 static struct _isa_names_table
30628 {
30631 }
30633 {
30645 };
30659 {
30660 /* *args must be a expr that can contain other EXPRS leading to a
30661 STRING_CST. */
30663 {
30666 }
30668 }
30673 {
30674 tree ref;
30675 tree field;
30676 tree final;
30679 unsigned int NUM_ARCH_NAMES
30688 {
30692 }
30697 /* CPU types are stored in the next field. */
30700 {
30703 }
30705 /* CPU subtypes are stored in the next field. */
30707 {
30710 }
30712 /* Get the appropriate field in __cpu_model. */
30716 /* Check the value. */
30720 }
30722 {
30723 tree ref;
30724 tree array_elt;
30725 tree field;
30726 tree final;
30729 unsigned int NUM_ISA_NAMES
30738 {
30742 }
30745 /* Get the last field, which is __cpu_features. */
30749 /* Get the appropriate field: __cpu_model.__cpu_features */
30753 /* Access the 0th element of __cpu_features array. */
30758 /* Return __cpu_model.__cpu_features[0] & field_val */
30762 }
30764 }
30766 static tree
30769 {
30771 {
30776 {
30779 }
30780 }
30782 #ifdef SUBTARGET_FOLD_BUILTIN
30784 #endif
30787 }
30789 /* Make builtins to detect cpu type and features supported. NAME is
30790 the builtin name, CODE is the builtin code, and FTYPE is the function
30791 type of the builtin. */
30793 static void
30796 {
30797 tree decl;
30798 tree type;
30806 }
30808 /* Make builtins to get CPU type and features supported. The created
30809 builtins are :
30811 __builtin_cpu_init (), to detect cpu type and features,
30812 __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
30813 __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
30814 */
30816 static void
30818 {
30825 }
30827 /* Internal method for ix86_init_builtins. */
30829 static void
30831 {
30843 sysv_va_ref =
30846 fnvoid_va_end_ms =
30848 fnvoid_va_start_ms =
30850 fnvoid_va_end_sysv =
30852 fnvoid_va_start_sysv =
30854 NULL_TREE);
30855 fnvoid_va_copy_ms =
30857 NULL_TREE);
30858 fnvoid_va_copy_sysv =
30874 }
30876 static void
30878 {
30881 /* The __float80 type. */
30884 {
30885 /* The __float80 type. */
30890 }
30893 /* The __float128 type. */
30899 /* This macro is built by i386-builtin-types.awk. */
30900 DEFINE_BUILTIN_PRIMITIVE_TYPES;
30901 }
30903 static void
30905 {
30906 tree t;
30910 /* Builtins to get CPU type and features. */
30913 /* TFmode support builtins. */
30919 /* We will expand them to normal call if SSE isn't available since
30920 they are used by libgcc. */
30939 #ifdef SUBTARGET_INIT_BUILTINS
30940 SUBTARGET_INIT_BUILTINS;
30941 #endif
30942 }
30944 /* Return the ix86 builtin for CODE. */
30946 static tree
30948 {
30953 }
30955 /* Errors in the source file can cause expand_expr to return const0_rtx
30956 where we expect a vector. To avoid crashing, use one of the vector
30957 clear instructions. */
30958 static rtx
30960 {
30964 }
30966 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
30968 static rtx
30970 {
30971 rtx pat;
30991 {
30995 }
31009 }
31011 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
31013 static rtx
31017 {
31018 rtx pat;
31026 rtx op;
31033 {
31113 }
31123 {
31130 {
31132 {
31135 {
31153 xop_rotl:
31155 {
31158 gcc_checking_assert
31160 }
31161 else
31162 {
31163 gcc_checking_assert
31174 }
31178 }
31179 }
31180 }
31181 else
31182 {
31183 non_constant:
31187 /* If we aren't optimizing, only allow one memory operand to be
31188 generated. */
31190 num_memory++;
31198 }
31202 }
31205 {
31216 else
31217 {
31223 }
31236 }
31243 }
31245 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
31246 insns with vec_merge. */
31248 static rtx
31250 rtx target)
31251 {
31252 rtx pat;
31279 }
31281 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
31283 static rtx
31286 {
31287 rtx pat;
31292 rtx op2;
31303 /* Swap operands if we have a comparison that isn't available in
31304 hardware. */
31306 {
31311 }
31331 }
31333 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
31335 static rtx
31337 rtx target)
31338 {
31339 rtx pat;
31353 /* Swap operands if we have a comparison that isn't available in
31354 hardware. */
31356 {
31360 }
31381 const0_rtx)));
31384 }
31386 /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
31388 static rtx
31390 rtx target)
31391 {
31392 rtx pat;
31417 }
31419 static rtx
31422 {
31423 rtx pat;
31428 rtx op2;
31455 }
31457 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
31459 static rtx
31461 rtx target)
31462 {
31463 rtx pat;
31496 const0_rtx)));
31499 }
31501 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
31503 static rtx
31506 {
31507 rtx pat;
31545 {
31548 }
31551 {
31560 }
31562 {
31571 }
31572 else
31573 {
31580 }
31588 {
31593 emit_insn
31597 FLAGS_REG),
31598 const0_rtx)));
31600 }
31601 else
31603 }
31606 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
31608 static rtx
31611 {
31612 rtx pat;
31640 {
31643 }
31646 {
31655 }
31657 {
31666 }
31667 else
31668 {
31675 }
31683 {
31688 emit_insn
31692 FLAGS_REG),
31693 const0_rtx)));
31695 }
31696 else
31698 }
31700 /* Subroutine of ix86_expand_builtin to take care of insns with
31701 variable number of operands. */
31703 static rtx
31706 {
31711 struct
31712 {
31713 rtx op;
31725 {
32004 }
32009 {
32012 }
32015 {
32022 }
32023 else
32024 {
32027 }
32030 {
32037 {
32038 /* SIMD shift insns take either an 8-bit immediate or
32039 register as count. But builtin functions take int as
32040 count. If count doesn't match, we put it in register. */
32042 {
32046 }
32047 }
32049 {
32052 {
32106 {
32109 {
32112 }
32118 }
32120 }
32121 }
32122 else
32123 {
32127 /* If we aren't optimizing, only allow one memory operand to
32128 be generated. */
32130 num_memory++;
32133 {
32136 }
32137 else
32138 {
32141 }
32142 }
32146 }
32149 {
32166 }
32173 }
32175 /* Subroutine of ix86_expand_builtin to take care of special insns
32176 with variable number of operands. */
32178 static rtx
32181 {
32182 tree arg;
32185 struct
32186 {
32187 rtx op;
32197 {
32245 /* Reserve memory operand for target. */
32276 /* Reserve memory operand for target. */
32290 }
32295 {
32300 {
32303 }
32304 else
32307 }
32308 else
32309 {
32316 }
32319 {
32328 {
32330 {
32336 else
32339 }
32340 }
32341 else
32342 {
32344 {
32345 /* This must be the memory operand. */
32350 }
32351 else
32352 {
32353 /* This must be register. */
32360 }
32361 }
32365 }
32368 {
32383 }
32389 }
32391 /* Return the integer constant in ARG. Constrain it to be in the range
32392 of the subparts of VEC_TYPE; issue an error if not. */
32394 static int
32396 {
32401 {
32404 }
32407 }
32409 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32410 ix86_expand_vector_init. We DO have language-level syntax for this, in
32411 the form of (type){ init-list }. Except that since we can't place emms
32412 instructions from inside the compiler, we can't allow the use of MMX
32413 registers unless the user explicitly asks for it. So we do *not* define
32414 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
32415 we have builtins invoked by mmintrin.h that gives us license to emit
32416 these sorts of instructions. */
32418 static rtx
32420 {
32430 {
32433 }
32440 }
32442 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32443 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
32444 had a language-level syntax for referencing vector elements. */
32446 static rtx
32448 {
32452 rtx op0;
32472 }
32474 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32475 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
32476 a language-level syntax for referencing vector elements. */
32478 static rtx
32480 {
32504 /* OP0 is the source of these builtin functions and shouldn't be
32505 modified. Create a copy, use it and return it as target. */
32511 }
32513 /* Expand an expression EXP that calls a built-in function,
32514 with result going to TARGET if that's convenient
32515 (and in mode MODE if that's convenient).
32516 SUBTARGET may be used as the target for computing one of EXP's operands.
32517 IGNORE is nonzero if the value is to be ignored. */
32519 static rtx
32522 {
32532 /* For CPU builtins that can be folded, fold first and expand the fold. */
32534 {
32536 {
32537 /* Make it call __cpu_indicator_init in libgcc. */
32543 }
32546 {
32551 }
32552 }
32554 /* Determine whether the builtin function is available under the current ISA.
32555 Originally the builtin was not created if it wasn't applicable to the
32556 current ISA based on the command line switches. With function specific
32557 options, we need to check in the context of the function making the call
32558 whether it is supported. */
32561 {
32567 else
32568 {
32572 }
32574 }
32577 {
32581 ? CODE_FOR_mmx_maskmovq
32583 /* Note the arg order is different from the operand order. */
32684 {
32685 REAL_VALUE_TYPE inf;
32686 rtx tmp;
32698 }
32708 {
32714 insn = (TARGET_64BIT
32718 }
32720 {
32721 insn = (TARGET_64BIT
32725 }
32726 else
32727 {
32730 insn = (TARGET_64BIT
32738 {
32741 }
32743 }
32746 {
32747 /* mode is VOIDmode if __builtin_rd* has been called
32748 without lhs. */
32752 }
32755 {
32760 }
32770 {
32785 }
32791 {
32794 }
32814 {
32817 }
32823 {
32827 {
32848 }
32853 }
32854 else
32855 {
32857 {
32869 }
32871 }
32901 ? CODE_FOR_tbm_bextri_si
32904 {
32907 }
32908 else
32909 {
32918 }
32934 rdrand_step:
32941 {
32944 }
32950 /* Emit SImode conditional move. */
32952 {
32955 }
32958 else
32965 const0_rtx);
32984 rdseed_step:
32991 {
32994 }
33000 const0_rtx);
33018 addcarryx:
33026 /* Generate CF from input operand. */
33031 /* Gen ADCX instruction to compute X+Y+CF. */
33046 /* Store the result. */
33049 {
33052 }
33055 /* Return current CF value. */
33124 gather_gen:
33135 /* Note the arg order is different from the operand order. */
33144 else
33149 {
33155 }
33158 {
33163 else
33173 }
33175 /* Force memory operand only with base register here. But we
33176 don't want to do it on memory operand for other builtin
33177 functions. */
33189 {
33192 }
33194 /* Optimize. If mask is known to have all high bits set,
33195 replace op0 with pc_rtx to signal that the instruction
33196 overwrites the whole destination and doesn't use its
33197 previous contents. */
33199 {
33201 {
33204 {
33208 negative++;
33211 negative++;
33212 }
33215 }
33217 {
33218 /* Recognize also when mask is like:
33219 __v2df src = _mm_setzero_pd ();
33220 __v2df mask = _mm_cmpeq_pd (src, src);
33221 or
33222 __v8sf src = _mm256_setzero_ps ();
33223 __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
33224 as that is a cheaper way to load all ones into
33225 a register than having to load a constant from
33226 memory. */
33229 {
33234 {
33241 /* FALLTHRU */
33251 }
33252 }
33253 }
33254 }
33263 {
33270 else
33272 }
33273 else
33284 {
33287 }
33293 }
33306 {
33310 /* Emit a normal call if SSE isn't available. */
33314 }
33339 }
33341 /* Returns a function decl for a vectorized version of the builtin function
33342 with builtin function code FN and the result vector type TYPE, or NULL_TREE
33343 if it is not available. */
33345 static tree
33347 tree type_in)
33348 {
33364 {
33367 {
33372 }
33377 {
33382 }
33388 /* The round insn does not trap on denormals. */
33393 {
33398 }
33404 /* The round insn does not trap on denormals. */
33409 {
33414 }
33420 /* The round insn does not trap on denormals. */
33425 {
33430 }
33436 /* The round insn does not trap on denormals. */
33441 {
33446 }
33453 {
33458 }
33465 {
33470 }
33476 /* The round insn does not trap on denormals. */
33481 {
33486 }
33492 /* The round insn does not trap on denormals. */
33497 {
33502 }
33507 {
33512 }
33517 {
33522 }
33526 /* The round insn does not trap on denormals. */
33531 {
33536 }
33540 /* The round insn does not trap on denormals. */
33545 {
33550 }
33554 /* The round insn does not trap on denormals. */
33559 {
33564 }
33568 /* The round insn does not trap on denormals. */
33573 {
33578 }
33582 /* The round insn does not trap on denormals. */
33587 {
33592 }
33596 /* The round insn does not trap on denormals. */
33601 {
33606 }
33610 /* The round insn does not trap on denormals. */
33615 {
33620 }
33624 /* The round insn does not trap on denormals. */
33629 {
33634 }
33638 /* The round insn does not trap on denormals. */
33643 {
33648 }
33652 /* The round insn does not trap on denormals. */
33657 {
33662 }
33667 {
33672 }
33677 {
33682 }
33687 }
33689 /* Dispatch to a handler for a vectorization library. */
33692 type_in);
33695 }
33697 /* Handler for an SVML-style interface to
33698 a library with vectorized intrinsics. */
33700 static tree
33702 {
33710 /* The SVML is suitable for unsafe math only. */
33723 {
33770 }
33779 {
33782 }
33783 else
33786 /* Convert to uppercase. */
33791 args;
33793 arity++;
33797 else
33800 /* Build a function declaration for the vectorized function. */
33809 }
33811 /* Handler for an ACML-style interface to
33812 a library with vectorized intrinsics. */
33814 static tree
33816 {
33824 /* The ACML is 64bits only and suitable for unsafe math only as
33825 it does not correctly support parts of IEEE with the required
33826 precision such as denormals. */
33840 {
33870 }
33877 args;
33879 arity++;
33883 else
33886 /* Build a function declaration for the vectorized function. */
33895 }
33897 /* Returns a decl of a function that implements gather load with
33898 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
33899 Return NULL_TREE if it is not available. */
33901 static tree
33904 {
33920 /* v*gather* insn sign extends index to pointer mode. */
33932 {
33959 }
33962 }
33964 /* Returns a code for a target-specific builtin that implements
33965 reciprocal of the function, or NULL_TREE if not available. */
33967 static tree
33970 {
33977 /* Machine dependent builtins. */
33979 {
33980 /* Vectorized version of sqrt to rsqrt conversion. */
33989 }
33990 else
33991 /* Normal builtins. */
33993 {
33994 /* Sqrt to rsqrt conversion. */
34000 }
34001 }
34002 \f
34003 /* Helper for avx_vpermilps256_operand et al. This is also used by
34004 the expansion functions to turn the parallel back into a mask.
34005 The return value is 0 for no match and the imm8+1 for a match. */
34007 int
34009 {
34017 /* Validate that all of the elements are constants, and not totally
34018 out of range. Copy the data into an integral array to make the
34019 subsequent checks easier. */
34021 {
34031 }
34034 {
34036 /* In the 256-bit DFmode case, we can only move elements within
34037 a 128-bit lane. */
34039 {
34043 }
34045 {
34049 }
34053 /* In the 256-bit SFmode case, we have full freedom of movement
34054 within the low 128-bit lane, but the high 128-bit lane must
34055 mirror the exact same pattern. */
34060 /* FALLTHRU */
34064 /* In the 128-bit case, we've full freedom in the placement of
34065 the elements from the source operand. */
34072 }
34074 /* Make sure success has a non-zero value by adding one. */
34076 }
34078 /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
34079 the expansion functions to turn the parallel back into a mask.
34080 The return value is 0 for no match and the imm8+1 for a match. */
34082 int
34084 {
34092 /* Validate that all of the elements are constants, and not totally
34093 out of range. Copy the data into an integral array to make the
34094 subsequent checks easier. */
34096 {
34106 }
34108 /* Validate that the halves of the permute are halves. */
34116 /* Reconstruct the mask. */
34118 {
34124 }
34126 /* Make sure success has a non-zero value by adding one. */
34128 }
34129 \f
34130 /* Store OPERAND to the memory after reload is completed. This means
34131 that we can't easily use assign_stack_local. */
34132 rtx
34134 {
34135 rtx result;
34139 {
34142 stack_pointer_rtx,
34145 }
34147 {
34149 {
34153 /* FALLTHRU */
34159 stack_pointer_rtx)),
34160 operand));
34164 }
34166 }
34167 else
34168 {
34170 {
34172 {
34179 stack_pointer_rtx)),
34185 stack_pointer_rtx)),
34187 }
34190 /* Store HImodes as SImodes. */
34192 /* FALLTHRU */
34198 stack_pointer_rtx)),
34199 operand));
34203 }
34205 }
34207 }
34209 /* Free operand from the memory. */
34210 void
34212 {
34214 {
34219 else
34221 /* Use LEA to deallocate stack space. In peephole2 it will be converted
34222 to pop or add instruction if registers are available. */
34226 }
34227 }
34229 /* Return a register priority for hard reg REGNO. */
34230 static int
34232 {
34233 /* ebp and r13 as the base always wants a displacement, r12 as the
34234 base always wants an index. So discourage their usage in an
34235 address. */
34240 /* New x86-64 int registers result in bigger code size. Discourage
34241 them. */
34244 /* New x86-64 SSE registers result in bigger code size. Discourage
34245 them. */
34248 /* Usage of AX register results in smaller code. Prefer it. */
34252 }
34254 /* Implement TARGET_PREFERRED_RELOAD_CLASS.
34256 Put float CONST_DOUBLE in the constant pool instead of fp regs.
34257 QImode must go into class Q_REGS.
34258 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
34259 movdf to do mem-to-mem moves through integer regs. */
34261 static reg_class_t
34263 {
34266 /* We're only allowed to return a subclass of CLASS. Many of the
34267 following checks fail for NO_REGS, so eliminate that early. */
34271 /* All classes can load zeros. */
34275 /* Force constants into memory if we are loading a (nonzero) constant into
34276 an MMX, SSE or MASK register. This is because there are no MMX/SSE/MASK
34277 instructions to load from a constant. */
34284 /* Prefer SSE regs only, if we can use them for math. */
34288 /* Floating-point constants need more complex checks. */
34290 {
34291 /* General regs can load everything. */
34295 /* Floats can load 0 and 1 plus some others. Note that we eliminated
34296 zero above. We only want to wind up preferring 80387 registers if
34297 we plan on doing computation with them. */
34300 {
34301 /* Limit class to non-sse. */
34310 }
34313 }
34315 /* Generally when we see PLUS here, it's the function invariant
34316 (plus soft-fp const_int). Which can only be computed into general
34317 regs. */
34321 /* QImode constants are easy to load, but non-constant QImode data
34322 must go into Q_REGS. */
34324 {
34330 }
34333 }
34335 /* Discourage putting floating-point values in SSE registers unless
34336 SSE math is being used, and likewise for the 387 registers. */
34337 static reg_class_t
34339 {
34342 /* Restrict the output reload class to the register bank that we are doing
34343 math on. If we would like not to return a subset of CLASS, reject this
34344 alternative: if reload cannot do this, it will still use its choice. */
34350 {
34355 else
34357 }
34360 }
34362 static reg_class_t
34365 {
34366 /* Double-word spills from general registers to non-offsettable memory
34367 references (zero-extended addresses) require special handling. */
34373 {
34375 ? CODE_FOR_reload_noff_load
34377 /* Add the cost of moving address to a temporary. */
34381 }
34383 /* QImode spills from non-QI registers require
34384 intermediate register on 32bit targets. */
34390 {
34395 else
34401 /* Return Q_REGS if the operand is in memory. */
34404 }
34406 /* This condition handles corner case where an expression involving
34407 pointers gets vectorized. We're trying to use the address of a
34408 stack slot as a vector initializer.
34410 (set (reg:V2DI 74 [ vect_cst_.2 ])
34411 (vec_duplicate:V2DI (reg/f:DI 20 frame)))
34413 Eventually frame gets turned into sp+offset like this:
34415 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34416 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
34417 (const_int 392 [0x188]))))
34419 That later gets turned into:
34421 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34422 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
34423 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
34425 We'll have the following reload recorded:
34427 Reload 0: reload_in (DI) =
34428 (plus:DI (reg/f:DI 7 sp)
34429 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
34430 reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34431 SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
34432 reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
34433 reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34434 reload_reg_rtx: (reg:V2DI 22 xmm1)
34436 Which isn't going to work since SSE instructions can't handle scalar
34437 additions. Returning GENERAL_REGS forces the addition into integer
34438 register and reload can handle subsequent reloads without problems. */
34446 }
34448 /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
34450 static bool
34452 {
34454 {
34470 }
34473 }
34475 /* If we are copying between general and FP registers, we need a memory
34476 location. The same is true for SSE and MMX registers.
34478 To optimize register_move_cost performance, allow inline variant.
34480 The macro can't work reliably when one of the CLASSES is class containing
34481 registers from multiple units (SSE, MMX, integer). We avoid this by never
34482 combining those units in single alternative in the machine description.
34483 Ensure that this constraint holds to avoid unexpected surprises.
34485 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
34486 enforce these sanity checks. */
34491 {
34500 {
34503 }
34508 /* ??? This is a lie. We do have moves between mmx/general, and for
34509 mmx/sse2. But by saying we need secondary memory we discourage the
34510 register allocator from using the mmx registers unless needed. */
34515 {
34516 /* SSE1 doesn't have any direct moves from other classes. */
34520 /* If the target says that inter-unit moves are more expensive
34521 than moving through memory, then don't generate them. */
34526 /* Between SSE and general, we have moves no larger than word size. */
34529 }
34532 }
34534 bool
34537 {
34539 }
34541 /* Implement the TARGET_CLASS_MAX_NREGS hook.
34543 On the 80386, this is the size of MODE in words,
34544 except in the FP regs, where a single reg is always enough. */
34546 static unsigned char
34548 {
34550 {
34555 else
34557 }
34558 else
34559 {
34562 else
34564 }
34565 }
34567 /* Return true if the registers in CLASS cannot represent the change from
34568 modes FROM to TO. */
34570 bool
34573 {
34577 /* x87 registers can't do subreg at all, as all values are reformatted
34578 to extended precision. */
34583 {
34584 /* Vector registers do not support QI or HImode loads. If we don't
34585 disallow a change to these modes, reload will assume it's ok to
34586 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
34587 the vec_dupv4hi pattern. */
34591 /* Vector registers do not support subreg with nonzero offsets, which
34592 are otherwise valid for integer registers. Since we can't see
34593 whether we have a nonzero offset from here, prohibit all
34594 nonparadoxical subregs changing size. */
34597 }
34600 }
34602 /* Return the cost of moving data of mode M between a
34603 register and memory. A value of 2 is the default; this cost is
34604 relative to those in `REGISTER_MOVE_COST'.
34606 This function is used extensively by register_move_cost that is used to
34607 build tables at startup. Make it inline in this case.
34608 When IN is 2, return maximum of in and out move cost.
34610 If moving between registers and memory is more expensive than
34611 between two registers, you should define this macro to express the
34612 relative cost.
34614 Model also increased moving costs of QImode registers in non
34615 Q_REGS classes.
34616 */
34620 {
34623 {
34626 {
34638 }
34642 }
34644 {
34647 {
34659 }
34663 }
34665 {
34668 {
34677 }
34681 }
34683 {
34686 {
34692 else
34697 }
34698 else
34699 {
34704 else
34706 }
34713 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
34720 else
34724 }
34725 }
34727 static int
34730 {
34732 }
34735 /* Return the cost of moving data from a register in class CLASS1 to
34736 one in class CLASS2.
34738 It is not required that the cost always equal 2 when FROM is the same as TO;
34739 on some machines it is expensive to move between registers if they are not
34740 general registers. */
34742 static int
34744 reg_class_t class2_i)
34745 {
34749 /* In case we require secondary memory, compute cost of the store followed
34750 by load. In order to avoid bad register allocation choices, we need
34751 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
34754 {
34760 /* In case of copying from general_purpose_register we may emit multiple
34761 stores followed by single load causing memory size mismatch stall.
34762 Count this as arbitrarily high cost of 20. */
34767 /* In the case of FP/MMX moves, the registers actually overlap, and we
34768 have to switch modes in order to treat them differently. */
34774 }
34776 /* Moves between SSE/MMX and integer unit are expensive. */
34780 /* ??? By keeping returned value relatively high, we limit the number
34781 of moves between integer and MMX/SSE registers for all targets.
34782 Additionally, high value prevents problem with x86_modes_tieable_p(),
34783 where integer modes in MMX/SSE registers are not tieable
34784 because of missing QImode and HImode moves to, from or between
34785 MMX/SSE registers. */
34795 }
34797 /* Return TRUE if hard register REGNO can hold a value of machine-mode
34798 MODE. */
34800 bool
34802 {
34803 /* Flags and only flags can only hold CCmode values. */
34817 {
34818 /* We implement the move patterns for all vector modes into and
34819 out of SSE registers, even when no operation instructions
34820 are available. */
34822 /* For AVX-512 we allow, regardless of regno:
34823 - XI mode
34824 - any of 512-bit wide vector mode
34825 - any scalar mode. */
34832 /* xmm16-xmm31 are only available for AVX-512. */
34836 /* OImode move is available only when AVX is enabled. */
34843 }
34845 {
34846 /* We implement the move patterns for 3DNOW modes even in MMX mode,
34847 so if the register is available at all, then we can move data of
34848 the given mode into or out of it. */
34851 }
34854 {
34855 /* Take care for QImode values - they can be in non-QI regs,
34856 but then they do cause partial register stalls. */
34861 /* LRA checks if the hard register is OK for the given mode.
34862 QImode values can live in non-QI regs, so we allow all
34863 registers here. */
34867 }
34868 /* We handle both integer and floats in the general purpose registers. */
34875 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
34876 on to use that value in smaller contexts, this can easily force a
34877 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
34878 supporting DImode, allow it. */
34883 }
34885 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
34886 tieable integer mode. */
34888 static bool
34890 {
34892 {
34905 }
34906 }
34908 /* Return true if MODE1 is accessible in a register that can hold MODE2
34909 without copying. That is, all register classes that can hold MODE2
34910 can also hold MODE1. */
34912 bool
34914 {
34922 /* MODE2 being XFmode implies fp stack or general regs, which means we
34923 can tie any smaller floating point modes to it. Note that we do not
34924 tie this with TFmode. */
34928 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
34929 that we can tie it with SFmode. */
34933 /* If MODE2 is only appropriate for an SSE register, then tie with
34934 any other mode acceptable to SSE registers. */
34944 /* If MODE2 is appropriate for an MMX register, then tie
34945 with any other mode acceptable to MMX registers. */
34952 }
34954 /* Return the cost of moving between two registers of mode MODE. */
34956 static int
34958 {
34962 {
34994 }
34996 /* Return the cost of moving between two registers of mode MODE,
34997 assuming that the move will be in pieces of at most UNITS bytes. */
34999 }
35001 /* Compute a (partial) cost for rtx X. Return true if the complete
35002 cost has been computed, and false if subexpressions should be
35003 scanned. In either case, *TOTAL contains the cost result. */
35005 static bool
35008 {
35015 {
35019 {
35022 }
35034 && (!TARGET_64BIT
35039 else
35045 {
35048 }
35050 {
35060 }
35062 {
35065 {
35074 }
35075 }
35076 /* Fall back to (MEM (SYMBOL_REF)), since that's where
35077 it'll probably end up. Add a penalty for size. */
35084 /* The zero extensions is often completely free on x86_64, so make
35085 it as cheap as possible. */
35091 else
35103 {
35106 {
35109 }
35112 {
35115 }
35116 }
35117 /* FALLTHRU */
35124 {
35125 /* ??? Should be SSE vector operation cost. */
35126 /* At least for published AMD latencies, this really is the same
35127 as the latency for a simple fpu operation like fabs. */
35128 /* V*QImode is emulated with 1-11 insns. */
35130 {
35133 {
35134 /* For XOP we use vpshab, which requires a broadcast of the
35135 value to the variable shift insn. For constants this
35136 means a V16Q const in mem; even when we can perform the
35137 shift with one insn set the cost to prefer paddb. */
35139 {
35144 }
35146 }
35150 }
35151 else
35153 }
35155 {
35157 {
35160 else
35162 }
35163 else
35164 {
35167 else
35169 }
35170 }
35171 else
35172 {
35177 {
35178 /* Return the cost after shift-and truncation. */
35181 }
35182 else
35184 }
35188 {
35189 rtx sub;
35194 /* ??? SSE scalar/vector cost should be used here. */
35195 /* ??? Bald assumption that fma has the same cost as fmul. */
35199 /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
35210 }
35214 {
35215 /* ??? SSE scalar cost should be used here. */
35218 }
35220 {
35223 }
35225 {
35226 /* ??? SSE vector cost should be used here. */
35229 }
35231 {
35232 /* V*QImode is emulated with 7-13 insns. */
35234 {
35241 }
35242 /* V*DImode is emulated with 5-8 insns. */
35244 {
35247 else
35249 }
35250 /* Without sse4.1, we don't have PMULLD; it's emulated with 7
35251 insns, including two PMULUDQ. */
35254 else
35257 }
35258 else
35259 {
35264 {
35267 nbits++;
35268 }
35269 else
35270 /* This is arbitrary. */
35273 /* Compute costs correctly for widening multiplication. */
35277 {
35284 {
35288 else
35290 }
35294 }
35302 }
35309 /* ??? SSE cost should be used here. */
35314 /* ??? SSE vector cost should be used here. */
35316 else
35323 {
35328 {
35331 {
35339 }
35340 }
35343 {
35346 {
35352 }
35353 }
35355 {
35363 }
35364 }
35365 /* FALLTHRU */
35369 {
35370 /* ??? SSE cost should be used here. */
35373 }
35375 {
35378 }
35380 {
35381 /* ??? SSE vector cost should be used here. */
35384 }
35385 /* FALLTHRU */
35392 {
35399 }
35400 /* FALLTHRU */
35404 {
35405 /* ??? SSE cost should be used here. */
35408 }
35410 {
35413 }
35415 {
35416 /* ??? SSE vector cost should be used here. */
35419 }
35420 /* FALLTHRU */
35424 {
35425 /* ??? Should be SSE vector operation cost. */
35426 /* At least for published AMD latencies, this really is the same
35427 as the latency for a simple fpu operation like fabs. */
35429 }
35432 else
35441 {
35442 /* This kind of construct is implemented using test[bwl].
35443 Treat it as if we had an AND. */
35448 }
35458 /* ??? SSE cost should be used here. */
35463 /* ??? SSE vector cost should be used here. */
35469 /* ??? SSE cost should be used here. */
35474 /* ??? SSE vector cost should be used here. */
35487 /* ??? Assume all of these vector manipulation patterns are
35488 recognizable. In which case they all pretty much have the
35489 same cost. */
35495 }
35496 }
35498 #if TARGET_MACHO
35502 /* Given a symbol name and its associated stub, write out the
35503 definition of the stub. */
35505 void
35507 {
35512 /* For 64-bit we shouldn't get here. */
35515 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
35532 else
35539 {
35541 }
35543 {
35544 /* PIC stub. */
35545 /* 25-byte PIC stub using "CALL get_pc_thunk". */
35551 }
35552 else
35555 /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
35556 it needs no stub-binding-helper. */
35563 {
35566 }
35567 else
35572 /* N.B. Keep the correspondence of these
35573 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
35574 old-pic/new-pic/non-pic stubs; altering this will break
35575 compatibility with existing dylibs. */
35577 {
35578 /* 25-byte PIC stub using "CALL get_pc_thunk". */
35580 }
35581 else
35582 /* 16-byte -mdynamic-no-pic stub. */
35588 }
35591 /* Order the registers for register allocator. */
35593 void
35595 {
35599 /* First allocate the local general purpose registers. */
35604 /* Global general purpose registers. */
35609 /* x87 registers come first in case we are doing FP math
35610 using them. */
35615 /* SSE registers. */
35621 /* Extended REX SSE registers. */
35625 /* Mask register. */
35629 /* MPX bound registers. */
35633 /* x87 registers. */
35641 /* Initialize the rest of array as we do not allocate some registers
35642 at all. */
35645 }
35647 /* Handle a "callee_pop_aggregate_return" attribute; arguments as
35648 in struct attribute_spec handler. */
35649 static tree
35651 tree args,
35654 {
35659 {
35661 name);
35664 }
35666 {
35668 name);
35671 }
35673 {
35674 tree cst;
35678 {
35681 name);
35683 }
35686 {
35689 name);
35691 }
35694 }
35697 }
35699 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
35700 struct attribute_spec.handler. */
35701 static tree
35703 tree args ATTRIBUTE_UNUSED,
35705 {
35710 {
35712 name);
35715 }
35717 /* Can combine regparm with all attributes but fastcall. */
35719 {
35721 {
35723 }
35726 }
35728 {
35730 {
35732 }
35735 }
35738 }
35740 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
35741 struct attribute_spec.handler. */
35742 static tree
35744 tree args ATTRIBUTE_UNUSED,
35746 {
35749 {
35752 }
35753 else
35757 {
35759 name);
35761 }
35767 {
35769 name);
35771 }
35774 }
35776 static tree
35778 tree args ATTRIBUTE_UNUSED,
35780 {
35782 {
35784 name);
35786 }
35788 }
35790 static bool
35792 {
35796 }
35798 /* Returns an expression indicating where the this parameter is
35799 located on entry to the FUNCTION. */
35801 static rtx
35803 {
35809 {
35814 else
35817 }
35822 {
35829 {
35834 }
35835 else
35836 {
35839 {
35845 }
35846 }
35848 }
35852 }
35854 /* Determine whether x86_output_mi_thunk can succeed. */
35856 static bool
35858 HOST_WIDE_INT delta ATTRIBUTE_UNUSED,
35860 {
35861 /* 64-bit can handle anything. */
35865 /* For 32-bit, everything's fine if we have one free register. */
35869 /* Need a free register for vcall_offset. */
35873 /* Need a free register for GOT references. */
35877 /* Otherwise ok. */
35879 }
35881 /* Output the assembler code for a thunk function. THUNK_DECL is the
35882 declaration for the thunk function itself, FUNCTION is the decl for
35883 the target function. DELTA is an immediate constant offset to be
35884 added to THIS. If VCALL_OFFSET is nonzero, the word at
35885 *(*this + vcall_offset) should be added to THIS. */
35887 static void
35891 {
35898 else
35899 {
35905 else
35907 }
35911 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
35912 pull it in now and let DELTA benefit. */
35916 {
35917 /* Put the this parameter into %eax. */
35920 }
35921 else
35924 /* Adjust the this parameter by a fixed constant. */
35926 {
35931 {
35933 {
35937 }
35938 }
35941 }
35943 /* Adjust the this parameter by a value stored in the vtable. */
35945 {
35955 /* Adjust the this parameter. */
35959 {
35963 }
35970 vcall_mem));
35971 else
35973 }
35975 /* If necessary, drop THIS back to its stack slot. */
35981 {
35984 ;
35985 else
35986 {
35990 }
35991 }
35992 else
35993 {
35995 ;
35996 #if TARGET_MACHO
35998 {
36001 }
36003 else
36004 {
36011 }
36012 }
36014 /* Our sibling call patterns do not allow memories, because we have no
36015 predicate that can distinguish between frame and non-frame memory.
36016 For our purposes here, we can get away with (ab)using a jump pattern,
36017 because we're going to do no optimization. */
36020 else
36021 {
36027 {
36033 }
36039 }
36042 /* Emit just enough of rest_of_compilation to get the insns emitted.
36043 Note that use_thunk calls assemble_start_function et al. */
36049 }
36051 static void
36053 {
36055 #if TARGET_MACHO
36057 #endif
36064 }
36066 int
36068 {
36080 }
36082 /* Output assembler code to FILE to increment profiler label # LABELNO
36083 for profiling a function entry. */
36084 void
36086 {
36091 {
36092 #ifndef NO_PROFILE_COUNTERS
36094 #endif
36098 else
36100 }
36102 {
36103 #ifndef NO_PROFILE_COUNTERS
36106 #endif
36108 }
36109 else
36110 {
36111 #ifndef NO_PROFILE_COUNTERS
36114 #endif
36116 }
36117 }
36119 /* We don't have exact information about the insn sizes, but we may assume
36120 quite safely that we are informed about all 1 byte insns and memory
36121 address sizes. This is enough to eliminate unnecessary padding in
36122 99% of cases. */
36124 static int
36126 {
36132 /* Discard alignments we've emit and jump instructions. */
36137 /* Important case - calls are always 5 bytes.
36138 It is common to have many calls in the row. */
36147 /* For normal instructions we rely on get_attr_length being exact,
36148 with a few exceptions. */
36150 {
36154 {
36164 /* Otherwise trust get_attr_length. */
36166 }
36171 }
36174 else
36176 }
36178 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
36180 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
36181 window. */
36183 static void
36185 {
36190 /* Look for all minimal intervals of instructions containing 4 jumps.
36191 The intervals are bounded by START and INSN. NBYTES is the total
36192 size of instructions in the interval including INSN and not including
36193 START. When the NBYTES is smaller than 16 bytes, it is possible
36194 that the end of START and INSN ends up in the same 16byte page.
36196 The smallest offset in the page INSN can start is the case where START
36197 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
36198 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
36199 */
36201 {
36205 {
36211 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
36212 already in the current 16 byte page, because otherwise
36213 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
36214 bytes to reach 16 byte boundary. */
36222 {
36224 {
36228 else
36231 }
36232 }
36234 }
36242 njumps++;
36243 else
36247 {
36251 else
36254 }
36261 {
36268 }
36269 }
36270 }
36271 #endif
36273 /* AMD Athlon works faster
36274 when RET is not destination of conditional jump or directly preceded
36275 by other jump instruction. We avoid the penalty by inserting NOP just
36276 before the RET instructions in such cases. */
36277 static void
36279 {
36280 edge e;
36281 edge_iterator ei;
36284 {
36287 rtx prev;
36297 {
36298 edge e;
36299 edge_iterator ei;
36304 {
36307 }
36308 }
36310 {
36316 /* Empty functions get branch mispredict even when
36317 the jump destination is not visible to us. */
36320 }
36322 {
36325 }
36326 }
36327 }
36329 /* Count the minimum number of instructions in BB. Return 4 if the
36330 number of instructions >= 4. */
36332 static int
36334 {
36335 rtx insn;
36338 /* Count number of instructions in this block. Return 4 if the number
36339 of instructions >= 4. */
36341 {
36342 /* Only happen in exit blocks. */
36350 {
36351 insn_count++;
36354 }
36355 }
36358 }
36361 /* Count the minimum number of instructions in code path in BB.
36362 Return 4 if the number of instructions >= 4. */
36364 static int
36366 {
36367 edge e;
36368 edge_iterator ei;
36371 /* Only bother counting instructions along paths with no
36372 more than 2 basic blocks between entry and exit. Given
36373 that BB has an edge to exit, determine if a predecessor
36374 of BB has an edge from entry. If so, compute the number
36375 of instructions in the predecessor block. If there
36376 happen to be multiple such blocks, compute the minimum. */
36379 {
36380 edge prev_e;
36381 edge_iterator prev_ei;
36384 {
36387 }
36389 {
36391 {
36396 }
36397 }
36398 }
36404 }
36406 /* Pad short function to 4 instructions. */
36408 static void
36410 {
36411 edge e;
36412 edge_iterator ei;
36415 {
36418 {
36421 /* Pad short function. */
36423 {
36426 /* Find epilogue. */
36435 /* Two NOPs count as one instruction. */
36438 }
36439 }
36440 }
36441 }
36443 /* Fix up a Windows system unwinder issue. If an EH region falls through into
36444 the epilogue, the Windows system unwinder will apply epilogue logic and
36445 produce incorrect offsets. This can be avoided by adding a nop between
36446 the last insn that can throw and the first insn of the epilogue. */
36448 static void
36450 {
36451 edge e;
36452 edge_iterator ei;
36455 {
36458 /* Find the beginning of the epilogue. */
36465 /* We only care about preceding insns that can throw. */
36470 /* Do not separate calls from their debug information. */
36476 else
36480 }
36481 }
36483 /* Implement machine specific optimizations. We implement padding of returns
36484 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
36485 static void
36487 {
36488 /* We are freeing block_for_insn in the toplev to keep compatibility
36489 with old MDEP_REORGS that are not CFG based. Recompute it now. */
36496 {
36501 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
36504 #endif
36505 }
36506 }
36508 /* Return nonzero when QImode register that must be represented via REX prefix
36509 is used. */
36510 bool
36512 {
36520 }
36522 /* Return nonzero when P points to register encoded via REX prefix.
36523 Called via for_each_rtx. */
36524 static int
36526 {
36532 }
36534 /* Return true when INSN mentions register that must be encoded using REX
36535 prefix. */
36536 bool
36538 {
36541 }
36543 /* If profitable, negate (without causing overflow) integer constant
36544 of mode MODE at location LOC. Return true in this case. */
36545 bool
36547 {
36548 HOST_WIDE_INT val;
36554 {
36556 /* DImode x86_64 constants must fit in 32 bits. */
36569 }
36571 /* Avoid overflows. */
36577 /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
36578 Exceptions: -128 encodes smaller than 128, so swap sign and op. */
36581 {
36584 }
36587 }
36589 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
36590 optabs would emit if we didn't have TFmode patterns. */
36592 void
36594 {
36628 }
36629 \f
36630 /* AVX512F does support 64-byte integer vector operations,
36631 thus the longest vector we are faced with is V64QImode. */
36632 #define MAX_VECT_LEN 64
36634 struct expand_vec_perm_d
36635 {
36642 };
36648 /* Get a vector mode of the same size as the original but with elements
36649 twice as wide. This is only guaranteed to apply to integral vectors. */
36653 {
36654 /* ??? Rely on the ordering that genmodes.c gives to vectors. */
36659 }
36661 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
36662 with all elements equal to VAR. Return true if successful. */
36664 static bool
36667 {
36671 {
36676 /* FALLTHRU */
36686 {
36689 /* First attempt to recognize VAL as-is. */
36693 {
36694 rtx seq;
36695 /* If that fails, force VAL into a register. */
36706 }
36707 }
36714 {
36715 rtx x;
36722 }
36732 {
36736 permute:
36744 /* Extend to SImode using a paradoxical SUBREG. */
36748 /* Insert the SImode value as low element of a V4SImode vector. */
36757 }
36765 widen:
36766 /* Replicate the value once into the next wider mode and recurse. */
36767 {
36769 rtx x;
36786 }
36790 {
36799 }
36804 }
36805 }
36807 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
36808 whose ONE_VAR element is VAR, and other elements are zero. Return true
36809 if successful. */
36811 static bool
36814 {
36816 rtx new_target;
36821 {
36823 /* For SSE4.1, we normally use vector set. But if the second
36824 element is zero and inter-unit moves are OK, we use movq
36825 instead. */
36827 && !(TARGET_INTER_UNIT_MOVES_TO_VEC
36849 /* Use ix86_expand_vector_set in 64bit mode only. */
36854 }
36857 {
36862 }
36865 {
36870 /* FALLTHRU */
36885 else
36892 {
36893 /* We need to shuffle the value to the correct position, so
36894 create a new pseudo to store the intermediate result. */
36896 /* With SSE2, we can use the integer shuffle insns. */
36898 {
36900 const1_rtx,
36907 }
36909 /* Otherwise convert the intermediate result to V4SFmode and
36910 use the SSE1 shuffle instructions. */
36912 {
36915 }
36916 else
36920 const1_rtx,
36929 }
36944 widen:
36948 /* Zero extend the variable element to SImode and recurse. */
36961 }
36962 }
36964 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
36965 consisting of the values in VALS. It is known that all elements
36966 except ONE_VAR are constants. Return true if successful. */
36968 static bool
36971 {
36981 {
36986 /* For the two element vectors, it's just as easy to use
36987 the general case. */
36991 /* Use ix86_expand_vector_set in 64bit mode only. */
37013 widen:
37014 /* There's no way to set one QImode entry easily. Combine
37015 the variable value with its adjacent constant value, and
37016 promote to an HImode set. */
37019 {
37024 }
37025 else
37026 {
37029 }
37043 }
37048 }
37050 /* A subroutine of ix86_expand_vector_init_general. Use vector
37051 concatenate to handle the most general case: all values variable,
37052 and none identical. */
37054 static void
37057 {
37060 rtvec v;
37064 {
37067 {
37100 }
37113 {
37128 }
37133 {
37144 }
37147 half:
37148 /* FIXME: We process inputs backward to help RA. PR 36222. */
37152 {
37157 }
37161 {
37164 {
37168 }
37171 }
37172 else
37178 }
37179 }
37181 /* A subroutine of ix86_expand_vector_init_general. Use vector
37182 interleave to handle the most general case: all values variable,
37183 and none identical. */
37185 static void
37188 {
37197 {
37218 }
37221 {
37222 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
37226 /* Insert the SImode value as low element of V4SImode vector. */
37230 op0),
37232 const1_rtx);
37235 /* Cast the V4SImode vector back to a vector in orignal mode. */
37239 /* Load even elements into the second position. */
37243 const1_rtx));
37245 /* Cast vector to FIRST_IMODE vector. */
37248 }
37250 /* Interleave low FIRST_IMODE vectors. */
37252 {
37256 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
37259 }
37261 /* Interleave low SECOND_IMODE vectors. */
37263 {
37266 {
37271 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
37272 vector. */
37275 }
37278 /* FALLTHRU */
37285 /* Cast the SECOND_IMODE vector back to a vector on original
37286 mode. */
37293 }
37294 }
37296 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
37297 all values variable, and none identical. */
37299 static void
37302 {
37308 {
37313 /* FALLTHRU */
37337 half:
37354 /* FALLTHRU */
37360 /* Don't use ix86_expand_vector_init_interleave if we can't
37361 move from GPR to SSE register directly. */
37377 }
37379 {
37391 {
37395 {
37401 else
37402 {
37407 }
37408 }
37411 }
37416 {
37422 }
37424 {
37430 }
37431 else
37433 }
37434 }
37436 /* Initialize vector TARGET via VALS. Suppress the use of MMX
37437 instructions unless MMX_OK is true. */
37439 void
37441 {
37448 rtx x;
37451 {
37461 }
37463 /* Constants are best loaded from the constant pool. */
37465 {
37468 }
37470 /* If all values are identical, broadcast the value. */
37476 /* Values where only one field is non-constant are best loaded from
37477 the pool and overwritten via move later. */
37479 {
37483 one_var))
37488 }
37491 }
37493 void
37495 {
37500 rtx tmp;
37502 = {
37509 };
37511 = {
37518 };
37522 {
37526 {
37531 else
37535 }
37547 else
37553 {
37556 /* For the two element vectors, we implement a VEC_CONCAT with
37557 the extraction of the other element. */
37564 else
37569 }
37578 {
37584 /* tmp = target = A B C D */
37586 /* target = A A B B */
37588 /* target = X A B B */
37590 /* target = A X C D */
37597 /* tmp = target = A B C D */
37599 /* tmp = X B C D */
37601 /* target = A B X D */
37608 /* tmp = target = A B C D */
37610 /* tmp = X B C D */
37612 /* target = A B X D */
37620 }
37628 /* Element 0 handled by vec_merge below. */
37630 {
37633 }
37636 {
37637 /* With SSE2, use integer shuffles to swap element 0 and ELT,
37638 store into element 0, then shuffle them back. */
37655 }
37656 else
37657 {
37658 /* For SSE1, we have to reuse the V4SF code. */
37662 }
37715 half:
37716 /* Compute offset. */
37722 /* Extract the half. */
37726 /* Put val in tmp at elt. */
37729 /* Put it back. */
37735 }
37738 {
37742 }
37743 else
37744 {
37753 }
37754 }
37756 void
37758 {
37762 rtx tmp;
37765 {
37770 /* FALLTHRU */
37783 {
37803 }
37815 {
37817 {
37837 }
37841 }
37842 else
37843 {
37844 /* For SSE1, we have to reuse the V4SF code. */
37848 }
37864 {
37868 else
37872 }
37877 {
37881 else
37885 }
37890 {
37894 else
37898 }
37903 {
37907 else
37911 }
37916 {
37920 else
37924 }
37929 {
37933 else
37937 }
37941 /* ??? Could extract the appropriate HImode element and shift. */
37944 }
37947 {
37951 /* Let the rtl optimizers know about the zero extension performed. */
37953 {
37956 }
37959 }
37960 else
37961 {
37968 }
37969 }
37971 /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
37972 to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
37973 The upper bits of DEST are undefined, though they shouldn't cause
37974 exceptions (some bits from src or all zeros are ok). */
37976 static void
37978 {
37981 {
37985 else
38003 else
38010 else
38018 {
38023 const1_rtx);
38024 }
38025 else
38026 {
38030 }
38034 }
38038 }
38040 /* Expand a vector reduction. FN is the binary pattern to reduce;
38041 DEST is the destination; IN is the input vector. */
38043 void
38045 {
38050 /* SSE4 has a special instruction for V8HImode UMIN reduction. */
38054 {
38057 }
38062 {
38067 else
38071 }
38072 }
38073 \f
38074 /* Target hook for scalar_mode_supported_p. */
38075 static bool
38077 {
38082 else
38084 }
38086 /* Implements target hook vector_mode_supported_p. */
38087 static bool
38089 {
38101 }
38103 /* Target hook for c_mode_for_suffix. */
38104 static enum machine_mode
38106 {
38113 }
38115 /* Worker function for TARGET_MD_ASM_CLOBBERS.
38117 We do this in the new i386 backend to maintain source compatibility
38118 with the old cc0-based compiler. */
38120 static tree
38122 tree inputs ATTRIBUTE_UNUSED,
38123 tree clobbers)
38124 {
38126 clobbers);
38128 clobbers);
38130 }
38132 /* Implements target vector targetm.asm.encode_section_info. */
38134 static void ATTRIBUTE_UNUSED
38136 {
38143 }
38145 /* Worker function for REVERSE_CONDITION. */
38147 enum rtx_code
38149 {
38153 }
38155 /* Output code to perform an x87 FP register move, from OPERANDS[1]
38156 to OPERANDS[0]. */
38160 {
38162 {
38165 {
38169 }
38173 }
38175 {
38179 else
38180 {
38181 /* There is no non-popping store to memory for XFmode.
38182 So if we need one, follow the store with a load. */
38185 else
38187 }
38188 }
38189 else
38191 }
38193 /* Output code to perform a conditional jump to LABEL, if C2 flag in
38194 FP status register is set. */
38196 void
38198 {
38200 rtx temp;
38205 {
38210 }
38211 else
38212 {
38217 }
38221 pc_rtx);
38226 }
38228 /* Output code to perform a log1p XFmode calculation. */
38231 {
38237 rtx test;
38243 XFmode));
38257 }
38259 /* Emit code for round calculation. */
38261 {
38268 rtx insn;
38273 {
38285 }
38288 {
38309 }
38317 /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
38319 /* scratch = fxam(op1) */
38322 UNSPEC_FXAM)));
38323 /* e1 = fabs(op1) */
38326 /* e2 = e1 + 0.5 */
38331 /* res = floor(e2) */
38333 {
38338 }
38339 else
38343 {
38346 {
38353 UNSPEC_TRUNC_NOOP)));
38354 }
38370 }
38372 /* flags = signbit(a) */
38375 /* if (flags) then res = -res */
38379 pc_rtx);
38390 }
38392 /* Output code to perform a Newton-Rhapson approximation of a single precision
38393 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
38396 {
38404 /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
38408 /* x0 = rcp(b) estimate */
38411 UNSPEC_RCP)));
38412 /* e0 = x0 * b */
38416 /* e0 = x0 * e0 */
38420 /* e1 = x0 + x0 */
38424 /* x1 = e1 - e0 */
38428 /* res = a * x1 */
38431 }
38433 /* Output code to perform a Newton-Rhapson approximation of a
38434 single precision floating point [reciprocal] square root. */
38438 {
38440 REAL_VALUE_TYPE r;
38455 {
38458 }
38460 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
38461 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
38465 /* x0 = rsqrt(a) estimate */
38468 UNSPEC_RSQRT)));
38470 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
38472 {
38484 }
38486 /* e0 = x0 * a */
38489 /* e1 = e0 * x0 */
38493 /* e2 = e1 - 3. */
38500 /* e3 = -.5 * x0 */
38503 else
38504 /* e3 = -.5 * e0 */
38507 /* ret = e2 * e3 */
38510 }
38512 #ifdef TARGET_SOLARIS
38513 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
38515 static void
38517 tree decl)
38518 {
38519 /* With Binutils 2.15, the "@unwind" marker must be specified on
38520 every occurrence of the ".eh_frame" section, not just the first
38521 one. */
38524 {
38528 }
38530 #ifndef USE_GAS
38532 {
38535 }
38536 #endif
38539 }
38542 /* Return the mangling of TYPE if it is an extended fundamental type. */
38546 {
38554 {
38556 /* __float128 is "g". */
38559 /* "long double" or __float80 is "e". */
38563 }
38564 }
38566 /* For 32-bit code we can save PIC register setup by using
38567 __stack_chk_fail_local hidden function instead of calling
38568 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
38569 register, so it is better to call __stack_chk_fail directly. */
38571 static tree ATTRIBUTE_UNUSED
38573 {
38574 return TARGET_64BIT
38577 }
38579 /* Select a format to encode pointers in exception handling data. CODE
38580 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
38581 true if the symbol may be affected by dynamic relocations.
38583 ??? All x86 object file formats are capable of representing this.
38584 After all, the relocation needed is the same as for the call insn.
38585 Whether or not a particular assembler allows us to enter such, I
38586 guess we'll have to see. */
38587 int
38589 {
38591 {
38598 }
38603 }
38604 \f
38605 /* Expand copysign from SIGN to the positive value ABS_VALUE
38606 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
38607 the sign-bit. */
38608 static void
38610 {
38614 {
38621 else
38626 {
38627 /* We need to generate a scalar mode mask in this case. */
38632 }
38633 }
38634 else
38640 }
38642 /* Expand fabs (OP0) and return a new rtx that holds the result. The
38643 mask for masking out the sign-bit is stored in *SMASK, if that is
38644 non-null. */
38645 static rtx
38647 {
38656 else
38660 {
38661 /* We need to generate a scalar mode mask in this case. */
38666 }
38674 }
38676 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
38677 swapping the operands if SWAP_OPERANDS is true. The expanded
38678 code is a forward jump to a newly created label in case the
38679 comparison is true. The generated label rtx is returned. */
38680 static rtx
38683 {
38688 {
38692 }
38705 }
38707 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
38708 using comparison code CODE. Operands are swapped for the comparison if
38709 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
38710 static rtx
38713 {
38719 {
38723 }
38730 }
38732 /* Generate and return a rtx of mode MODE for 2**n where n is the number
38733 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
38734 static rtx
38736 {
38737 REAL_VALUE_TYPE TWO52r;
38738 rtx TWO52;
38745 }
38747 /* Expand SSE sequence for computing lround from OP1 storing
38748 into OP0. */
38749 void
38751 {
38752 /* C code for the stuff we're doing below:
38753 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
38754 return (long)tmp;
38755 */
38759 rtx adj;
38761 /* load nextafter (0.5, 0.0) */
38766 /* adj = copysign (0.5, op1) */
38770 /* adj = op1 + adj */
38773 /* op0 = (imode)adj */
38775 }
38777 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
38778 into OPERAND0. */
38779 void
38781 {
38782 /* C code for the stuff we're doing below (for do_floor):
38783 xi = (long)op1;
38784 xi -= (double)xi > op1 ? 1 : 0;
38785 return xi;
38786 */
38791 /* reg = (long)op1 */
38795 /* freg = (double)reg */
38799 /* ireg = (freg > op1) ? ireg - 1 : ireg */
38810 }
38812 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
38813 result in OPERAND0. */
38814 void
38816 {
38817 /* C code for the stuff we're doing below:
38818 xa = fabs (operand1);
38819 if (!isless (xa, 2**52))
38820 return operand1;
38821 xa = xa + 2**52 - 2**52;
38822 return copysign (xa, operand1);
38823 */
38830 /* xa = abs (operand1) */
38833 /* if (!isless (xa, TWO52)) goto label; */
38846 }
38848 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
38849 into OPERAND0. */
38850 void
38852 {
38853 /* C code for the stuff we expand below.
38854 double xa = fabs (x), x2;
38855 if (!isless (xa, TWO52))
38856 return x;
38857 xa = xa + TWO52 - TWO52;
38858 x2 = copysign (xa, x);
38859 Compensate. Floor:
38860 if (x2 > x)
38861 x2 -= 1;
38862 Compensate. Ceil:
38863 if (x2 < x)
38864 x2 -= -1;
38865 return x2;
38866 */
38872 /* Temporary for holding the result, initialized to the input
38873 operand to ease control flow. */
38877 /* xa = abs (operand1) */
38880 /* if (!isless (xa, TWO52)) goto label; */
38883 /* xa = xa + TWO52 - TWO52; */
38887 /* xa = copysign (xa, operand1) */
38890 /* generate 1.0 or -1.0 */
38895 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
38899 /* We always need to subtract here to preserve signed zero. */
38908 }
38910 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
38911 into OPERAND0. */
38912 void
38914 {
38915 /* C code for the stuff we expand below.
38916 double xa = fabs (x), x2;
38917 if (!isless (xa, TWO52))
38918 return x;
38919 x2 = (double)(long)x;
38920 Compensate. Floor:
38921 if (x2 > x)
38922 x2 -= 1;
38923 Compensate. Ceil:
38924 if (x2 < x)
38925 x2 += 1;
38926 if (HONOR_SIGNED_ZEROS (mode))
38927 return copysign (x2, x);
38928 return x2;
38929 */
38935 /* Temporary for holding the result, initialized to the input
38936 operand to ease control flow. */
38940 /* xa = abs (operand1) */
38943 /* if (!isless (xa, TWO52)) goto label; */
38946 /* xa = (double)(long)x */
38951 /* generate 1.0 */
38954 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
38969 }
38971 /* Expand SSE sequence for computing round from OPERAND1 storing
38972 into OPERAND0. Sequence that works without relying on DImode truncation
38973 via cvttsd2siq that is only available on 64bit targets. */
38974 void
38976 {
38977 /* C code for the stuff we expand below.
38978 double xa = fabs (x), xa2, x2;
38979 if (!isless (xa, TWO52))
38980 return x;
38981 Using the absolute value and copying back sign makes
38982 -0.0 -> -0.0 correct.
38983 xa2 = xa + TWO52 - TWO52;
38984 Compensate.
38985 dxa = xa2 - xa;
38986 if (dxa <= -0.5)
38987 xa2 += 1;
38988 else if (dxa > 0.5)
38989 xa2 -= 1;
38990 x2 = copysign (xa2, x);
38991 return x2;
38992 */
38998 /* Temporary for holding the result, initialized to the input
38999 operand to ease control flow. */
39003 /* xa = abs (operand1) */
39006 /* if (!isless (xa, TWO52)) goto label; */
39009 /* xa2 = xa + TWO52 - TWO52; */
39013 /* dxa = xa2 - xa; */
39016 /* generate 0.5, 1.0 and -0.5 */
39022 /* Compensate. */
39024 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
39029 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
39035 /* res = copysign (xa2, operand1) */
39042 }
39044 /* Expand SSE sequence for computing trunc from OPERAND1 storing
39045 into OPERAND0. */
39046 void
39048 {
39049 /* C code for SSE variant we expand below.
39050 double xa = fabs (x), x2;
39051 if (!isless (xa, TWO52))
39052 return x;
39053 x2 = (double)(long)x;
39054 if (HONOR_SIGNED_ZEROS (mode))
39055 return copysign (x2, x);
39056 return x2;
39057 */
39063 /* Temporary for holding the result, initialized to the input
39064 operand to ease control flow. */
39068 /* xa = abs (operand1) */
39071 /* if (!isless (xa, TWO52)) goto label; */
39074 /* x = (double)(long)x */
39086 }
39088 /* Expand SSE sequence for computing trunc from OPERAND1 storing
39089 into OPERAND0. */
39090 void
39092 {
39096 /* C code for SSE variant we expand below.
39097 double xa = fabs (x), x2;
39098 if (!isless (xa, TWO52))
39099 return x;
39100 xa2 = xa + TWO52 - TWO52;
39101 Compensate:
39102 if (xa2 > xa)
39103 xa2 -= 1.0;
39104 x2 = copysign (xa2, x);
39105 return x2;
39106 */
39110 /* Temporary for holding the result, initialized to the input
39111 operand to ease control flow. */
39115 /* xa = abs (operand1) */
39118 /* if (!isless (xa, TWO52)) goto label; */
39121 /* res = xa + TWO52 - TWO52; */
39126 /* generate 1.0 */
39129 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
39137 /* res = copysign (res, operand1) */
39144 }
39146 /* Expand SSE sequence for computing round from OPERAND1 storing
39147 into OPERAND0. */
39148 void
39150 {
39151 /* C code for the stuff we're doing below:
39152 double xa = fabs (x);
39153 if (!isless (xa, TWO52))
39154 return x;
39155 xa = (double)(long)(xa + nextafter (0.5, 0.0));
39156 return copysign (xa, x);
39157 */
39163 /* Temporary for holding the result, initialized to the input
39164 operand to ease control flow. */
39172 /* load nextafter (0.5, 0.0) */
39177 /* xa = xa + 0.5 */
39181 /* xa = (double)(int64_t)xa */
39186 /* res = copysign (xa, operand1) */
39193 }
39195 /* Expand SSE sequence for computing round
39196 from OP1 storing into OP0 using sse4 round insn. */
39197 void
39199 {
39208 {
39219 }
39221 /* round (a) = trunc (a + copysign (0.5, a)) */
39223 /* load nextafter (0.5, 0.0) */
39229 /* e1 = copysign (0.5, op1) */
39233 /* e2 = op1 + e1 */
39236 /* res = trunc (e2) */
39241 }
39242 \f
39244 /* Table of valid machine attributes. */
39246 {
39247 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
39248 affects_type_identity } */
39249 /* Stdcall attribute says callee is responsible for popping arguments
39250 if they are not variable. */
39253 /* Fastcall attribute says callee is responsible for popping arguments
39254 if they are not variable. */
39257 /* Thiscall attribute says callee is responsible for popping arguments
39258 if they are not variable. */
39261 /* Cdecl attribute says the callee is a normal C declaration */
39264 /* Regparm attribute specifies how many integer arguments are to be
39265 passed in registers. */
39268 /* Sseregparm attribute says we are using x86_64 calling conventions
39269 for FP arguments. */
39272 /* The transactional memory builtins are implicitly regparm or fastcall
39273 depending on the ABI. Override the generic do-nothing attribute that
39274 these builtins were declared with. */
39277 /* force_align_arg_pointer says this function realigns the stack at entry. */
39280 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
39285 #endif
39290 #ifdef SUBTARGET_ATTRIBUTE_TABLE
39291 SUBTARGET_ATTRIBUTE_TABLE,
39292 #endif
39293 /* ms_abi and sysv_abi calling convention function attributes. */
39300 /* End element. */
39302 };
39304 /* Implement targetm.vectorize.builtin_vectorization_cost. */
39305 static int
39307 tree vectype,
39309 {
39313 {
39358 }
39359 }
39361 /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
39362 insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
39363 insn every time. */
39367 /* Initialize vselect_insn. */
39369 static void
39371 {
39373 rtx x;
39384 }
39386 /* Construct (set target (vec_select op0 (parallel perm))) and
39387 return true if that's a valid instruction in the active ISA. */
39389 static bool
39392 {
39418 }
39420 /* Similar, but generate a vec_concat from op0 and op1 as well. */
39422 static bool
39426 {
39428 rtx x;
39443 }
39445 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39446 in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
39448 static bool
39450 {
39459 ;
39461 ;
39463 ;
39464 else
39467 /* This is a blend, not a permute. Elements must stay in their
39468 respective lanes. */
39470 {
39474 }
39479 /* ??? Without SSE4.1, we could implement this with and/andn/or. This
39480 decision should be extracted elsewhere, so that we only try that
39481 sequence once all budget==3 options have been tried. */
39488 {
39512 /* See if bytes move in pairs so we can use pblendw with
39513 an immediate argument, rather than pblendvb with a vector
39514 argument. */
39517 {
39518 use_pblendvb:
39522 finish_pblendvb:
39528 else
39533 }
39538 /* FALLTHRU */
39540 do_subreg:
39547 /* See if bytes move in pairs. If not, vpblendvb must be used. */
39551 /* See if bytes move in quadruplets. If yes, vpblendd
39552 with immediate can be used. */
39557 {
39558 /* See if bytes move the same in both lanes. If yes,
39559 vpblendw with immediate can be used. */
39564 /* Use vpblendw. */
39569 }
39571 /* Use vpblendd. */
39578 /* See if words move in pairs. If yes, vpblendd can be used. */
39583 {
39584 /* See if words move the same in both lanes. If not,
39585 vpblendvb must be used. */
39588 {
39589 /* Use vpblendvb. */
39599 }
39601 /* Use vpblendw. */
39605 }
39607 /* Use vpblendd. */
39614 /* Use vpblendd. */
39622 }
39624 /* This matches five different patterns with the different modes. */
39632 }
39634 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39635 in terms of the variable form of vpermilps.
39637 Note that we will have already failed the immediate input vpermilps,
39638 which requires that the high and low part shuffle be identical; the
39639 variable form doesn't require that. */
39641 static bool
39643 {
39650 /* We can only permute within the 128-bit lane. */
39652 {
39656 }
39662 {
39665 /* Within each 128-bit lane, the elements of op0 are numbered
39666 from 0 and the elements of op1 are numbered from 4. */
39673 }
39680 }
39682 /* Return true if permutation D can be performed as VMODE permutation
39683 instead. */
39685 static bool
39687 {
39702 else
39708 }
39710 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39711 in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
39713 static bool
39715 {
39724 {
39726 {
39729 {
39733 /* Use vperm2i128 insn. The pattern uses
39734 V4DImode instead of V2TImode. */
39747 }
39749 }
39750 }
39751 else
39752 {
39754 {
39757 }
39759 {
39763 /* V4DImode should be already handled through
39764 expand_vselect by vpermq instruction. */
39771 {
39772 /* First see if vpermq can be used for
39773 V8SImode/V16HImode/V32QImode. */
39775 {
39783 {
39787 }
39789 }
39791 /* Next see if vpermd can be used. */
39794 }
39795 /* Or if vpermps can be used. */
39800 {
39801 /* vpshufb only works intra lanes, it is not
39802 possible to shuffle bytes in between the lanes. */
39806 }
39807 }
39808 else
39810 }
39818 else
39819 {
39825 else
39829 {
39833 }
39834 }
39845 {
39852 else
39854 }
39855 else
39856 {
39859 }
39864 }
39866 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to instantiate D
39867 in a single instruction. */
39869 static bool
39871 {
39875 /* Check plain VEC_SELECT first, because AVX has instructions that could
39876 match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
39877 input where SEL+CONCAT may not. */
39879 {
39885 {
39891 }
39894 {
39898 }
39900 {
39901 /* Use vpbroadcast{b,w,d}. */
39904 {
39923 /* For other modes prefer other shuffles this function creates. */
39925 }
39927 {
39931 }
39932 }
39937 /* There are plenty of patterns in sse.md that are written for
39938 SEL+CONCAT and are not replicated for a single op. Perhaps
39939 that should be changed, to avoid the nastiness here. */
39941 /* Recognize interleave style patterns, which means incrementing
39942 every other permutation operand. */
39944 {
39947 }
39952 /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
39954 {
39956 {
39961 }
39966 }
39967 }
39969 /* Finally, try the fully general two operand permute. */
39974 /* Recognize interleave style patterns with reversed operands. */
39976 {
39978 {
39982 else
39985 }
39990 }
39992 /* Try the SSE4.1 blend variable merge instructions. */
39996 /* Try one of the AVX vpermil variable permutations. */
40000 /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
40001 vpshufb, vpermd, vpermps or vpermq variable permutation. */
40006 }
40008 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
40009 in terms of a pair of pshuflw + pshufhw instructions. */
40011 static bool
40013 {
40021 /* The two permutations only operate in 64-bit lanes. */
40032 /* Emit the pshuflw. */
40039 /* Emit the pshufhw. */
40047 }
40049 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40050 the permutation using the SSSE3 palignr instruction. This succeeds
40051 when all of the elements in PERM fit within one vector and we merely
40052 need to shift them down so that a single vector permutation has a
40053 chance to succeed. */
40055 static bool
40057 {
40064 /* Even with AVX, palignr only operates on 128-bit vectors. */
40070 {
40076 }
40080 /* Given that we have SSSE3, we know we'll be able to implement the
40081 single operand permutation after the palignr with pshufb. */
40096 {
40101 }
40103 /* Test for the degenerate case where the alignment by itself
40104 produces the desired permutation. */
40106 {
40109 }
40115 }
40119 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40120 a two vector permutation into a single vector permutation by using
40121 an interleave operation to merge the vectors. */
40123 static bool
40125 {
40130 rtx seq;
40134 {
40137 }
40139 {
40142 /* For 32-byte modes allow even d->one_operand_p.
40143 The lack of cross-lane shuffling in some instructions
40144 might prevent a single insn shuffle. */
40147 /* If expand_vec_perm_interleave3 can expand this into
40148 a 3 insn sequence, give up and let it be expanded as
40149 3 insn sequence. While that is one insn longer,
40150 it doesn't need a memory operand and in the common
40151 case that both interleave low and high permutations
40152 with the same operands are adjacent needs 4 insns
40153 for both after CSE. */
40156 }
40157 else
40160 /* Examine from whence the elements come. */
40169 {
40172 /* Split the two input vectors into 4 halves. */
40178 /* If the elements from the low halves use interleave low, and similarly
40179 for interleave high. If the elements are from mis-matched halves, we
40180 can use shufps for V4SF/V4SI or do a DImode shuffle. */
40182 {
40183 /* punpckl* */
40185 {
40190 }
40193 }
40195 {
40196 /* punpckh* */
40198 {
40203 }
40206 }
40208 {
40209 /* shufps */
40211 {
40216 }
40218 {
40219 /* shufpd */
40224 }
40225 }
40227 {
40228 /* shufps */
40230 {
40235 }
40237 {
40238 /* shufpd */
40243 }
40244 }
40245 else
40247 }
40248 else
40249 {
40254 /* Split the two input vectors into 8 quarters. */
40260 {
40263 }
40266 {
40270 }
40272 {
40275 }
40278 {
40280 {
40281 /* Attempt to increase the likelihood that dfinal
40282 shuffle will be intra-lane. */
40286 }
40288 /* vperm2f128 or vperm2i128. */
40290 {
40295 }
40300 {
40304 {
40307 }
40308 }
40309 }
40314 {
40315 /* vpunpckl* */
40317 {
40326 }
40327 }
40330 {
40331 /* vpunpckh* */
40333 {
40342 }
40343 }
40344 else
40346 }
40348 /* Use the remapping array set up above to move the elements from their
40349 swizzled locations into their final destinations. */
40352 {
40355 /* If same_halves is true, both halves of the remapped vector are the
40356 same. Avoid cross-lane accesses if possible. */
40358 {
40361 }
40362 else
40364 }
40370 /* Test if the final remap can be done with a single insn. For V4SFmode or
40371 V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
40384 {
40387 }
40394 }
40396 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40397 a single vector cross-lane permutation into vpermq followed
40398 by any of the single insn permutations. */
40400 static bool
40402 {
40416 {
40419 }
40422 {
40427 }
40440 {
40447 }
40454 {
40459 ;
40462 else
40464 }
40473 }
40475 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to expand
40476 a vector permutation using two instructions, vperm2f128 resp.
40477 vperm2i128 followed by any single in-lane permutation. */
40479 static bool
40481 {
40495 /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
40496 immediate. For perm < 16 the second permutation uses
40497 d->op0 as first operand, for perm >= 16 it uses d->op1
40498 as first operand. The second operand is the result of
40499 vperm2[fi]128. */
40501 {
40502 /* Ignore permutations which do not move anything cross-lane. */
40504 {
40505 /* The second shuffle for e.g. V4DFmode has
40506 0123 and ABCD operands.
40507 Ignore AB23, as 23 is already in the second lane
40508 of the first operand. */
40510 /* And 01CD, as 01 is in the first lane of the first
40511 operand. */
40513 /* And 4567, as then the vperm2[fi]128 doesn't change
40514 anything on the original 4567 second operand. */
40516 }
40517 else
40518 {
40519 /* The second shuffle for e.g. V4DFmode has
40520 4567 and ABCD operands.
40521 Ignore AB67, as 67 is already in the second lane
40522 of the first operand. */
40524 /* And 45CD, as 45 is in the first lane of the first
40525 operand. */
40527 /* And 0123, as then the vperm2[fi]128 doesn't change
40528 anything on the original 0123 first operand. */
40530 }
40533 {
40539 else
40541 }
40544 {
40548 }
40549 else
40553 {
40557 /* Found a usable second shuffle. dfirst will be
40558 vperm2f128 on d->op0 and d->op1. */
40569 /* And dsecond is some single insn shuffle, taking
40570 d->op0 and result of vperm2f128 (if perm < 16) or
40571 d->op1 and result of vperm2f128 (otherwise). */
40580 }
40582 /* For one operand, the only useful vperm2f128 permutation is 0x10. */
40585 }
40588 }
40590 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40591 a two vector permutation using 2 intra-lane interleave insns
40592 and cross-lane shuffle for 32-byte vectors. */
40594 static bool
40596 {
40603 ;
40605 ;
40606 else
40621 {
40625 else
40631 else
40637 else
40643 else
40649 else
40655 else
40660 }
40664 }
40666 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement
40667 a single vector permutation using a single intra-lane vector
40668 permutation, vperm2f128 swapping the lanes and vblend* insn blending
40669 the non-swapped and swapped vectors together. */
40671 static bool
40673 {
40676 rtx seq;
40681 || TARGET_AVX2
40690 {
40697 }
40732 }
40734 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement a V4DF
40735 permutation using two vperm2f128, followed by a vshufpd insn blending
40736 the two vectors together. */
40738 static bool
40740 {
40783 }
40785 /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
40786 permutation with two pshufb insns and an ior. We should have already
40787 failed all two instruction sequences. */
40789 static bool
40791 {
40802 /* Generate two permutation masks. If the required element is within
40803 the given vector it is shuffled into the proper lane. If the required
40804 element is in the other vector, force a zero into the lane by setting
40805 bit 7 in the permutation mask. */
40808 {
40815 {
40818 }
40819 }
40843 }
40845 /* Implement arbitrary permutation of one V32QImode and V16QImode operand
40846 with two vpshufb insns, vpermq and vpor. We should have already failed
40847 all two or three instruction sequences. */
40849 static bool
40851 {
40866 /* Generate two permutation masks. If the required element is within
40867 the same lane, it is shuffled in. If the required element from the
40868 other lane, force a zero by setting bit 7 in the permutation mask.
40869 In the other mask the mask has non-negative elements if element
40870 is requested from the other lane, but also moved to the other lane,
40871 so that the result of vpshufb can have the two V2TImode halves
40872 swapped. */
40875 {
40880 {
40883 }
40884 }
40893 /* Swap the 128-byte lanes of h into hp. */
40897 const1_rtx));
40914 }
40916 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
40917 and extract-odd permutations of two V32QImode and V16QImode operand
40918 with two vpshufb insns, vpor and vpermq. We should have already
40919 failed all two or three instruction sequences. */
40921 static bool
40923 {
40942 /* Generate two permutation masks. In the first permutation mask
40943 the first quarter will contain indexes for the first half
40944 of the op0, the second quarter will contain bit 7 set, third quarter
40945 will contain indexes for the second half of the op0 and the
40946 last quarter bit 7 set. In the second permutation mask
40947 the first quarter will contain bit 7 set, the second quarter
40948 indexes for the first half of the op1, the third quarter bit 7 set
40949 and last quarter indexes for the second half of the op1.
40950 I.e. the first mask e.g. for V32QImode extract even will be:
40951 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
40952 (all values masked with 0xf except for -128) and second mask
40953 for extract even will be
40954 -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
40957 {
40963 {
40966 }
40967 }
40986 /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
40994 }
40996 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
40997 and extract-odd permutations. */
40999 static bool
41001 {
41005 {
41010 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
41014 /* Now an unpck[lh]pd will produce the result required. */
41017 else
41023 {
41030 /* Shuffle within the 128-bit lanes to produce:
41031 { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
41035 /* Shuffle the lanes around to produce:
41036 { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
41040 /* Shuffle within the 128-bit lanes to produce:
41041 { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
41044 /* Shuffle within the 128-bit lanes to produce:
41045 { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
41048 /* Shuffle the lanes around to produce:
41049 { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
41052 }
41059 /* These are always directly implementable by expand_vec_perm_1. */
41065 else
41066 {
41067 /* We need 2*log2(N)-1 operations to achieve odd/even
41068 with interleave. */
41077 else
41080 }
41086 else
41087 {
41099 else
41102 }
41111 {
41118 {
41123 }
41125 }
41130 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
41134 /* Now an vpunpck[lh]qdq will produce the result required. */
41137 else
41144 {
41151 {
41156 }
41158 }
41166 /* Shuffle the lanes around into
41167 { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
41175 /* Swap the 2nd and 3rd position in each lane into
41176 { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
41182 /* Now an vpunpck[lh]qdq will produce
41183 { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
41187 else
41196 }
41199 }
41201 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
41202 extract-even and extract-odd permutations. */
41204 static bool
41206 {
41218 }
41220 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement broadcast
41221 permutations. We assume that expand_vec_perm_1 has already failed. */
41223 static bool
41225 {
41233 {
41236 /* These are special-cased in sse.md so that we can optionally
41237 use the vbroadcast instruction. They expand to two insns
41238 if the input happens to be in a register. */
41245 /* These are always implementable using standard shuffle patterns. */
41250 /* These can be implemented via interleave. We save one insn by
41251 stopping once we have promoted to V4SImode and then use pshufd. */
41252 do
41253 {
41254 rtx dest;
41260 {
41264 }
41271 }
41286 /* For AVX2 broadcasts of the first element vpbroadcast* or
41287 vpermq should be used by expand_vec_perm_1. */
41293 }
41294 }
41296 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
41297 broadcast permutations. */
41299 static bool
41301 {
41313 }
41315 /* Implement arbitrary permutation of two V32QImode and V16QImode operands
41316 with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
41317 all the shorter instruction sequences. */
41319 static bool
41321 {
41337 /* Generate 4 permutation masks. If the required element is within
41338 the same lane, it is shuffled in. If the required element from the
41339 other lane, force a zero by setting bit 7 in the permutation mask.
41340 In the other mask the mask has non-negative elements if element
41341 is requested from the other lane, but also moved to the other lane,
41342 so that the result of vpshufb can have the two V2TImode halves
41343 swapped. */
41346 {
41351 }
41357 {
41365 }
41368 {
41370 {
41373 }
41380 }
41382 /* Swap the 128-byte lanes of h[X]. */
41384 {
41390 const1_rtx));
41392 }
41395 {
41397 {
41400 }
41406 }
41409 {
41411 {
41415 }
41418 }
41428 }
41430 /* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
41431 With all of the interface bits taken care of, perform the expansion
41432 in D and return true on success. */
41434 static bool
41436 {
41437 /* Try a single instruction expansion. */
41441 /* Try sequences of two instructions. */
41461 /* Try sequences of three instructions. */
41475 /* Try sequences of four instructions. */
41483 /* ??? Look for narrow permutations whose element orderings would
41484 allow the promotion to a wider mode. */
41486 /* ??? Look for sequences of interleave or a wider permute that place
41487 the data into the correct lanes for a half-vector shuffle like
41488 pshuf[lh]w or vpermilps. */
41490 /* ??? Look for sequences of interleave that produce the desired results.
41491 The combinatorics of punpck[lh] get pretty ugly... */
41496 /* Even longer sequences. */
41501 }
41503 /* If a permutation only uses one operand, make it clear. Returns true
41504 if the permutation references both operands. */
41506 static bool
41508 {
41516 {
41522 {
41525 }
41526 /* The elements of PERM do not suggest that only the first operand
41527 is used, but both operands are identical. Allow easier matching
41528 of the permutation by folding the permutation into the single
41529 input vector. */
41530 /* FALLTHRU */
41541 }
41544 }
41546 bool
41548 {
41553 rtx sel;
41570 {
41575 }
41582 /* If the selector says both arguments are needed, but the operands are the
41583 same, the above tried to expand with one_operand_p and flattened selector.
41584 If that didn't work, retry without one_operand_p; we succeeded with that
41585 during testing. */
41587 {
41591 }
41594 }
41596 /* Implement targetm.vectorize.vec_perm_const_ok. */
41598 static bool
41601 {
41610 /* Given sufficient ISA support we can just return true here
41611 for selected vector modes. */
41613 {
41614 /* All implementable with a single vpperm insn. */
41617 /* All implementable with 2 pshufb + 1 ior. */
41620 /* All implementable with shufpd or unpck[lh]pd. */
41623 }
41625 /* Extract the values from the vector CST into the permutation
41626 array in D. */
41629 {
41633 }
41635 /* For all elements from second vector, fold the elements to first. */
41640 /* Check whether the mask can be applied to the vector type. */
41643 /* Implementable with shufps or pshufd. */
41647 /* Otherwise we have to go through the motions and see if we can
41648 figure out how to generate the requested permutation. */
41659 }
41661 void
41663 {
41678 /* We'll either be able to implement the permutation directly... */
41682 /* ... or we use the special-case patterns. */
41684 }
41686 static void
41688 {
41703 {
41706 }
41708 /* Note that for AVX this isn't one instruction. */
41711 }
41714 /* Expand a vector operation CODE for a V*QImode in terms of the
41715 same operation on V*HImode. */
41717 void
41719 {
41731 {
41744 }
41748 {
41750 /* Unpack data such that we've got a source byte in each low byte of
41751 each word. We don't care what goes into the high byte of each word.
41752 Rather than trying to get zero in there, most convenient is to let
41753 it be a copy of the low byte. */
41758 /* FALLTHRU */
41770 /* FALLTHRU */
41780 }
41782 /* Perform the operation. */
41789 /* Merge the data back into the right place. */
41799 {
41800 /* For SSE2, we used an full interleave, so the desired
41801 results are in the even elements. */
41804 }
41805 else
41806 {
41807 /* For AVX, the interleave used above was not cross-lane. So the
41808 extraction is evens but with the second and third quarter swapped.
41809 Happily, that is even one insn shorter than even extraction. */
41812 }
41819 }
41821 /* Helper function of ix86_expand_mul_widen_evenodd. Return true
41822 if op is CONST_VECTOR with all odd elements equal to their
41823 preceding element. */
41825 static bool
41827 {
41837 }
41839 void
41842 {
41845 rtx x;
41853 /* We only play even/odd games with vectors of SImode. */
41856 /* If we're looking for the odd results, shift those members down to
41857 the even slots. For some cpus this is faster than a PSHUFD. */
41859 {
41860 /* For XOP use vpmacsdqh, but only for smult, as it is only
41861 signed. */
41863 {
41867 }
41878 }
41881 {
41884 else
41886 }
41891 else
41892 {
41895 /* The easiest way to implement this without PMULDQ is to go through
41896 the motions as if we are performing a full 64-bit multiply. With
41897 the exception that we need to do less shuffling of the elements. */
41899 /* Compute the sign-extension, aka highparts, of the two operands. */
41905 /* Multiply LO(A) * HI(B), and vice-versa. */
41911 /* Multiply LO(A) * LO(B). */
41915 /* Combine and shift the highparts into place. */
41920 /* Combine high and low parts. */
41923 }
41925 }
41927 void
41930 {
41936 {
41941 {
41942 /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
41943 shuffle the elements once so that all elements are in the right
41944 place for immediate use: { A C B D }. */
41949 }
41950 else
41951 {
41952 /* Put the elements into place for the multiply. */
41956 }
41961 /* Shuffle the elements between the lanes. After this we
41962 have { A B E F | C D G H } for each operand. */
41972 /* Shuffle the elements within the lanes. After this we
41973 have { A A B B | C C D D } or { E E F F | G G H H }. */
42011 }
42012 }
42014 void
42016 {
42026 /* Move the results in element 2 down to element 1; we don't care
42027 what goes in elements 2 and 3. Then we can merge the parts
42028 back together with an interleave.
42030 Note that two other sequences were tried:
42031 (1) Use interleaves at the start instead of psrldq, which allows
42032 us to use a single shufps to merge things back at the end.
42033 (2) Use shufps here to combine the two vectors, then pshufd to
42034 put the elements in the correct order.
42035 In both cases the cost of the reformatting stall was too high
42036 and the overall sequence slower. */
42047 }
42049 void
42051 {
42056 {
42057 /* op1: A,B,C,D, op2: E,F,G,H */
42066 /* t1: B,A,D,C */
42073 /* t2: (B*E),(A*F),(D*G),(C*H) */
42076 /* t3: (B*E)+(A*F), (D*G)+(C*H) */
42079 /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
42082 /* op0: (((B*E)+(A*F))<<32)+(B*F), (((D*G)+(C*H))<<32)+(D*H) */
42084 }
42085 else
42086 {
42091 {
42094 }
42096 {
42099 }
42100 else
42104 /* Multiply low parts. */
42108 /* Shift input vectors right 32 bits so we can multiply high parts. */
42113 /* Multiply high parts by low parts. */
42119 /* Combine and shift the highparts back. */
42123 /* Combine high and low parts. */
42125 }
42129 }
42131 /* Return 1 if control tansfer instruction INSN
42132 should be encoded with bnd prefix.
42133 If insn is NULL then return 1 when control
42134 transfer instructions should be prefixed with
42135 bnd by default for current function. */
42137 bool
42139 {
42141 }
42143 /* Calculate integer abs() using only SSE2 instructions. */
42145 void
42147 {
42152 {
42153 /* For 32-bit signed integer X, the best way to calculate the absolute
42154 value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
42166 /* For 16-bit signed integer X, the best way to calculate the absolute
42167 value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
42175 /* For 8-bit signed integer X, the best way to calculate the absolute
42176 value of X is min ((unsigned char) X, (unsigned char) (-X)),
42177 as SSE2 provides the PMINUB insn. */
42187 }
42191 }
42193 /* Expand an insert into a vector register through pinsr insn.
42194 Return true if successful. */
42196 bool
42198 {
42206 {
42209 }
42215 {
42220 {
42227 {
42259 }
42273 }
42277 }
42278 }
42279 \f
42280 /* This function returns the calling abi specific va_list type node.
42281 It returns the FNDECL specific va_list type. */
42283 static tree
42285 {
42292 else
42294 }
42296 /* Returns the canonical va_list type specified by TYPE. If there
42297 is no valid TYPE provided, it return NULL_TREE. */
42299 static tree
42301 {
42304 /* Resolve references and pointers to va_list type. */
42313 {
42318 {
42319 /* If va_list is an array type, the argument may have decayed
42320 to a pointer type, e.g. by being passed to another function.
42321 In that case, unwrap both types so that we can compare the
42322 underlying records. */
42325 {
42328 }
42329 }
42336 {
42337 /* If va_list is an array type, the argument may have decayed
42338 to a pointer type, e.g. by being passed to another function.
42339 In that case, unwrap both types so that we can compare the
42340 underlying records. */
42343 {
42346 }
42347 }
42354 {
42355 /* If va_list is an array type, the argument may have decayed
42356 to a pointer type, e.g. by being passed to another function.
42357 In that case, unwrap both types so that we can compare the
42358 underlying records. */
42361 {
42364 }
42365 }
42369 }
42371 }
42373 /* Iterate through the target-specific builtin types for va_list.
42374 IDX denotes the iterator, *PTREE is set to the result type of
42375 the va_list builtin, and *PNAME to its internal type.
42376 Returns zero if there is no element for this index, otherwise
42377 IDX should be increased upon the next call.
42378 Note, do not iterate a base builtin's name like __builtin_va_list.
42379 Used from c_common_nodes_and_builtins. */
42381 static int
42383 {
42385 {
42387 {
42400 }
42401 }
42404 }
42406 #undef TARGET_SCHED_DISPATCH
42407 #define TARGET_SCHED_DISPATCH has_dispatch
42408 #undef TARGET_SCHED_DISPATCH_DO
42409 #define TARGET_SCHED_DISPATCH_DO do_dispatch
42410 #undef TARGET_SCHED_REASSOCIATION_WIDTH
42411 #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
42412 #undef TARGET_SCHED_REORDER
42413 #define TARGET_SCHED_REORDER ix86_sched_reorder
42414 #undef TARGET_SCHED_ADJUST_PRIORITY
42415 #define TARGET_SCHED_ADJUST_PRIORITY ix86_adjust_priority
42416 #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
42417 #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK \
42418 ix86_dependencies_evaluation_hook
42420 /* The size of the dispatch window is the total number of bytes of
42421 object code allowed in a window. */
42422 #define DISPATCH_WINDOW_SIZE 16
42424 /* Number of dispatch windows considered for scheduling. */
42425 #define MAX_DISPATCH_WINDOWS 3
42427 /* Maximum number of instructions in a window. */
42428 #define MAX_INSN 4
42430 /* Maximum number of immediate operands in a window. */
42431 #define MAX_IMM 4
42433 /* Maximum number of immediate bits allowed in a window. */
42434 #define MAX_IMM_SIZE 128
42436 /* Maximum number of 32 bit immediates allowed in a window. */
42437 #define MAX_IMM_32 4
42439 /* Maximum number of 64 bit immediates allowed in a window. */
42440 #define MAX_IMM_64 2
42442 /* Maximum total of loads or prefetches allowed in a window. */
42443 #define MAX_LOAD 2
42445 /* Maximum total of stores allowed in a window. */
42446 #define MAX_STORE 1
42448 #undef BIG
42449 #define BIG 100
42452 /* Dispatch groups. Istructions that affect the mix in a dispatch window. */
42455 disp_load,
42456 disp_store,
42457 disp_load_store,
42458 disp_prefetch,
42459 disp_imm,
42460 disp_imm_32,
42461 disp_imm_64,
42462 disp_branch,
42463 disp_cmp,
42464 disp_jcc,
42465 disp_last
42466 };
42468 /* Number of allowable groups in a dispatch window. It is an array
42469 indexed by dispatch_group enum. 100 is used as a big number,
42470 because the number of these kind of operations does not have any
42471 effect in dispatch window, but we need them for other reasons in
42472 the table. */
42475 };
42481 };
42483 /* Instruction path. */
42489 last_path
42490 };
42492 /* sched_insn_info defines a window to the instructions scheduled in
42493 the basic block. It contains a pointer to the insn_info table and
42494 the instruction scheduled.
42496 Windows are allocated for each basic block and are linked
42497 together. */
42499 rtx insn;
42506 /* Linked list of dispatch windows. This is a two way list of
42507 dispatch windows of a basic block. It contains information about
42508 the number of uops in the window and the total number of
42509 instructions and of bytes in the object code for this dispatch
42510 window. */
42528 /* Immediate valuse used in an insn. */
42530 {
42539 /* Get dispatch group of insn. */
42541 static enum dispatch_group
42543 {
42559 }
42561 /* Return true if insn is a compare instruction. */
42563 static bool
42565 {
42573 }
42575 /* Return true if a dispatch violation encountered. */
42577 static bool
42579 {
42583 }
42585 /* Return true if insn is a branch instruction. */
42587 static bool
42589 {
42591 }
42593 /* Return true if insn is a prefetch instruction. */
42595 static bool
42597 {
42599 }
42601 /* This function initializes a dispatch window and the list container holding a
42602 pointer to the window. */
42604 static void
42606 {
42612 else
42630 {
42636 }
42638 }
42640 /* This function allocates and initializes a dispatch window and the
42641 list container holding a pointer to the window. */
42645 {
42650 }
42652 /* This routine initializes the dispatch scheduling information. It
42653 initiates building dispatch scheduler tables and constructs the
42654 first dispatch window. */
42656 static void
42658 {
42659 /* Allocate a dispatch list and a window. */
42664 }
42666 /* This function returns true if a branch is detected. End of a basic block
42667 does not have to be a branch, but here we assume only branches end a
42668 window. */
42670 static bool
42672 {
42674 }
42676 /* This function is called when the end of a window processing is reached. */
42678 static void
42680 {
42683 {
42688 }
42690 }
42692 /* Allocates a new dispatch window and adds it to WINDOW_LIST.
42693 WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
42694 for 48 bytes of instructions. Note that these windows are not dispatch
42695 windows that their sizes are DISPATCH_WINDOW_SIZE. */
42699 {
42701 {
42706 }
42712 }
42714 /* Increment the number of immediate operands of an instruction. */
42716 static int
42718 {
42723 {
42730 else
42741 {
42744 }
42749 }
42752 }
42754 /* Compute number of immediate operands of an instruction. */
42756 static void
42758 {
42761 }
42763 /* Return total size of immediate operands of an instruction along with number
42764 of corresponding immediate-operands. It initializes its parameters to zero
42765 befor calling FIND_CONSTANT.
42766 INSN is the input instruction. IMM is the total of immediates.
42767 IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
42768 bit immediates. */
42770 static int
42772 {
42780 }
42782 /* This function indicates if an operand of an instruction is an
42783 immediate. */
42785 static bool
42787 {
42796 }
42798 /* Return single or double path for instructions. */
42800 static enum insn_path
42802 {
42812 }
42814 /* Return insn dispatch group. */
42816 static enum dispatch_group
42818 {
42836 }
42838 /* Count number of GROUP restricted instructions in a dispatch
42839 window WINDOW_LIST. */
42841 static int
42843 {
42854 {
42873 }
42886 }
42888 /* This function returns true if insn satisfies dispatch rules on the
42889 last window scheduled. */
42891 static bool
42893 {
42901 /* Make disp_cmp and disp_jcc get scheduled at the latest. These
42902 instructions should be given the lowest priority in the
42903 scheduling process in Haifa scheduler to make sure they will be
42904 scheduled in the same dispatch window as the reference to them. */
42908 /* Check nonrestricted. */
42912 /* Get last dispatch window. */
42917 {
42922 /* Window 1 is full. Go for next window. */
42924 }
42931 /* See if it fits in the first window. */
42933 {
42934 /* The first widow should have only single and double path
42935 uops. */
42941 }
42943 }
42945 /* Add an instruction INSN with NUM_UOPS micro-operations to the
42946 dispatch window WINDOW_LIST. */
42948 static void
42950 {
42968 /* Initialize window with new instruction. */
42989 {
42992 }
42993 }
42995 /* Adds a scheduled instruction, INSN, to the current dispatch window.
42996 If the total bytes of instructions or the number of instructions in
42997 the window exceed allowable, it allocates a new window. */
42999 static void
43001 {
43024 /* Get the last dispatch window. */
43032 else
43035 /* If current window is full, get a new window.
43036 Window number zero is full, if MAX_INSN uops are scheduled in it.
43037 Window number one is full, if window zero's bytes plus window
43038 one's bytes is 32, or if the bytes of the new instruction added
43039 to the total makes it greater than 48, or it has already MAX_INSN
43040 instructions in it. */
43049 {
43052 }
43055 {
43059 {
43062 }
43063 }
43065 {
43070 {
43073 }
43076 }
43077 else
43081 {
43082 /* End of basic block is reached do end-basic-block process. */
43085 }
43086 }
43088 /* Print the dispatch window, WINDOW_NUM, to FILE. */
43090 DEBUG_FUNCTION static void
43092 {
43098 else
43112 {
43115 fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
43121 }
43122 }
43124 /* Print to stdout a dispatch window. */
43126 DEBUG_FUNCTION void
43128 {
43130 }
43132 /* Print INSN dispatch information to FILE. */
43134 DEBUG_FUNCTION static void
43136 {
43159 }
43161 /* Print to STDERR the status of the ready list with respect to
43162 dispatch windows. */
43164 DEBUG_FUNCTION void
43166 {
43174 }
43176 /* This routine is the driver of the dispatch scheduler. */
43178 static void
43180 {
43185 }
43187 /* Return TRUE if Dispatch Scheduling is supported. */
43189 static bool
43191 {
43195 {
43211 }
43214 }
43216 /* Implementation of reassociation_width target hook used by
43217 reassoc phase to identify parallelism level in reassociated
43218 tree. Statements tree_code is passed in OPC. Arguments type
43219 is passed in MODE.
43221 Currently parallel reassociation is enabled for Atom
43222 processors only and we set reassociation width to be 2
43223 because Atom may issue up to 2 instructions per cycle.
43225 Return value should be fixed if parallel reassociation is
43226 enabled for other processors. */
43228 static int
43231 {
43240 }
43242 /* ??? No autovectorization into MMX or 3DNOW until we can reliably
43243 place emms and femms instructions. */
43245 static enum machine_mode
43247 {
43252 {
43265 else
43275 /* FALLTHRU */
43279 }
43280 }
43282 /* If AVX is enabled then try vectorizing with both 256bit and 128bit
43283 vectors. */
43285 static unsigned int
43287 {
43289 }
43291 \f
43293 /* Return class of registers which could be used for pseudo of MODE
43294 and of class RCLASS for spilling instead of memory. Return NO_REGS
43295 if it is not possible or non-profitable. */
43296 static reg_class_t
43298 {
43304 }
43306 /* Implement targetm.vectorize.init_cost. */
43310 {
43314 }
43316 /* Implement targetm.vectorize.add_stmt_cost. */
43318 static unsigned
43322 {
43329 /* Statements in an inner loop relative to the loop being
43330 vectorized are weighted more heavily. The value here is
43331 arbitrary and could potentially be improved with analysis. */
43339 }
43341 /* Implement targetm.vectorize.finish_cost. */
43343 static void
43346 {
43351 }
43353 /* Implement targetm.vectorize.destroy_cost_data. */
43355 static void
43357 {
43359 }
43361 /* Validate target specific memory model bits in VAL. */
43363 static unsigned HOST_WIDE_INT
43365 {
43372 {
43376 }
43379 {
43383 }
43385 {
43389 }
43391 }
43393 /* Initialize the GCC target structure. */
43394 #undef TARGET_RETURN_IN_MEMORY
43395 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
43397 #undef TARGET_LEGITIMIZE_ADDRESS
43398 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
43400 #undef TARGET_ATTRIBUTE_TABLE
43401 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
43402 #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
43403 #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P hook_bool_const_tree_true
43404 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
43405 # undef TARGET_MERGE_DECL_ATTRIBUTES
43406 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
43407 #endif
43409 #undef TARGET_COMP_TYPE_ATTRIBUTES
43410 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
43412 #undef TARGET_INIT_BUILTINS
43413 #define TARGET_INIT_BUILTINS ix86_init_builtins
43414 #undef TARGET_BUILTIN_DECL
43415 #define TARGET_BUILTIN_DECL ix86_builtin_decl
43416 #undef TARGET_EXPAND_BUILTIN
43417 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
43419 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
43420 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
43421 ix86_builtin_vectorized_function
43423 #undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
43424 #define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
43426 #undef TARGET_VECTORIZE_BUILTIN_TM_STORE
43427 #define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
43429 #undef TARGET_VECTORIZE_BUILTIN_GATHER
43430 #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
43432 #undef TARGET_BUILTIN_RECIPROCAL
43433 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
43435 #undef TARGET_ASM_FUNCTION_EPILOGUE
43436 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
43438 #undef TARGET_ENCODE_SECTION_INFO
43439 #ifndef SUBTARGET_ENCODE_SECTION_INFO
43440 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
43441 #else
43442 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
43443 #endif
43445 #undef TARGET_ASM_OPEN_PAREN
43447 #undef TARGET_ASM_CLOSE_PAREN
43450 #undef TARGET_ASM_BYTE_OP
43451 #define TARGET_ASM_BYTE_OP ASM_BYTE
43453 #undef TARGET_ASM_ALIGNED_HI_OP
43454 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
43455 #undef TARGET_ASM_ALIGNED_SI_OP
43456 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
43457 #ifdef ASM_QUAD
43458 #undef TARGET_ASM_ALIGNED_DI_OP
43459 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
43460 #endif
43462 #undef TARGET_PROFILE_BEFORE_PROLOGUE
43463 #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
43465 #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
43466 #define TARGET_MANGLE_DECL_ASSEMBLER_NAME ix86_mangle_decl_assembler_name
43468 #undef TARGET_ASM_UNALIGNED_HI_OP
43469 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
43470 #undef TARGET_ASM_UNALIGNED_SI_OP
43471 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
43472 #undef TARGET_ASM_UNALIGNED_DI_OP
43473 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
43475 #undef TARGET_PRINT_OPERAND
43476 #define TARGET_PRINT_OPERAND ix86_print_operand
43477 #undef TARGET_PRINT_OPERAND_ADDRESS
43478 #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
43479 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
43480 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
43481 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
43482 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
43484 #undef TARGET_SCHED_INIT_GLOBAL
43485 #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
43486 #undef TARGET_SCHED_ADJUST_COST
43487 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
43488 #undef TARGET_SCHED_ISSUE_RATE
43489 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
43490 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
43491 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
43492 ia32_multipass_dfa_lookahead
43494 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
43495 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
43497 #undef TARGET_MEMMODEL_CHECK
43498 #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
43500 #ifdef HAVE_AS_TLS
43501 #undef TARGET_HAVE_TLS
43502 #define TARGET_HAVE_TLS true
43503 #endif
43504 #undef TARGET_CANNOT_FORCE_CONST_MEM
43505 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
43506 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
43507 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
43509 #undef TARGET_DELEGITIMIZE_ADDRESS
43510 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
43512 #undef TARGET_MS_BITFIELD_LAYOUT_P
43513 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
43515 #if TARGET_MACHO
43516 #undef TARGET_BINDS_LOCAL_P
43517 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
43518 #endif
43519 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
43520 #undef TARGET_BINDS_LOCAL_P
43521 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
43522 #endif
43524 #undef TARGET_ASM_OUTPUT_MI_THUNK
43525 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
43526 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
43527 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
43529 #undef TARGET_ASM_FILE_START
43530 #define TARGET_ASM_FILE_START x86_file_start
43532 #undef TARGET_OPTION_OVERRIDE
43533 #define TARGET_OPTION_OVERRIDE ix86_option_override
43535 #undef TARGET_REGISTER_MOVE_COST
43536 #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
43537 #undef TARGET_MEMORY_MOVE_COST
43538 #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
43539 #undef TARGET_RTX_COSTS
43540 #define TARGET_RTX_COSTS ix86_rtx_costs
43541 #undef TARGET_ADDRESS_COST
43542 #define TARGET_ADDRESS_COST ix86_address_cost
43544 #undef TARGET_FIXED_CONDITION_CODE_REGS
43545 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
43546 #undef TARGET_CC_MODES_COMPATIBLE
43547 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
43549 #undef TARGET_MACHINE_DEPENDENT_REORG
43550 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
43552 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
43553 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
43555 #undef TARGET_BUILD_BUILTIN_VA_LIST
43556 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
43558 #undef TARGET_FOLD_BUILTIN
43559 #define TARGET_FOLD_BUILTIN ix86_fold_builtin
43561 #undef TARGET_COMPARE_VERSION_PRIORITY
43562 #define TARGET_COMPARE_VERSION_PRIORITY ix86_compare_version_priority
43564 #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
43565 #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
43566 ix86_generate_version_dispatcher_body
43568 #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
43569 #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
43570 ix86_get_function_versions_dispatcher
43572 #undef TARGET_ENUM_VA_LIST_P
43573 #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
43575 #undef TARGET_FN_ABI_VA_LIST
43576 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
43578 #undef TARGET_CANONICAL_VA_LIST_TYPE
43579 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
43581 #undef TARGET_EXPAND_BUILTIN_VA_START
43582 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
43584 #undef TARGET_MD_ASM_CLOBBERS
43585 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
43587 #undef TARGET_PROMOTE_PROTOTYPES
43588 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
43589 #undef TARGET_STRUCT_VALUE_RTX
43590 #define TARGET_STRUCT_VALUE_RTX ix86_struct_value_rtx
43591 #undef TARGET_SETUP_INCOMING_VARARGS
43592 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
43593 #undef TARGET_MUST_PASS_IN_STACK
43594 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
43595 #undef TARGET_FUNCTION_ARG_ADVANCE
43596 #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
43597 #undef TARGET_FUNCTION_ARG
43598 #define TARGET_FUNCTION_ARG ix86_function_arg
43599 #undef TARGET_FUNCTION_ARG_BOUNDARY
43600 #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
43601 #undef TARGET_PASS_BY_REFERENCE
43602 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
43603 #undef TARGET_INTERNAL_ARG_POINTER
43604 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
43605 #undef TARGET_UPDATE_STACK_BOUNDARY
43606 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
43607 #undef TARGET_GET_DRAP_RTX
43608 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
43609 #undef TARGET_STRICT_ARGUMENT_NAMING
43610 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
43611 #undef TARGET_STATIC_CHAIN
43612 #define TARGET_STATIC_CHAIN ix86_static_chain
43613 #undef TARGET_TRAMPOLINE_INIT
43614 #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
43615 #undef TARGET_RETURN_POPS_ARGS
43616 #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
43618 #undef TARGET_LEGITIMATE_COMBINED_INSN
43619 #define TARGET_LEGITIMATE_COMBINED_INSN ix86_legitimate_combined_insn
43621 #undef TARGET_ASAN_SHADOW_OFFSET
43622 #define TARGET_ASAN_SHADOW_OFFSET ix86_asan_shadow_offset
43624 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
43625 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
43627 #undef TARGET_SCALAR_MODE_SUPPORTED_P
43628 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
43630 #undef TARGET_VECTOR_MODE_SUPPORTED_P
43631 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
43633 #undef TARGET_C_MODE_FOR_SUFFIX
43634 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
43636 #ifdef HAVE_AS_TLS
43637 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
43638 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
43639 #endif
43641 #ifdef SUBTARGET_INSERT_ATTRIBUTES
43642 #undef TARGET_INSERT_ATTRIBUTES
43643 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
43644 #endif
43646 #undef TARGET_MANGLE_TYPE
43647 #define TARGET_MANGLE_TYPE ix86_mangle_type
43649 #if !TARGET_MACHO
43650 #undef TARGET_STACK_PROTECT_FAIL
43651 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
43652 #endif
43654 #undef TARGET_FUNCTION_VALUE
43655 #define TARGET_FUNCTION_VALUE ix86_function_value
43657 #undef TARGET_FUNCTION_VALUE_REGNO_P
43658 #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
43660 #undef TARGET_PROMOTE_FUNCTION_MODE
43661 #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
43663 #undef TARGET_MEMBER_TYPE_FORCES_BLK
43664 #define TARGET_MEMBER_TYPE_FORCES_BLK ix86_member_type_forces_blk
43666 #undef TARGET_INSTANTIATE_DECLS
43667 #define TARGET_INSTANTIATE_DECLS ix86_instantiate_decls
43669 #undef TARGET_SECONDARY_RELOAD
43670 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
43672 #undef TARGET_CLASS_MAX_NREGS
43673 #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
43675 #undef TARGET_PREFERRED_RELOAD_CLASS
43676 #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
43677 #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
43678 #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
43679 #undef TARGET_CLASS_LIKELY_SPILLED_P
43680 #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
43682 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
43683 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
43684 ix86_builtin_vectorization_cost
43685 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
43686 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
43687 ix86_vectorize_vec_perm_const_ok
43688 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
43689 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
43690 ix86_preferred_simd_mode
43691 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
43692 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
43693 ix86_autovectorize_vector_sizes
43694 #undef TARGET_VECTORIZE_INIT_COST
43695 #define TARGET_VECTORIZE_INIT_COST ix86_init_cost
43696 #undef TARGET_VECTORIZE_ADD_STMT_COST
43697 #define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
43698 #undef TARGET_VECTORIZE_FINISH_COST
43699 #define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
43700 #undef TARGET_VECTORIZE_DESTROY_COST_DATA
43701 #define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
43703 #undef TARGET_SET_CURRENT_FUNCTION
43704 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
43706 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
43707 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
43709 #undef TARGET_OPTION_SAVE
43710 #define TARGET_OPTION_SAVE ix86_function_specific_save
43712 #undef TARGET_OPTION_RESTORE
43713 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
43715 #undef TARGET_OPTION_PRINT
43716 #define TARGET_OPTION_PRINT ix86_function_specific_print
43718 #undef TARGET_OPTION_FUNCTION_VERSIONS
43719 #define TARGET_OPTION_FUNCTION_VERSIONS ix86_function_versions
43721 #undef TARGET_CAN_INLINE_P
43722 #define TARGET_CAN_INLINE_P ix86_can_inline_p
43724 #undef TARGET_EXPAND_TO_RTL_HOOK
43725 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
43727 #undef TARGET_LEGITIMATE_ADDRESS_P
43728 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
43730 #undef TARGET_LRA_P
43731 #define TARGET_LRA_P hook_bool_void_true
43733 #undef TARGET_REGISTER_PRIORITY
43734 #define TARGET_REGISTER_PRIORITY ix86_register_priority
43736 #undef TARGET_REGISTER_USAGE_LEVELING_P
43737 #define TARGET_REGISTER_USAGE_LEVELING_P hook_bool_void_true
43739 #undef TARGET_LEGITIMATE_CONSTANT_P
43740 #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
43742 #undef TARGET_FRAME_POINTER_REQUIRED
43743 #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
43745 #undef TARGET_CAN_ELIMINATE
43746 #define TARGET_CAN_ELIMINATE ix86_can_eliminate
43748 #undef TARGET_EXTRA_LIVE_ON_ENTRY
43749 #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
43751 #undef TARGET_ASM_CODE_END
43752 #define TARGET_ASM_CODE_END ix86_code_end
43754 #undef TARGET_CONDITIONAL_REGISTER_USAGE
43755 #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
43757 #if TARGET_MACHO
43758 #undef TARGET_INIT_LIBFUNCS
43759 #define TARGET_INIT_LIBFUNCS darwin_rename_builtins
43760 #endif
43762 #undef TARGET_SPILL_CLASS
43763 #define TARGET_SPILL_CLASS ix86_spill_class
43766 \f