| blob | 509cb6e8f832d1fc42300f85ce97375c3a7b5310 |
1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988-2013 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
9 any later version.
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
72 #ifndef CHECK_STACK_LIMIT
73 #define CHECK_STACK_LIMIT (-1)
74 #endif
76 /* Return index of given mode in mult and division cost tables. */
77 #define MODE_INDEX(mode) \
78 ((mode) == QImode ? 0 \
79 : (mode) == HImode ? 1 \
80 : (mode) == SImode ? 2 \
81 : (mode) == DImode ? 3 \
82 : 4)
84 /* Processor costs (relative to an add) */
85 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
86 #define COSTS_N_BYTES(N) ((N) * 2)
88 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall, false}}}
97 const
120 in QImode, HImode and SImode.
121 Relative to reg-reg move (2). */
125 in SFmode, DFmode and XFmode */
127 in SFmode, DFmode and XFmode */
130 in SImode and DImode */
132 in SImode and DImode */
135 in SImode, DImode and TImode */
137 in SImode, DImode and TImode */
150 ix86_size_memcpy,
151 ix86_size_memset,
163 };
165 /* Processor costs (relative to an add) */
168 DUMMY_STRINGOP_ALGS};
171 DUMMY_STRINGOP_ALGS};
173 static const
196 in QImode, HImode and SImode.
197 Relative to reg-reg move (2). */
201 in SFmode, DFmode and XFmode */
203 in SFmode, DFmode and XFmode */
206 in SImode and DImode */
208 in SImode and DImode */
211 in SImode, DImode and TImode */
213 in SImode, DImode and TImode */
226 i386_memcpy,
227 i386_memset,
239 };
243 DUMMY_STRINGOP_ALGS};
246 DUMMY_STRINGOP_ALGS};
248 static const
271 in QImode, HImode and SImode.
272 Relative to reg-reg move (2). */
276 in SFmode, DFmode and XFmode */
278 in SFmode, DFmode and XFmode */
281 in SImode and DImode */
283 in SImode and DImode */
286 in SImode, DImode and TImode */
288 in SImode, DImode and TImode */
291 shared for code and data, so 4kB is
292 not really precise. */
303 i486_memcpy,
304 i486_memset,
316 };
320 DUMMY_STRINGOP_ALGS};
323 DUMMY_STRINGOP_ALGS};
325 static const
348 in QImode, HImode and SImode.
349 Relative to reg-reg move (2). */
353 in SFmode, DFmode and XFmode */
355 in SFmode, DFmode and XFmode */
358 in SImode and DImode */
360 in SImode and DImode */
363 in SImode, DImode and TImode */
365 in SImode, DImode and TImode */
378 pentium_memcpy,
379 pentium_memset,
391 };
393 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
394 (we ensure the alignment). For small blocks inline loop is still a
395 noticeable win, for bigger blocks either rep movsl or rep movsb is
396 way to go. Rep movsb has apparently more expensive startup time in CPU,
397 but after 4K the difference is down in the noise. */
402 DUMMY_STRINGOP_ALGS};
407 DUMMY_STRINGOP_ALGS};
408 static const
431 in QImode, HImode and SImode.
432 Relative to reg-reg move (2). */
436 in SFmode, DFmode and XFmode */
438 in SFmode, DFmode and XFmode */
441 in SImode and DImode */
443 in SImode and DImode */
446 in SImode, DImode and TImode */
448 in SImode, DImode and TImode */
461 pentiumpro_memcpy,
462 pentiumpro_memset,
474 };
478 DUMMY_STRINGOP_ALGS};
481 DUMMY_STRINGOP_ALGS};
482 static const
505 in QImode, HImode and SImode.
506 Relative to reg-reg move (2). */
510 in SFmode, DFmode and XFmode */
512 in SFmode, DFmode and XFmode */
516 in SImode and DImode */
518 in SImode and DImode */
521 in SImode, DImode and TImode */
523 in SImode, DImode and TImode */
536 geode_memcpy,
537 geode_memset,
549 };
553 DUMMY_STRINGOP_ALGS};
556 DUMMY_STRINGOP_ALGS};
557 static const
580 in QImode, HImode and SImode.
581 Relative to reg-reg move (2). */
585 in SFmode, DFmode and XFmode */
587 in SFmode, DFmode and XFmode */
590 in SImode and DImode */
592 in SImode and DImode */
595 in SImode, DImode and TImode */
597 in SImode, DImode and TImode */
601 have integrated l2 cache, but
602 optimizing for k6 is not important
603 enough to worry about that. */
613 k6_memcpy,
614 k6_memset,
626 };
628 /* For some reason, Athlon deals better with REP prefix (relative to loops)
629 compared to K8. Alignment becomes important after 8 bytes for memcpy and
630 128 bytes for memset. */
633 DUMMY_STRINGOP_ALGS};
636 DUMMY_STRINGOP_ALGS};
637 static const
660 in QImode, HImode and SImode.
661 Relative to reg-reg move (2). */
665 in SFmode, DFmode and XFmode */
667 in SFmode, DFmode and XFmode */
670 in SImode and DImode */
672 in SImode and DImode */
675 in SImode, DImode and TImode */
677 in SImode, DImode and TImode */
690 athlon_memcpy,
691 athlon_memset,
703 };
705 /* K8 has optimized REP instruction for medium sized blocks, but for very
706 small blocks it is better to use loop. For large blocks, libcall can
707 do nontemporary accesses and beat inline considerably. */
718 static const
741 in QImode, HImode and SImode.
742 Relative to reg-reg move (2). */
746 in SFmode, DFmode and XFmode */
748 in SFmode, DFmode and XFmode */
751 in SImode and DImode */
753 in SImode and DImode */
756 in SImode, DImode and TImode */
758 in SImode, DImode and TImode */
763 /* New AMD processors never drop prefetches; if they cannot be performed
764 immediately, they are queued. We set number of simultaneous prefetches
765 to a large constant to reflect this (it probably is not a good idea not
766 to limit number of prefetches at all, as their execution also takes some
767 time). */
777 k8_memcpy,
778 k8_memset,
790 };
792 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
793 very small blocks it is better to use loop. For large blocks, libcall can
794 do nontemporary accesses and beat inline considerably. */
827 in QImode, HImode and SImode.
828 Relative to reg-reg move (2). */
832 in SFmode, DFmode and XFmode */
834 in SFmode, DFmode and XFmode */
837 in SImode and DImode */
839 in SImode and DImode */
842 in SImode, DImode and TImode */
844 in SImode, DImode and TImode */
846 /* On K8:
847 MOVD reg64, xmmreg Double FSTORE 4
848 MOVD reg32, xmmreg Double FSTORE 4
849 On AMDFAM10:
850 MOVD reg64, xmmreg Double FADD 3
851 1/1 1/1
852 MOVD reg32, xmmreg Double FADD 3
853 1/1 1/1 */
857 /* New AMD processors never drop prefetches; if they cannot be performed
858 immediately, they are queued. We set number of simultaneous prefetches
859 to a large constant to reflect this (it probably is not a good idea not
860 to limit number of prefetches at all, as their execution also takes some
861 time). */
871 amdfam10_memcpy,
872 amdfam10_memset,
884 };
886 /* BDVER1 has optimized REP instruction for medium sized blocks, but for
887 very small blocks it is better to use loop. For large blocks, libcall
888 can do nontemporary accesses and beat inline considerably. */
922 in QImode, HImode and SImode.
923 Relative to reg-reg move (2). */
927 in SFmode, DFmode and XFmode */
929 in SFmode, DFmode and XFmode */
932 in SImode and DImode */
934 in SImode and DImode */
937 in SImode, DImode and TImode */
939 in SImode, DImode and TImode */
941 /* On K8:
942 MOVD reg64, xmmreg Double FSTORE 4
943 MOVD reg32, xmmreg Double FSTORE 4
944 On AMDFAM10:
945 MOVD reg64, xmmreg Double FADD 3
946 1/1 1/1
947 MOVD reg32, xmmreg Double FADD 3
948 1/1 1/1 */
952 /* New AMD processors never drop prefetches; if they cannot be performed
953 immediately, they are queued. We set number of simultaneous prefetches
954 to a large constant to reflect this (it probably is not a good idea not
955 to limit number of prefetches at all, as their execution also takes some
956 time). */
966 bdver1_memcpy,
967 bdver1_memset,
979 };
981 /* BDVER2 has optimized REP instruction for medium sized blocks, but for
982 very small blocks it is better to use loop. For large blocks, libcall
983 can do nontemporary accesses and beat inline considerably. */
1018 in QImode, HImode and SImode.
1019 Relative to reg-reg move (2). */
1023 in SFmode, DFmode and XFmode */
1025 in SFmode, DFmode and XFmode */
1028 in SImode and DImode */
1030 in SImode and DImode */
1033 in SImode, DImode and TImode */
1035 in SImode, DImode and TImode */
1037 /* On K8:
1038 MOVD reg64, xmmreg Double FSTORE 4
1039 MOVD reg32, xmmreg Double FSTORE 4
1040 On AMDFAM10:
1041 MOVD reg64, xmmreg Double FADD 3
1042 1/1 1/1
1043 MOVD reg32, xmmreg Double FADD 3
1044 1/1 1/1 */
1048 /* New AMD processors never drop prefetches; if they cannot be performed
1049 immediately, they are queued. We set number of simultaneous prefetches
1050 to a large constant to reflect this (it probably is not a good idea not
1051 to limit number of prefetches at all, as their execution also takes some
1052 time). */
1062 bdver2_memcpy,
1063 bdver2_memset,
1075 };
1078 /* BDVER3 has optimized REP instruction for medium sized blocks, but for
1079 very small blocks it is better to use loop. For large blocks, libcall
1080 can do nontemporary accesses and beat inline considerably. */
1113 in QImode, HImode and SImode.
1114 Relative to reg-reg move (2). */
1118 in SFmode, DFmode and XFmode */
1120 in SFmode, DFmode and XFmode */
1123 in SImode and DImode */
1125 in SImode and DImode */
1128 in SImode, DImode and TImode */
1130 in SImode, DImode and TImode */
1135 /* New AMD processors never drop prefetches; if they cannot be performed
1136 immediately, they are queued. We set number of simultaneous prefetches
1137 to a large constant to reflect this (it probably is not a good idea not
1138 to limit number of prefetches at all, as their execution also takes some
1139 time). */
1149 bdver3_memcpy,
1150 bdver3_memset,
1162 };
1164 /* BTVER1 has optimized REP instruction for medium sized blocks, but for
1165 very small blocks it is better to use loop. For large blocks, libcall can
1166 do nontemporary accesses and beat inline considerably. */
1199 in QImode, HImode and SImode.
1200 Relative to reg-reg move (2). */
1204 in SFmode, DFmode and XFmode */
1206 in SFmode, DFmode and XFmode */
1209 in SImode and DImode */
1211 in SImode and DImode */
1214 in SImode, DImode and TImode */
1216 in SImode, DImode and TImode */
1218 /* On K8:
1219 MOVD reg64, xmmreg Double FSTORE 4
1220 MOVD reg32, xmmreg Double FSTORE 4
1221 On AMDFAM10:
1222 MOVD reg64, xmmreg Double FADD 3
1223 1/1 1/1
1224 MOVD reg32, xmmreg Double FADD 3
1225 1/1 1/1 */
1238 btver1_memcpy,
1239 btver1_memset,
1251 };
1285 in QImode, HImode and SImode.
1286 Relative to reg-reg move (2). */
1290 in SFmode, DFmode and XFmode */
1292 in SFmode, DFmode and XFmode */
1295 in SImode and DImode */
1297 in SImode and DImode */
1300 in SImode, DImode and TImode */
1302 in SImode, DImode and TImode */
1304 /* On K8:
1305 MOVD reg64, xmmreg Double FSTORE 4
1306 MOVD reg32, xmmreg Double FSTORE 4
1307 On AMDFAM10:
1308 MOVD reg64, xmmreg Double FADD 3
1309 1/1 1/1
1310 MOVD reg32, xmmreg Double FADD 3
1311 1/1 1/1 */
1323 btver2_memcpy,
1324 btver2_memset,
1336 };
1340 DUMMY_STRINGOP_ALGS};
1344 DUMMY_STRINGOP_ALGS};
1346 static const
1369 in QImode, HImode and SImode.
1370 Relative to reg-reg move (2). */
1374 in SFmode, DFmode and XFmode */
1376 in SFmode, DFmode and XFmode */
1379 in SImode and DImode */
1381 in SImode and DImode */
1384 in SImode, DImode and TImode */
1386 in SImode, DImode and TImode */
1399 pentium4_memcpy,
1400 pentium4_memset,
1412 };
1425 static const
1448 in QImode, HImode and SImode.
1449 Relative to reg-reg move (2). */
1453 in SFmode, DFmode and XFmode */
1455 in SFmode, DFmode and XFmode */
1458 in SImode and DImode */
1460 in SImode and DImode */
1463 in SImode, DImode and TImode */
1465 in SImode, DImode and TImode */
1478 nocona_memcpy,
1479 nocona_memset,
1491 };
1502 static const
1525 in QImode, HImode and SImode.
1526 Relative to reg-reg move (2). */
1530 in SFmode, DFmode and XFmode */
1532 in SFmode, DFmode and XFmode */
1535 in SImode and DImode */
1537 in SImode and DImode */
1540 in SImode, DImode and TImode */
1542 in SImode, DImode and TImode */
1555 atom_memcpy,
1556 atom_memset,
1568 };
1579 static const
1602 in QImode, HImode and SImode.
1603 Relative to reg-reg move (2). */
1607 in SFmode, DFmode and XFmode */
1609 in SFmode, DFmode and XFmode */
1612 in SImode and DImode */
1614 in SImode and DImode */
1617 in SImode, DImode and TImode */
1619 in SImode, DImode and TImode */
1632 slm_memcpy,
1633 slm_memset,
1645 };
1647 /* Generic64 should produce code tuned for Nocona and K8. */
1650 DUMMY_STRINGOP_ALGS,
1654 DUMMY_STRINGOP_ALGS,
1657 static const
1660 /* On all chips taken into consideration lea is 2 cycles and more. With
1661 this cost however our current implementation of synth_mult results in
1662 use of unnecessary temporary registers causing regression on several
1663 SPECfp benchmarks. */
1684 in QImode, HImode and SImode.
1685 Relative to reg-reg move (2). */
1689 in SFmode, DFmode and XFmode */
1691 in SFmode, DFmode and XFmode */
1694 in SImode and DImode */
1696 in SImode and DImode */
1699 in SImode, DImode and TImode */
1701 in SImode, DImode and TImode */
1707 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
1708 value is increased to perhaps more appropriate value of 5. */
1716 generic64_memcpy,
1717 generic64_memset,
1729 };
1731 /* core_cost should produce code tuned for Core familly of CPUs. */
1744 static const
1747 /* On all chips taken into consideration lea is 2 cycles and more. With
1748 this cost however our current implementation of synth_mult results in
1749 use of unnecessary temporary registers causing regression on several
1750 SPECfp benchmarks. */
1771 in QImode, HImode and SImode.
1772 Relative to reg-reg move (2). */
1776 in SFmode, DFmode and XFmode */
1778 in SFmode, DFmode and XFmode */
1781 in SImode and DImode */
1783 in SImode and DImode */
1786 in SImode, DImode and TImode */
1788 in SImode, DImode and TImode */
1794 /* FIXME perhaps more appropriate value is 5. */
1802 core_memcpy,
1803 core_memset,
1815 };
1817 /* Generic32 should produce code tuned for PPro, Pentium4, Nocona,
1818 Athlon and K8. */
1822 DUMMY_STRINGOP_ALGS};
1826 DUMMY_STRINGOP_ALGS};
1827 static const
1850 in QImode, HImode and SImode.
1851 Relative to reg-reg move (2). */
1855 in SFmode, DFmode and XFmode */
1857 in SFmode, DFmode and XFmode */
1860 in SImode and DImode */
1862 in SImode and DImode */
1865 in SImode, DImode and TImode */
1867 in SImode, DImode and TImode */
1880 generic32_memcpy,
1881 generic32_memset,
1893 };
1895 /* Set by -mtune. */
1898 /* Set by -mtune or -Os. */
1901 /* Processor feature/optimization bitmasks. */
1902 #define m_386 (1<<PROCESSOR_I386)
1903 #define m_486 (1<<PROCESSOR_I486)
1904 #define m_PENT (1<<PROCESSOR_PENTIUM)
1905 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
1906 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
1907 #define m_NOCONA (1<<PROCESSOR_NOCONA)
1908 #define m_P4_NOCONA (m_PENT4 | m_NOCONA)
1909 #define m_CORE2 (1<<PROCESSOR_CORE2)
1910 #define m_COREI7 (1<<PROCESSOR_COREI7)
1911 #define m_HASWELL (1<<PROCESSOR_HASWELL)
1912 #define m_CORE_ALL (m_CORE2 | m_COREI7 | m_HASWELL)
1913 #define m_ATOM (1<<PROCESSOR_ATOM)
1914 #define m_SLM (1<<PROCESSOR_SLM)
1916 #define m_GEODE (1<<PROCESSOR_GEODE)
1917 #define m_K6 (1<<PROCESSOR_K6)
1918 #define m_K6_GEODE (m_K6 | m_GEODE)
1919 #define m_K8 (1<<PROCESSOR_K8)
1920 #define m_ATHLON (1<<PROCESSOR_ATHLON)
1921 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
1922 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
1923 #define m_BDVER1 (1<<PROCESSOR_BDVER1)
1924 #define m_BDVER2 (1<<PROCESSOR_BDVER2)
1925 #define m_BDVER3 (1<<PROCESSOR_BDVER3)
1926 #define m_BTVER1 (1<<PROCESSOR_BTVER1)
1927 #define m_BTVER2 (1<<PROCESSOR_BTVER2)
1928 #define m_BDVER (m_BDVER1 | m_BDVER2 | m_BDVER3)
1929 #define m_BTVER (m_BTVER1 | m_BTVER2)
1930 #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER)
1932 #define m_GENERIC32 (1<<PROCESSOR_GENERIC32)
1933 #define m_GENERIC64 (1<<PROCESSOR_GENERIC64)
1935 /* Generic instruction choice should be common subset of supported CPUs
1936 (PPro/PENT4/NOCONA/CORE2/Athlon/K8). */
1937 #define m_GENERIC (m_GENERIC32 | m_GENERIC64)
1940 #undef DEF_TUNE
1941 #define DEF_TUNE(tune, name) name,
1943 #undef DEF_TUNE
1944 };
1946 /* Feature tests against the various tunings. */
1949 /* Feature tests against the various tunings used to create ix86_tune_features
1950 based on the processor mask. */
1952 /* X86_TUNE_USE_LEAVE: Leave does not affect Nocona SPEC2000 results
1953 negatively, so enabling for Generic64 seems like good code size
1954 tradeoff. We can't enable it for 32bit generic because it does not
1955 work well with PPro base chips. */
1958 /* X86_TUNE_PUSH_MEMORY */
1961 /* X86_TUNE_ZERO_EXTEND_WITH_AND */
1964 /* X86_TUNE_UNROLL_STRLEN */
1967 /* X86_TUNE_BRANCH_PREDICTION_HINTS: Branch hints were put in P4 based
1968 on simulation result. But after P4 was made, no performance benefit
1969 was observed with branch hints. It also increases the code size.
1970 As a result, icc never generates branch hints. */
1973 /* X86_TUNE_DOUBLE_WITH_ADD */
1976 /* X86_TUNE_USE_SAHF */
1977 m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_K6_GEODE | m_K8 | m_AMDFAM10 | m_BDVER | m_BTVER | m_GENERIC,
1979 /* X86_TUNE_MOVX: Enable to zero extend integer registers to avoid
1980 partial dependencies. */
1983 /* X86_TUNE_PARTIAL_REG_STALL: We probably ought to watch for partial
1984 register stalls on Generic32 compilation setting as well. However
1985 in current implementation the partial register stalls are not eliminated
1986 very well - they can be introduced via subregs synthesized by combine
1987 and can happen in caller/callee saving sequences. Because this option
1988 pays back little on PPro based chips and is in conflict with partial reg
1989 dependencies used by Athlon/P4 based chips, it is better to leave it off
1990 for generic32 for now. */
1991 m_PPRO,
1993 /* X86_TUNE_PARTIAL_FLAG_REG_STALL */
1996 /* X86_TUNE_LCP_STALL: Avoid an expensive length-changing prefix stall
1997 * on 16-bit immediate moves into memory on Core2 and Corei7. */
2000 /* X86_TUNE_USE_HIMODE_FIOP */
2003 /* X86_TUNE_USE_SIMODE_FIOP */
2006 /* X86_TUNE_USE_MOV0 */
2007 m_K6,
2009 /* X86_TUNE_USE_CLTD */
2012 /* X86_TUNE_USE_XCHGB: Use xchgb %rh,%rl instead of rolw/rorw $8,rx. */
2013 m_PENT4,
2015 /* X86_TUNE_SPLIT_LONG_MOVES */
2016 m_PPRO,
2018 /* X86_TUNE_READ_MODIFY_WRITE */
2021 /* X86_TUNE_READ_MODIFY */
2024 /* X86_TUNE_PROMOTE_QIMODE */
2025 m_386 | m_486 | m_PENT | m_CORE_ALL | m_ATOM | m_SLM | m_K6_GEODE | m_AMD_MULTIPLE | m_GENERIC,
2027 /* X86_TUNE_FAST_PREFIX */
2030 /* X86_TUNE_SINGLE_STRINGOP */
2033 /* X86_TUNE_QIMODE_MATH */
2036 /* X86_TUNE_HIMODE_MATH: On PPro this flag is meant to avoid partial
2037 register stalls. Just like X86_TUNE_PARTIAL_REG_STALL this option
2038 might be considered for Generic32 if our scheme for avoiding partial
2039 stalls was more effective. */
2042 /* X86_TUNE_PROMOTE_QI_REGS */
2045 /* X86_TUNE_PROMOTE_HI_REGS */
2046 m_PPRO,
2048 /* X86_TUNE_SINGLE_POP: Enable if single pop insn is preferred
2049 over esp addition. */
2052 /* X86_TUNE_DOUBLE_POP: Enable if double pop insn is preferred
2053 over esp addition. */
2054 m_PENT,
2056 /* X86_TUNE_SINGLE_PUSH: Enable if single push insn is preferred
2057 over esp subtraction. */
2060 /* X86_TUNE_DOUBLE_PUSH. Enable if double push insn is preferred
2061 over esp subtraction. */
2064 /* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred
2065 for DFmode copies */
2068 /* X86_TUNE_PARTIAL_REG_DEPENDENCY */
2071 /* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: In the Generic model we have a
2072 conflict here in between PPro/Pentium4 based chips that thread 128bit
2073 SSE registers as single units versus K8 based chips that divide SSE
2074 registers to two 64bit halves. This knob promotes all store destinations
2075 to be 128bit to allow register renaming on 128bit SSE units, but usually
2076 results in one extra microop on 64bit SSE units. Experimental results
2077 shows that disabling this option on P4 brings over 20% SPECfp regression,
2078 while enabling it on K8 brings roughly 2.4% regression that can be partly
2079 masked by careful scheduling of moves. */
2082 /* X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL */
2085 /* X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL */
2088 /* X86_TUNE_SSE_PACKED_SINGLE_INSN_OPTIMAL */
2089 m_BDVER ,
2091 /* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies
2092 are resolved on SSE register parts instead of whole registers, so we may
2093 maintain just lower part of scalar values in proper format leaving the
2094 upper part undefined. */
2095 m_ATHLON_K8,
2097 /* X86_TUNE_SSE_TYPELESS_STORES */
2098 m_AMD_MULTIPLE,
2100 /* X86_TUNE_SSE_LOAD0_BY_PXOR */
2103 /* X86_TUNE_MEMORY_MISMATCH_STALL */
2106 /* X86_TUNE_PROLOGUE_USING_MOVE */
2109 /* X86_TUNE_EPILOGUE_USING_MOVE */
2112 /* X86_TUNE_SHIFT1 */
2115 /* X86_TUNE_USE_FFREEP */
2116 m_AMD_MULTIPLE,
2118 /* X86_TUNE_INTER_UNIT_MOVES_TO_VEC */
2121 /* X86_TUNE_INTER_UNIT_MOVES_FROM_VEC */
2124 /* X86_TUNE_INTER_UNIT_CONVERSIONS */
2127 /* X86_TUNE_FOUR_JUMP_LIMIT: Some CPU cores are not able to predict more
2128 than 4 branch instructions in the 16 byte window. */
2131 /* X86_TUNE_SCHEDULE */
2134 /* X86_TUNE_USE_BT */
2137 /* X86_TUNE_USE_INCDEC */
2140 /* X86_TUNE_PAD_RETURNS */
2143 /* X86_TUNE_PAD_SHORT_FUNCTION: Pad short function. */
2144 m_ATOM,
2146 /* X86_TUNE_EXT_80387_CONSTANTS */
2149 /* X86_TUNE_AVOID_VECTOR_DECODE */
2152 /* X86_TUNE_PROMOTE_HIMODE_IMUL: Modern CPUs have same latency for HImode
2153 and SImode multiply, but 386 and 486 do HImode multiply faster. */
2156 /* X86_TUNE_SLOW_IMUL_IMM32_MEM: Imul of 32-bit constant and memory is
2157 vector path on AMD machines. */
2160 /* X86_TUNE_SLOW_IMUL_IMM8: Imul of 8-bit constant is vector path on AMD
2161 machines. */
2164 /* X86_TUNE_MOVE_M1_VIA_OR: On pentiums, it is faster to load -1 via OR
2165 than a MOV. */
2166 m_PENT,
2168 /* X86_TUNE_NOT_UNPAIRABLE: NOT is not pairable on Pentium, while XOR is,
2169 but one byte longer. */
2170 m_PENT,
2172 /* X86_TUNE_NOT_VECTORMODE: On AMD K6, NOT is vector decoded with memory
2173 operand that cannot be represented using a modRM byte. The XOR
2174 replacement is long decoded, so this split helps here as well. */
2175 m_K6,
2177 /* X86_TUNE_USE_VECTOR_FP_CONVERTS: Prefer vector packed SSE conversion
2178 from FP to FP. */
2181 /* X86_TUNE_USE_VECTOR_CONVERTS: Prefer vector packed SSE conversion
2182 from integer to FP. */
2183 m_AMDFAM10,
2185 /* X86_TUNE_FUSE_CMP_AND_BRANCH: Fuse a compare or test instruction
2186 with a subsequent conditional jump instruction into a single
2187 compare-and-branch uop. */
2188 m_BDVER,
2190 /* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag
2191 will impact LEA instruction selection. */
2194 /* X86_TUNE_VECTORIZE_DOUBLE: Enable double precision vector
2195 instructions. */
2198 /* X86_TUNE_SOFTWARE_PREFETCHING_BENEFICIAL: Enable software prefetching
2199 at -O3. For the moment, the prefetching seems badly tuned for Intel
2200 chips. */
2203 /* X86_TUNE_AVX128_OPTIMAL: Enable 128-bit AVX instruction generation for
2204 the auto-vectorizer. */
2207 /* X86_TUNE_REASSOC_INT_TO_PARALLEL: Try to produce parallel computations
2208 during reassociation of integer computation. */
2209 m_ATOM,
2211 /* X86_TUNE_REASSOC_FP_TO_PARALLEL: Try to produce parallel computations
2212 during reassociation of fp computation. */
2215 /* X86_TUNE_GENERAL_REGS_SSE_SPILL: Try to spill general regs to SSE
2216 regs instead of memory. */
2217 m_CORE_ALL,
2219 /* X86_TUNE_AVOID_MEM_OPND_FOR_CMOVE: Try to avoid memory operands for
2220 a conditional move. */
2221 m_ATOM,
2223 /* X86_TUNE_SPLIT_MEM_OPND_FOR_FP_CONVERTS: Try to split memory operand for
2224 fp converts to destination register. */
2225 m_SLM
2227 };
2229 /* Feature tests against the various architecture variations. */
2232 /* Feature tests against the various architecture variations, used to create
2233 ix86_arch_features based on the processor mask. */
2235 /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */
2238 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
2241 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
2244 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
2247 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
2249 };
2251 static const unsigned int x86_accumulate_outgoing_args
2254 static const unsigned int x86_arch_always_fancy_math_387
2257 static const unsigned int x86_avx256_split_unaligned_load
2260 static const unsigned int x86_avx256_split_unaligned_store
2263 /* In case the average insn count for single function invocation is
2264 lower than this constant, emit fast (but longer) prologue and
2265 epilogue code. */
2266 #define FAST_PROLOGUE_INSN_COUNT 20
2268 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
2273 /* Array of the smallest class containing reg number REGNO, indexed by
2274 REGNO. Used by REGNO_REG_CLASS in i386.h. */
2277 {
2278 /* ax, dx, cx, bx */
2280 /* si, di, bp, sp */
2282 /* FP registers */
2285 /* arg pointer */
2286 NON_Q_REGS,
2287 /* flags, fpsr, fpcr, frame */
2289 /* SSE registers */
2292 /* MMX registers */
2295 /* REX registers */
2298 /* SSE REX registers */
2301 };
2303 /* The "default" register map used in 32bit mode. */
2306 {
2314 };
2316 /* The "default" register map used in 64bit mode. */
2319 {
2327 };
2329 /* Define the register numbers to be used in Dwarf debugging information.
2330 The SVR4 reference port C compiler uses the following register numbers
2331 in its Dwarf output code:
2332 0 for %eax (gcc regno = 0)
2333 1 for %ecx (gcc regno = 2)
2334 2 for %edx (gcc regno = 1)
2335 3 for %ebx (gcc regno = 3)
2336 4 for %esp (gcc regno = 7)
2337 5 for %ebp (gcc regno = 6)
2338 6 for %esi (gcc regno = 4)
2339 7 for %edi (gcc regno = 5)
2340 The following three DWARF register numbers are never generated by
2341 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
2342 believes these numbers have these meanings.
2343 8 for %eip (no gcc equivalent)
2344 9 for %eflags (gcc regno = 17)
2345 10 for %trapno (no gcc equivalent)
2346 It is not at all clear how we should number the FP stack registers
2347 for the x86 architecture. If the version of SDB on x86/svr4 were
2348 a bit less brain dead with respect to floating-point then we would
2349 have a precedent to follow with respect to DWARF register numbers
2350 for x86 FP registers, but the SDB on x86/svr4 is so completely
2351 broken with respect to FP registers that it is hardly worth thinking
2352 of it as something to strive for compatibility with.
2353 The version of x86/svr4 SDB I have at the moment does (partially)
2354 seem to believe that DWARF register number 11 is associated with
2355 the x86 register %st(0), but that's about all. Higher DWARF
2356 register numbers don't seem to be associated with anything in
2357 particular, and even for DWARF regno 11, SDB only seems to under-
2358 stand that it should say that a variable lives in %st(0) (when
2359 asked via an `=' command) if we said it was in DWARF regno 11,
2360 but SDB still prints garbage when asked for the value of the
2361 variable in question (via a `/' command).
2362 (Also note that the labels SDB prints for various FP stack regs
2363 when doing an `x' command are all wrong.)
2364 Note that these problems generally don't affect the native SVR4
2365 C compiler because it doesn't allow the use of -O with -g and
2366 because when it is *not* optimizing, it allocates a memory
2367 location for each floating-point variable, and the memory
2368 location is what gets described in the DWARF AT_location
2369 attribute for the variable in question.
2370 Regardless of the severe mental illness of the x86/svr4 SDB, we
2371 do something sensible here and we use the following DWARF
2372 register numbers. Note that these are all stack-top-relative
2373 numbers.
2374 11 for %st(0) (gcc regno = 8)
2375 12 for %st(1) (gcc regno = 9)
2376 13 for %st(2) (gcc regno = 10)
2377 14 for %st(3) (gcc regno = 11)
2378 15 for %st(4) (gcc regno = 12)
2379 16 for %st(5) (gcc regno = 13)
2380 17 for %st(6) (gcc regno = 14)
2381 18 for %st(7) (gcc regno = 15)
2382 */
2384 {
2392 };
2394 /* Define parameter passing and return registers. */
2397 {
2399 };
2402 {
2404 };
2407 {
2409 };
2411 /* Additional registers that are clobbered by SYSV calls. */
2414 {
2419 };
2421 /* Define the structure for the machine field in struct function. */
2426 rtx rtl;
2428 };
2430 /* Structure describing stack frame layout.
2431 Stack grows downward:
2433 [arguments]
2434 <- ARG_POINTER
2435 saved pc
2437 saved static chain if ix86_static_chain_on_stack
2439 saved frame pointer if frame_pointer_needed
2440 <- HARD_FRAME_POINTER
2441 [saved regs]
2442 <- regs_save_offset
2443 [padding0]
2445 [saved SSE regs]
2446 <- sse_regs_save_offset
2447 [padding1] |
2448 | <- FRAME_POINTER
2449 [va_arg registers] |
2450 |
2451 [frame] |
2452 |
2453 [padding2] | = to_allocate
2454 <- STACK_POINTER
2455 */
2456 struct ix86_frame
2457 {
2464 /* The offsets relative to ARG_POINTER. */
2465 HOST_WIDE_INT frame_pointer_offset;
2466 HOST_WIDE_INT hard_frame_pointer_offset;
2467 HOST_WIDE_INT stack_pointer_offset;
2468 HOST_WIDE_INT hfp_save_offset;
2469 HOST_WIDE_INT reg_save_offset;
2470 HOST_WIDE_INT sse_reg_save_offset;
2472 /* When save_regs_using_mov is set, emit prologue using
2473 move instead of push instructions. */
2475 };
2477 /* Which cpu are we scheduling for. */
2480 /* Which cpu are we optimizing for. */
2483 /* Which instruction set architecture to use. */
2486 /* True if processor has SSE prefetch instruction. */
2489 /* -mstackrealign option */
2506 /* Preferred alignment for stack boundary in bits. */
2509 /* Alignment for incoming stack boundary in bits specified at
2510 command line. */
2513 /* Default alignment for incoming stack boundary in bits. */
2516 /* Alignment for incoming stack boundary in bits. */
2519 /* Calling abi specific va_list type nodes. */
2523 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
2527 /* Fence to use after loop using movnt. */
2528 tree x86_mfence;
2530 /* Register class used for passing given 64bit part of the argument.
2531 These represent classes as documented by the PS ABI, with the exception
2532 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
2533 use SF or DFmode move instead of DImode to avoid reformatting penalties.
2535 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
2536 whenever possible (upper half does contain padding). */
2537 enum x86_64_reg_class
2538 {
2539 X86_64_NO_CLASS,
2540 X86_64_INTEGER_CLASS,
2541 X86_64_INTEGERSI_CLASS,
2542 X86_64_SSE_CLASS,
2543 X86_64_SSESF_CLASS,
2544 X86_64_SSEDF_CLASS,
2545 X86_64_SSEUP_CLASS,
2546 X86_64_X87_CLASS,
2547 X86_64_X87UP_CLASS,
2548 X86_64_COMPLEX_X87_CLASS,
2549 X86_64_MEMORY_CLASS
2550 };
2552 #define MAX_CLASSES 4
2554 /* Table of constants used by fldpi, fldln2, etc.... */
2558 \f
2563 const_tree);
2575 enum ix86_function_specific_strings
2576 {
2577 IX86_FUNCTION_SPECIFIC_ARCH,
2578 IX86_FUNCTION_SPECIFIC_TUNE,
2579 IX86_FUNCTION_SPECIFIC_MAX
2580 };
2598 \f
2599 #ifndef SUBTARGET32_DEFAULT_CPU
2601 #endif
2603 /* Whether -mtune= or -march= were specified */
2607 /* Vectorization library interface and handlers. */
2613 /* Processor target table, indexed by processor number */
2614 struct ptt
2615 {
2622 };
2625 {
2636 /* Core 2 */
2638 /* Core i7 */
2640 /* Core avx2 */
2652 };
2655 {
2685 "btver2"
2686 };
2687 \f
2688 static bool
2690 {
2692 }
2694 static unsigned int
2696 {
2699 /* vzeroupper instructions are inserted immediately after reload to
2700 account for possible spills from 256bit registers. The pass
2701 reuses mode switching infrastructure by re-running mode insertion
2702 pass, so disable entities that have already been processed. */
2708 /* Call optimize_mode_switching. */
2711 }
2716 {
2728 };
2731 {
2735 {}
2737 /* opt_pass methods: */
2745 rtl_opt_pass *
2747 {
2749 }
2751 /* Return true if a red-zone is in use. */
2755 {
2757 }
2758 \f
2759 /* Return a string that documents the current -m options. The caller is
2760 responsible for freeing the string. */
2766 {
2767 struct ix86_target_opts
2768 {
2771 };
2773 /* This table is ordered so that options like -msse4.2 that imply
2774 preceding options while match those first. */
2776 {
2812 };
2814 /* Flag options. */
2816 {
2844 };
2861 /* Add -march= option. */
2863 {
2866 }
2868 /* Add -mtune= option. */
2870 {
2873 }
2875 /* Add -m32/-m64/-mx32. */
2877 {
2880 else
2882 isa &= ~ (OPTION_MASK_ISA_64BIT
2883 | OPTION_MASK_ABI_64
2885 }
2886 else
2890 /* Pick out the options in isa options. */
2892 {
2894 {
2897 }
2898 }
2901 {
2904 isa);
2905 }
2907 /* Add flag options. */
2909 {
2911 {
2914 }
2915 }
2918 {
2921 }
2923 /* Add -fpmath= option. */
2925 {
2928 {
2943 }
2944 }
2946 /* Any options? */
2952 /* Size the string. */
2956 {
2961 }
2963 /* Build the string. */
2968 {
2975 {
2977 line_len++;
2980 {
2984 }
2985 }
2989 {
2993 }
2994 }
3000 }
3002 /* Return true, if profiling code should be emitted before
3003 prologue. Otherwise it returns false.
3004 Note: For x86 with "hotfix" it is sorried. */
3005 static bool
3007 {
3009 }
3011 /* Function that is callable from the debugger to print the current
3012 options. */
3013 void
3015 {
3021 {
3024 }
3025 else
3029 }
3032 #define DEF_ENUM
3033 #define DEF_ALG(alg, name) #name,
3035 #undef DEF_ENUM
3036 #undef DEF_ALG
3037 };
3039 /* Parse parameter string passed to -mmemcpy-strategy= or -mmemset-strategy=.
3040 The string is of the following form (or comma separated list of it):
3042 strategy_alg:max_size:[align|noalign]
3044 where the full size range for the strategy is either [0, max_size] or
3045 [min_size, max_size], in which min_size is the max_size + 1 of the
3046 preceding range. The last size range must have max_size == -1.
3048 Examples:
3050 1.
3051 -mmemcpy-strategy=libcall:-1:noalign
3053 this is equivalent to (for known size memcpy) -mstringop-strategy=libcall
3056 2.
3057 -mmemset-strategy=rep_8byte:16:noalign,vector_loop:2048:align,libcall:-1:noalign
3059 This is to tell the compiler to use the following strategy for memset
3060 1) when the expected size is between [1, 16], use rep_8byte strategy;
3061 2) when the size is between [17, 2048], use vector_loop;
3062 3) when the size is > 2048, use libcall. */
3064 struct stringop_size_range
3065 {
3068 stringop_alg alg;
3070 };
3072 static void
3074 {
3082 else
3087 do
3088 {
3090 stringop_alg alg;
3099 {
3103 }
3106 {
3110 }
3113 {
3115 {
3118 }
3119 }
3122 {
3124 alg_name,
3127 }
3136 else
3137 {
3141 }
3142 n++;
3144 }
3148 {
3153 }
3156 {
3160 }
3162 /* Now override the default algs array. */
3164 {
3170 }
3171 }
3173 \f
3174 /* Override various settings based on options. If MAIN_ARGS_P, the
3175 options are from the command line, otherwise they are from
3176 attributes. */
3178 static void
3180 {
3188 #define PTA_3DNOW (HOST_WIDE_INT_1 << 0)
3189 #define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1)
3190 #define PTA_64BIT (HOST_WIDE_INT_1 << 2)
3191 #define PTA_ABM (HOST_WIDE_INT_1 << 3)
3192 #define PTA_AES (HOST_WIDE_INT_1 << 4)
3193 #define PTA_AVX (HOST_WIDE_INT_1 << 5)
3194 #define PTA_BMI (HOST_WIDE_INT_1 << 6)
3195 #define PTA_CX16 (HOST_WIDE_INT_1 << 7)
3196 #define PTA_F16C (HOST_WIDE_INT_1 << 8)
3197 #define PTA_FMA (HOST_WIDE_INT_1 << 9)
3198 #define PTA_FMA4 (HOST_WIDE_INT_1 << 10)
3199 #define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11)
3200 #define PTA_LWP (HOST_WIDE_INT_1 << 12)
3201 #define PTA_LZCNT (HOST_WIDE_INT_1 << 13)
3202 #define PTA_MMX (HOST_WIDE_INT_1 << 14)
3203 #define PTA_MOVBE (HOST_WIDE_INT_1 << 15)
3204 #define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16)
3205 #define PTA_PCLMUL (HOST_WIDE_INT_1 << 17)
3206 #define PTA_POPCNT (HOST_WIDE_INT_1 << 18)
3207 #define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19)
3208 #define PTA_RDRND (HOST_WIDE_INT_1 << 20)
3209 #define PTA_SSE (HOST_WIDE_INT_1 << 21)
3210 #define PTA_SSE2 (HOST_WIDE_INT_1 << 22)
3211 #define PTA_SSE3 (HOST_WIDE_INT_1 << 23)
3212 #define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24)
3213 #define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25)
3214 #define PTA_SSE4A (HOST_WIDE_INT_1 << 26)
3215 #define PTA_SSSE3 (HOST_WIDE_INT_1 << 27)
3216 #define PTA_TBM (HOST_WIDE_INT_1 << 28)
3217 #define PTA_XOP (HOST_WIDE_INT_1 << 29)
3218 #define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
3219 #define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
3220 #define PTA_RTM (HOST_WIDE_INT_1 << 32)
3221 #define PTA_HLE (HOST_WIDE_INT_1 << 33)
3222 #define PTA_PRFCHW (HOST_WIDE_INT_1 << 34)
3223 #define PTA_RDSEED (HOST_WIDE_INT_1 << 35)
3224 #define PTA_ADX (HOST_WIDE_INT_1 << 36)
3225 #define PTA_FXSR (HOST_WIDE_INT_1 << 37)
3226 #define PTA_XSAVE (HOST_WIDE_INT_1 << 38)
3227 #define PTA_XSAVEOPT (HOST_WIDE_INT_1 << 39)
3229 /* if this reaches 64, need to widen struct pta flags below */
3231 static struct pta
3232 {
3237 }
3239 {
3374 PTA_64BIT
3376 };
3378 /* -mrecip options. */
3379 static struct
3380 {
3383 }
3385 {
3392 };
3396 /* Set up prefix/suffix so the error messages refer to either the command
3397 line argument, or the attribute(target). */
3399 {
3403 }
3404 else
3405 {
3409 }
3411 /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
3412 TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
3415 #ifdef TARGET_BI_ARCH
3416 else
3417 {
3418 #if TARGET_BI_ARCH == 1
3419 /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
3420 is on and OPTION_MASK_ABI_X32 is off. We turn off
3421 OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
3422 -mx32. */
3425 #else
3426 /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
3427 on and OPTION_MASK_ABI_64 is off. We turn off
3428 OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
3429 -m64. */
3432 #endif
3433 }
3434 #endif
3437 {
3438 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3439 OPTION_MASK_ABI_64 for TARGET_X32. */
3442 }
3444 {
3445 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3446 OPTION_MASK_ABI_X32 for TARGET_LP64. */
3449 }
3451 #ifdef SUBTARGET_OVERRIDE_OPTIONS
3452 SUBTARGET_OVERRIDE_OPTIONS;
3453 #endif
3455 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
3456 SUBSUBTARGET_OVERRIDE_OPTIONS;
3457 #endif
3459 /* -fPIC is the default for x86_64. */
3463 /* Need to check -mtune=generic first. */
3465 {
3468 /* As special support for cross compilers we read -mtune=native
3469 as -mtune=generic. With native compilers we won't see the
3470 -mtune=native, as it was changed by the driver. */
3472 {
3475 else
3477 }
3478 /* If this call is for setting the option attribute, allow the
3479 generic32/generic64 that was previously set. */
3483 ;
3491 }
3492 else
3493 {
3497 {
3500 }
3502 /* ix86_tune_string is set to ix86_arch_string or defaulted. We
3503 need to use a sensible tune option. */
3507 {
3510 else
3512 }
3513 }
3516 {
3517 /* rep; movq isn't available in 32-bit code. */
3520 }
3524 else
3528 {
3534 }
3535 else
3542 {
3544 {
3588 {
3591 }
3599 }
3600 }
3601 else
3602 {
3603 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
3604 use of rip-relative addressing. This eliminates fixups that
3605 would otherwise be needed if this object is to be placed in a
3606 DLL, and is essentially just as efficient as direct addressing. */
3611 else
3613 }
3615 {
3618 }
3625 {
3628 /* Default cpu tuning to the architecture. */
3753 }
3768 {
3772 {
3774 {
3776 {
3784 }
3785 else
3788 }
3789 }
3790 else
3791 {
3792 /* Adjust tuning when compiling for 32-bit ABI. */
3794 {
3802 }
3803 }
3804 /* Intel CPUs have always interpreted SSE prefetch instructions as
3805 NOPs; so, we can enable SSE prefetch instructions even when
3806 -mtune (rather than -march) points us to a processor that has them.
3807 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
3808 higher processors. */
3813 }
3824 {
3825 /* parse the tune ctrl string in the following form:
3826 [^]tune_name1,[^]tune_name2,..a */
3837 {
3838 curr_feature_string++;
3840 }
3842 {
3844 {
3847 }
3848 }
3855 }
3857 #ifndef USE_IX86_FRAME_POINTER
3858 #define USE_IX86_FRAME_POINTER 0
3859 #endif
3861 #ifndef USE_X86_64_FRAME_POINTER
3862 #define USE_X86_64_FRAME_POINTER 0
3863 #endif
3865 /* Set the default values for switches whose default depends on TARGET_64BIT
3866 in case they weren't overwritten by command line options. */
3868 {
3875 }
3876 else
3877 {
3884 }
3889 else
3892 /* Arrange to set up i386_stack_locals for all functions. */
3895 /* Validate -mregparm= value. */
3897 {
3901 {
3905 }
3906 }
3910 /* Default align_* from the processor table. */
3912 {
3915 }
3917 {
3920 }
3922 {
3924 }
3926 /* Provide default for -mbranch-cost= value. */
3931 {
3934 /* Enable by default the SSE and MMX builtins. Do allow the user to
3935 explicitly disable any of these. In particular, disabling SSE and
3936 MMX for kernel code is extremely useful. */
3938 ix86_isa_flags
3944 }
3945 else
3946 {
3950 ix86_isa_flags
3953 /* i386 ABI does not specify red zone. It still makes sense to use it
3954 when programmer takes care to stack from being destroyed. */
3957 }
3959 /* Keep nonleaf frame pointers. */
3965 /* If we're doing fast math, we don't care about comparison order
3966 wrt NaNs. This lets us use a shorter comparison sequence. */
3970 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3971 since the insns won't need emulation. */
3975 /* Likewise, if the target doesn't have a 387, or we've specified
3976 software floating point, don't use 387 inline intrinsics. */
3980 /* Turn on MMX builtins for -msse. */
3984 /* Enable SSE prefetch. */
3988 /* Enable prefetch{,w} instructions for -m3dnow. */
3992 /* Enable popcnt instruction for -msse4.2 or -mabm. */
3996 /* Enable lzcnt instruction for -mabm. */
4000 /* Validate -mpreferred-stack-boundary= value or default it to
4001 PREFERRED_STACK_BOUNDARY_DEFAULT. */
4004 {
4010 {
4014 else
4017 }
4018 else
4019 ix86_preferred_stack_boundary
4021 }
4023 /* Set the default value for -mstackrealign. */
4029 /* Validate -mincoming-stack-boundary= value or default it to
4030 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
4033 {
4038 else
4039 {
4040 ix86_user_incoming_stack_boundary
4042 ix86_incoming_stack_boundary
4044 }
4045 }
4047 /* Accept -msseregparm only if at least SSE support is enabled. */
4053 {
4055 {
4057 {
4060 }
4062 {
4065 }
4066 }
4067 }
4068 else
4071 /* If the i387 is disabled, then do not return values in it. */
4075 /* Use external vectorized library in vectorizing intrinsics. */
4078 {
4089 }
4097 /* ??? Unwind info is not correct around the CFG unless either a frame
4098 pointer is present or M_A_O_A is set. Fixing this requires rewriting
4099 unwind info generation to be aware of the CFG and propagating states
4100 around edges. */
4103 && flag_omit_frame_pointer
4105 {
4111 }
4113 /* If stack probes are required, the space used for large function
4114 arguments on the stack must also be probed, so enable
4115 -maccumulate-outgoing-args so this happens in the prologue. */
4118 {
4123 }
4125 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
4126 {
4132 }
4134 /* When scheduling description is not available, disable scheduler pass
4135 so it won't slow down the compilation and make x87 code slower. */
4156 /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
4158 && HAVE_prefetch
4163 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
4164 can be optimized to ap = __builtin_next_arg (0). */
4169 {
4172 {
4175 ix86_gen_tls_local_dynamic_base_64
4177 }
4178 else
4179 {
4182 ix86_gen_tls_local_dynamic_base_64
4184 }
4185 }
4186 else
4187 {
4190 }
4193 {
4202 }
4203 else
4204 {
4213 }
4215 #ifdef USE_IX86_CLD
4216 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
4219 #endif
4222 {
4227 }
4229 {
4233 }
4235 {
4236 #if defined(PROFILE_BEFORE_PROLOGUE)
4238 #else
4240 #endif
4241 }
4243 /* When not optimize for size, enable vzeroupper optimization for
4244 TARGET_AVX with -fexpensive-optimizations and split 32-byte
4245 AVX unaligned load/store. */
4247 {
4257 /* Enable 128-bit AVX instruction generation
4258 for the auto-vectorizer. */
4262 }
4265 {
4272 {
4275 {
4277 q++;
4278 }
4279 else
4284 else
4285 {
4288 {
4291 }
4294 {
4298 }
4299 }
4304 else
4306 }
4307 }
4314 /* Default long double to 64-bit for Bionic. */
4319 /* Save the initial options in case the user does function specific
4320 options. */
4322 target_option_default_node = target_option_current_node
4325 /* Handle stack protector */
4329 /* Handle -mmemcpy-strategy= and -mmemset-strategy= */
4331 {
4335 }
4338 {
4342 }
4343 }
4345 /* Implement the TARGET_OPTION_OVERRIDE hook. */
4347 static void
4349 {
4351 static struct register_pass_info insert_vzeroupper_info
4354 };
4359 /* This needs to be done at start up. It's convenient to do it here. */
4361 }
4363 /* Update register usage after having seen the compiler flags. */
4365 static void
4367 {
4371 /* The PIC register, if it exists, is fixed. */
4376 /* For 32-bit targets, squash the REX registers. */
4378 {
4383 }
4385 /* See the definition of CALL_USED_REGISTERS in i386.h. */
4393 {
4394 /* Set/reset conditionally defined registers from
4395 CALL_USED_REGISTERS initializer. */
4399 /* Calculate registers of CLOBBERED_REGS register set
4400 as call used registers from GENERAL_REGS register set. */
4404 }
4406 /* If MMX is disabled, squash the registers. */
4412 /* If SSE is disabled, squash the registers. */
4418 /* If the FPU is disabled, squash the registers. */
4423 }
4425 \f
4426 /* Save the current options */
4428 static void
4430 {
4441 /* The fields are char but the variables are not; make sure the
4442 values fit in the fields. */
4447 }
4449 /* Restore the current options */
4451 static void
4453 {
4469 /* Recreate the arch feature tests if the arch changed */
4471 {
4476 }
4478 /* Recreate the tune optimization tests */
4480 {
4485 }
4486 }
4488 /* Print the current options */
4490 static void
4493 {
4515 {
4518 }
4519 }
4521 \f
4522 /* Inner function to process the attribute((target(...))), take an argument and
4523 set the current options from the argument. If we have a list, recursively go
4524 over the list. */
4526 static bool
4529 {
4533 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
4534 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
4535 #define IX86_ATTR_ENUM(S,O) { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
4536 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
4537 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
4539 enum ix86_opt_type
4540 {
4541 ix86_opt_unknown,
4542 ix86_opt_yes,
4543 ix86_opt_no,
4544 ix86_opt_str,
4545 ix86_opt_enum,
4546 ix86_opt_isa
4547 };
4549 static const struct
4550 {
4557 /* isa options */
4594 /* enum options */
4597 /* string options */
4601 /* flag options */
4603 OPT_mcld,
4604 MASK_CLD),
4607 OPT_mfancy_math_387,
4608 MASK_NO_FANCY_MATH_387),
4611 OPT_mieee_fp,
4612 MASK_IEEE_FP),
4615 OPT_minline_all_stringops,
4616 MASK_INLINE_ALL_STRINGOPS),
4619 OPT_minline_stringops_dynamically,
4620 MASK_INLINE_STRINGOPS_DYNAMICALLY),
4623 OPT_mno_align_stringops,
4624 MASK_NO_ALIGN_STRINGOPS),
4627 OPT_mrecip,
4628 MASK_RECIP),
4630 };
4632 /* If this is a list, recurse to get the options. */
4634 {
4644 }
4647 {
4650 }
4652 /* Handle multiple arguments separated by commas. */
4656 {
4670 {
4674 }
4675 else
4676 {
4679 }
4681 /* Recognize no-xxx. */
4683 {
4687 }
4688 else
4691 /* Find the option. */
4695 {
4703 {
4708 }
4709 }
4711 /* Process the option. */
4713 {
4716 }
4719 {
4725 }
4728 {
4734 else
4736 }
4739 {
4741 {
4744 }
4745 else
4747 }
4750 {
4758 global_dc);
4759 else
4760 {
4763 }
4764 }
4766 else
4768 }
4771 }
4773 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
4775 tree
4777 {
4792 /* Process each of the options on the chain. */
4797 /* If the changed options are different from the default, rerun
4798 ix86_option_override_internal, and then save the options away.
4799 The string options are are attribute options, and will be undone
4800 when we copy the save structure. */
4806 {
4807 /* If we are using the default tune= or arch=, undo the string assigned,
4808 and use the default. */
4819 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
4823 {
4826 }
4828 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
4831 /* Add any builtin functions with the new isa if any. */
4834 /* Save the current options unless we are validating options for
4835 #pragma. */
4842 /* Free up memory allocated to hold the strings */
4845 }
4848 }
4850 /* Hook to validate attribute((target("string"))). */
4852 static bool
4855 tree args,
4857 {
4861 /* attribute((target("default"))) does nothing, beyond
4862 affecting multi-versioning. */
4873 /* If the function changed the optimization levels as well as setting target
4874 options, start with the optimizations specified. */
4879 /* The target attributes may also change some optimization flags, so update
4880 the optimization options if necessary. */
4889 {
4894 }
4903 }
4905 \f
4906 /* Hook to determine if one function can safely inline another. */
4908 static bool
4910 {
4915 /* If callee has no option attributes, then it is ok to inline. */
4919 /* If caller has no option attributes, but callee does then it is not ok to
4920 inline. */
4924 else
4925 {
4929 /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
4930 can inline a SSE2 function but a SSE2 function can't inline a SSE4
4931 function. */
4936 /* See if we have the same non-isa options. */
4940 /* See if arch, tune, etc. are the same. */
4953 else
4955 }
4958 }
4960 \f
4961 /* Remember the last target of ix86_set_current_function. */
4964 /* Invalidate ix86_previous_fndecl cache. */
4965 void
4967 {
4969 }
4971 /* Establish appropriate back-end context for processing the function
4972 FNDECL. The argument might be NULL to indicate processing at top
4973 level, outside of any function scope. */
4974 static void
4976 {
4977 /* Only change the context if the function changes. This hook is called
4978 several times in the course of compiling a function, and we don't want to
4979 slow things down too much or call target_reinit when it isn't safe. */
4981 {
4982 tree old_tree = (ix86_previous_fndecl
4986 tree new_tree = (fndecl
4992 ;
4995 {
4999 }
5002 {
5008 }
5009 }
5010 }
5012 \f
5013 /* Return true if this goes in large data/bss. */
5015 static bool
5017 {
5021 /* Functions are never large data. */
5026 {
5032 }
5033 else
5034 {
5037 /* If this is an incomplete type with size 0, then we can't put it
5038 in data because it might be too big when completed. */
5041 }
5044 }
5046 /* Switch to the appropriate section for output of DECL.
5047 DECL is either a `VAR_DECL' node or a constant of some sort.
5048 RELOC indicates whether forming the initial value of DECL requires
5049 link-time relocations. */
5052 ATTRIBUTE_UNUSED;
5057 {
5060 {
5064 {
5098 /* We don't split these for medium model. Place them into
5099 default sections and hope for best. */
5101 }
5103 {
5104 /* We might get called with string constants, but get_named_section
5105 doesn't like them as they are not DECLs. Also, we need to set
5106 flags in that case. */
5110 }
5111 }
5113 }
5115 /* Build up a unique section name, expressed as a
5116 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
5117 RELOC indicates whether the initial value of EXP requires
5118 link-time relocations. */
5120 static void ATTRIBUTE_UNUSED
5122 {
5125 {
5127 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
5131 {
5155 /* We don't split these for medium model. Place them into
5156 default sections and hope for best. */
5158 }
5160 {
5167 /* If we're using one_only, then there needs to be a .gnu.linkonce
5168 prefix to the section name. */
5175 }
5176 }
5178 }
5180 #ifdef COMMON_ASM_OP
5181 /* This says how to output assembler code to declare an
5182 uninitialized external linkage data object.
5184 For medium model x86-64 we need to use .largecomm opcode for
5185 large objects. */
5186 void
5190 {
5194 else
5199 }
5200 #endif
5202 /* Utility function for targets to use in implementing
5203 ASM_OUTPUT_ALIGNED_BSS. */
5205 void
5209 {
5213 else
5216 #ifdef ASM_DECLARE_OBJECT_NAME
5219 #else
5220 /* Standard thing is just output label for the object. */
5224 }
5225 \f
5226 /* Decide whether we must probe the stack before any space allocation
5227 on this target. It's essentially TARGET_STACK_PROBE except when
5228 -fstack-check causes the stack to be already probed differently. */
5230 bool
5232 {
5233 /* Do not probe the stack twice if static stack checking is enabled. */
5238 }
5239 \f
5240 /* Decide whether we can make a sibling call to a function. DECL is the
5241 declaration of the function being targeted by the call and EXP is the
5242 CALL_EXPR representing the call. */
5244 static bool
5246 {
5250 /* If we are generating position-independent code, we cannot sibcall
5251 optimize any indirect call, or a direct call to a global function,
5252 as the PLT requires %ebx be live. (Darwin does not have a PLT.) */
5254 && !TARGET_64BIT
5255 && flag_pic
5259 /* If we need to align the outgoing stack, then sibcalling would
5260 unalign the stack, which may break the called function. */
5266 {
5269 }
5270 else
5271 {
5272 /* We're looking at the CALL_EXPR, we need the type of the function. */
5277 }
5279 /* Check that the return value locations are the same. Like
5280 if we are returning floats on the 80387 register stack, we cannot
5281 make a sibcall from a function that doesn't return a float to a
5282 function that does or, conversely, from a function that does return
5283 a float to a function that doesn't; the necessary stack adjustment
5284 would not be executed. This is also the place we notice
5285 differences in the return value ABI. Note that it is ok for one
5286 of the functions to have void return type as long as the return
5287 value of the other is passed in a register. */
5292 {
5295 }
5297 ;
5302 {
5303 /* The SYSV ABI has more call-clobbered registers;
5304 disallow sibcalls from MS to SYSV. */
5308 }
5309 else
5310 {
5311 /* If this call is indirect, we'll need to be able to use a
5312 call-clobbered register for the address of the target function.
5313 Make sure that all such registers are not used for passing
5314 parameters. Note that DLLIMPORT functions are indirect. */
5317 {
5319 {
5320 /* ??? Need to count the actual number of registers to be used,
5321 not the possible number of registers. Fix later. */
5323 }
5324 }
5325 }
5327 /* Otherwise okay. That also includes certain types of indirect calls. */
5329 }
5331 /* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
5332 and "sseregparm" calling convention attributes;
5333 arguments as in struct attribute_spec.handler. */
5335 static tree
5337 tree args,
5340 {
5345 {
5347 name);
5350 }
5352 /* Can combine regparm with all attributes but fastcall, and thiscall. */
5354 {
5355 tree cst;
5358 {
5360 }
5363 {
5365 }
5369 {
5372 name);
5374 }
5376 {
5380 }
5383 }
5386 {
5387 /* Do not warn when emulating the MS ABI. */
5392 name);
5395 }
5397 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
5399 {
5401 {
5403 }
5405 {
5407 }
5409 {
5411 }
5413 {
5415 }
5416 }
5418 /* Can combine stdcall with fastcall (redundant), regparm and
5419 sseregparm. */
5421 {
5423 {
5425 }
5427 {
5429 }
5431 {
5433 }
5434 }
5436 /* Can combine cdecl with regparm and sseregparm. */
5438 {
5440 {
5442 }
5444 {
5446 }
5448 {
5450 }
5451 }
5453 {
5456 name);
5458 {
5460 }
5462 {
5464 }
5466 {
5468 }
5469 }
5471 /* Can combine sseregparm with all attributes. */
5474 }
5476 /* The transactional memory builtins are implicitly regparm or fastcall
5477 depending on the ABI. Override the generic do-nothing attribute that
5478 these builtins were declared with, and replace it with one of the two
5479 attributes that we expect elsewhere. */
5481 static tree
5483 tree args ATTRIBUTE_UNUSED,
5486 {
5487 tree alt;
5489 /* In no case do we want to add the placeholder attribute. */
5492 /* The 64-bit ABI is unchanged for transactional memory. */
5496 /* ??? Is there a better way to validate 32-bit windows? We have
5497 cfun->machine->call_abi, but that seems to be set only for 64-bit. */
5500 else
5501 {
5504 }
5508 }
5510 /* This function determines from TYPE the calling-convention. */
5512 unsigned int
5514 {
5517 tree attrs;
5524 {
5534 /* Regparam isn't allowed for thiscall and fastcall. */
5536 {
5541 }
5545 }
5552 || is_stdarg
5558 }
5560 /* Return 0 if the attributes for two types are incompatible, 1 if they
5561 are compatible, and 2 if they are nearly compatible (which causes a
5562 warning to be generated). */
5564 static int
5566 {
5582 }
5583 \f
5584 /* Return the regparm value for a function with the indicated TYPE and DECL.
5585 DECL may be NULL when calling function indirectly
5586 or considering a libcall. */
5588 static int
5590 {
5591 tree attr;
5602 {
5605 {
5608 }
5609 }
5615 /* Use register calling convention for local functions when possible. */
5618 && optimize
5620 {
5621 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5624 {
5627 /* Make sure no regparm register is taken by a
5628 fixed register variable. */
5633 /* We don't want to use regparm(3) for nested functions as
5634 these use a static chain pointer in the third argument. */
5638 /* In 32-bit mode save a register for the split stack. */
5642 /* Each fixed register usage increases register pressure,
5643 so less registers should be used for argument passing.
5644 This functionality can be overriden by an explicit
5645 regparm value. */
5648 globals++;
5650 local_regparm
5655 }
5656 }
5659 }
5661 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
5662 DFmode (2) arguments in SSE registers for a function with the
5663 indicated TYPE and DECL. DECL may be NULL when calling function
5664 indirectly or considering a libcall. Otherwise return 0. */
5666 static int
5668 {
5671 /* Use SSE registers to pass SFmode and DFmode arguments if requested
5672 by the sseregparm attribute. */
5675 {
5677 {
5679 {
5683 else
5686 }
5688 }
5691 }
5693 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
5694 (and DFmode for SSE2) arguments in SSE registers. */
5697 {
5698 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5702 }
5705 }
5707 /* Return true if EAX is live at the start of the function. Used by
5708 ix86_expand_prologue to determine if we need special help before
5709 calling allocate_stack_worker. */
5711 static bool
5713 {
5714 /* Cheat. Don't bother working forward from ix86_function_regparm
5715 to the function type to whether an actual argument is located in
5716 eax. Instead just look at cfg info, which is still close enough
5717 to correct at this point. This gives false positives for broken
5718 functions that might use uninitialized data that happens to be
5719 allocated in eax, but who cares? */
5721 }
5723 static bool
5725 {
5726 tree attr;
5729 {
5735 /* For 32-bit MS-ABI the default is to keep aggregate
5736 return pointer. */
5739 }
5741 }
5743 /* Value is the number of bytes of arguments automatically
5744 popped when returning from a subroutine call.
5745 FUNDECL is the declaration node of the function (as a tree),
5746 FUNTYPE is the data type of the function (as a tree),
5747 or for a library call it is an identifier node for the subroutine name.
5748 SIZE is the number of bytes of arguments passed on the stack.
5750 On the 80386, the RTD insn may be used to pop them if the number
5751 of args is fixed, but if the number is variable then the caller
5752 must pop them all. RTD can't be used for library calls now
5753 because the library is compiled with the Unix compiler.
5754 Use of RTD is a selectable option, since it is incompatible with
5755 standard Unix calling sequences. If the option is not selected,
5756 the caller must always pop the args.
5758 The attribute stdcall is equivalent to RTD on a per module basis. */
5760 static int
5762 {
5765 /* None of the 64-bit ABIs pop arguments. */
5776 /* Lose any fake structure return argument if it is passed on the stack. */
5779 {
5783 }
5786 }
5788 /* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook. */
5790 static bool
5792 {
5793 /* Check operand constraints in case hard registers were propagated
5794 into insn pattern. This check prevents combine pass from
5795 generating insn patterns with invalid hard register operands.
5796 These invalid insns can eventually confuse reload to error out
5797 with a spill failure. See also PRs 46829 and 46843. */
5799 {
5806 {
5814 /* A unary operator may be accepted by the predicate, but it
5815 is irrelevant for matching constraints. */
5820 {
5828 }
5835 /* Operand has no constraints, anything is OK. */
5839 {
5842 && operands_match_p
5846 {
5849 }
5850 }
5854 }
5855 }
5858 }
5859 \f
5860 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
5862 static unsigned HOST_WIDE_INT
5864 {
5868 }
5869 \f
5870 /* Argument support functions. */
5872 /* Return true when register may be used to pass function parameters. */
5873 bool
5875 {
5880 {
5884 else
5890 }
5893 {
5896 }
5897 else
5898 {
5902 }
5904 /* TODO: The function should depend on current function ABI but
5905 builtins.c would need updating then. Therefore we use the
5906 default ABI. */
5908 /* RAX is used as hidden argument to va_arg functions. */
5914 else
5921 }
5923 /* Return if we do not know how to pass TYPE solely in registers. */
5925 static bool
5927 {
5931 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
5932 The layout_type routine is crafty and tries to trick us into passing
5933 currently unsupported vector types on the stack by using TImode. */
5936 }
5938 /* It returns the size, in bytes, of the area reserved for arguments passed
5939 in registers for the function represented by fndecl dependent to the used
5940 abi format. */
5941 int
5943 {
5947 else
5952 }
5954 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
5955 call abi used. */
5956 enum calling_abi
5958 {
5960 {
5963 {
5966 }
5970 }
5972 }
5974 static bool
5976 {
5978 {
5982 else
5984 }
5986 }
5988 static enum calling_abi
5990 {
5994 }
5996 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
5997 call abi used. */
5998 enum calling_abi
6000 {
6004 }
6006 /* Write the extra assembler code needed to declare a function properly. */
6008 void
6010 tree decl)
6011 {
6015 {
6021 }
6023 #ifdef SUBTARGET_ASM_UNWIND_INIT
6025 #endif
6029 /* Output magic byte marker, if hot-patch attribute is set. */
6031 {
6033 {
6034 /* leaq [%rsp + 0], %rsp */
6037 }
6038 else
6039 {
6040 /* movl.s %edi, %edi
6041 push %ebp
6042 movl.s %esp, %ebp */
6045 }
6046 }
6047 }
6049 /* regclass.c */
6052 /* Implementation of call abi switching target hook. Specific to FNDECL
6053 the specific call register sets are set. See also
6054 ix86_conditional_register_usage for more details. */
6055 void
6057 {
6060 else
6062 }
6064 /* 64-bit MS and SYSV ABI have different set of call used registers. Avoid
6065 expensive re-initialization of init_regs each time we switch function context
6066 since this is needed only during RTL expansion. */
6067 static void
6069 {
6073 }
6075 /* Initialize a variable CUM of type CUMULATIVE_ARGS
6076 for a call to a function whose data type is FNTYPE.
6077 For a library call, FNTYPE is 0. */
6079 void
6083 tree fndecl,
6085 {
6091 {
6094 }
6095 else
6096 {
6099 }
6103 /* Set up the number of registers to use for passing arguments. */
6110 {
6112 ? X86_64_REGPARM_MAX
6114 }
6116 {
6119 {
6121 ? X86_64_SSE_REGPARM_MAX
6123 }
6124 }
6131 /* Because type might mismatch in between caller and callee, we need to
6132 use actual type of function for local calls.
6133 FIXME: cgraph_analyze can be told to actually record if function uses
6134 va_start so for local functions maybe_vaarg can be made aggressive
6135 helping K&R code.
6136 FIXME: once typesytem is fixed, we won't need this code anymore. */
6144 {
6145 /* If there are variable arguments, then we won't pass anything
6146 in registers in 32-bit mode. */
6148 {
6156 }
6158 /* Use ecx and edx registers if function has fastcall attribute,
6159 else look for regparm information. */
6161 {
6164 {
6167 }
6169 {
6172 }
6173 else
6175 }
6177 /* Set up the number of SSE registers used for passing SFmode
6178 and DFmode arguments. Warn for mismatching ABI. */
6180 }
6181 }
6183 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
6184 But in the case of vector types, it is some vector mode.
6186 When we have only some of our vector isa extensions enabled, then there
6187 are some modes for which vector_mode_supported_p is false. For these
6188 modes, the generic vector support in gcc will choose some non-vector mode
6189 in order to implement the type. By computing the natural mode, we'll
6190 select the proper ABI location for the operand and not depend on whatever
6191 the middle-end decides to do with these vector types.
6193 The midde-end can't deal with the vector types > 16 bytes. In this
6194 case, we return the original mode and warn ABI change if CUM isn't
6195 NULL. */
6197 static enum machine_mode
6199 {
6203 {
6206 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
6208 {
6213 else
6216 /* Get the mode which has this inner mode and number of units. */
6220 {
6222 {
6226 && !warnedavx
6228 {
6232 }
6234 }
6236 {
6240 && !warnedsse
6242 {
6246 }
6248 }
6249 else
6251 }
6254 }
6255 }
6258 }
6260 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
6261 this may not agree with the mode that the type system has chosen for the
6262 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
6263 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
6265 static rtx
6268 {
6269 rtx tmp;
6273 else
6274 {
6278 }
6281 }
6283 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
6284 of this code is to classify each 8bytes of incoming argument by the register
6285 class and assign registers accordingly. */
6287 /* Return the union class of CLASS1 and CLASS2.
6288 See the x86-64 PS ABI for details. */
6290 static enum x86_64_reg_class
6292 {
6293 /* Rule #1: If both classes are equal, this is the resulting class. */
6297 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
6298 the other class. */
6304 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
6308 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
6316 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
6317 MEMORY is used. */
6326 /* Rule #6: Otherwise class SSE is used. */
6328 }
6330 /* Classify the argument of type TYPE and mode MODE.
6331 CLASSES will be filled by the register class used to pass each word
6332 of the operand. The number of words is returned. In case the parameter
6333 should be passed in memory, 0 is returned. As a special case for zero
6334 sized containers, classes[0] will be NO_CLASS and 1 is returned.
6336 BIT_OFFSET is used internally for handling records and specifies offset
6337 of the offset in bits modulo 256 to avoid overflow cases.
6339 See the x86-64 PS ABI for details.
6340 */
6342 static int
6345 {
6346 HOST_WIDE_INT bytes =
6348 int words
6351 /* Variable sized entities are always passed/returned in memory. */
6360 {
6362 tree field;
6365 /* On x86-64 we pass structures larger than 32 bytes on the stack. */
6372 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
6373 signalize memory class, so handle it as special case. */
6375 {
6378 }
6380 /* Classify each field of record and merge classes. */
6382 {
6384 /* And now merge the fields of structure. */
6386 {
6388 {
6394 /* Bitfields are always classified as integer. Handle them
6395 early, since later code would consider them to be
6396 misaligned integers. */
6398 {
6407 }
6408 else
6409 {
6414 /* Flexible array member is ignored. */
6421 {
6425 {
6428 "the ABI of passing struct with"
6429 " a flexible array member has"
6431 }
6433 }
6435 subclasses,
6445 }
6446 }
6447 }
6451 /* Arrays are handled as small records. */
6452 {
6459 /* The partial classes are now full classes. */
6470 }
6473 /* Unions are similar to RECORD_TYPE but offset is always 0.
6474 */
6476 {
6478 {
6486 bit_offset);
6491 }
6492 }
6497 }
6500 {
6501 /* When size > 16 bytes, if the first one isn't
6502 X86_64_SSE_CLASS or any other ones aren't
6503 X86_64_SSEUP_CLASS, everything should be passed in
6504 memory. */
6511 }
6513 /* Final merger cleanup. */
6515 {
6516 /* If one class is MEMORY, everything should be passed in
6517 memory. */
6521 /* The X86_64_SSEUP_CLASS should be always preceded by
6522 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
6526 {
6527 /* The first one should never be X86_64_SSEUP_CLASS. */
6530 }
6532 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
6533 everything should be passed in memory. */
6536 {
6539 /* The first one should never be X86_64_X87UP_CLASS. */
6542 {
6545 "the ABI of passing union with long double"
6547 }
6549 }
6550 }
6552 }
6554 /* Compute alignment needed. We align all types to natural boundaries with
6555 exception of XFmode that is aligned to 64bits. */
6557 {
6566 /* Misaligned fields are always returned in memory. */
6569 }
6571 /* for V1xx modes, just use the base mode */
6576 /* Classification of atomic types. */
6578 {
6594 {
6598 {
6601 }
6603 {
6606 }
6608 {
6612 }
6614 {
6617 }
6618 else
6620 }
6627 /* OImode shouldn't be used directly. */
6634 else
6652 else
6653 {
6657 {
6660 "the ABI of passing structure with complex float"
6662 }
6665 }
6674 /* This modes is larger than 16 bytes. */
6717 else
6721 }
6722 }
6724 /* Examine the argument and return set number of register required in each
6725 class. Return 0 iff parameter should be passed in memory. */
6726 static int
6729 {
6739 {
6761 }
6763 }
6765 /* Construct container for the argument used by GCC interface. See
6766 FUNCTION_ARG for the detailed description. */
6768 static rtx
6772 {
6773 /* The following variables hold the static issued_error state. */
6787 rtx ret;
6798 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
6799 some less clueful developer tries to use floating-point anyway. */
6801 {
6803 {
6805 {
6808 }
6809 }
6811 {
6814 }
6816 }
6818 /* Likewise, error if the ABI requires us to return values in the
6819 x87 registers and the user specified -mno-80387. */
6825 {
6827 {
6830 }
6832 }
6834 /* First construct simple cases. Avoid SCmode, since we want to use
6835 single register to pass this type. */
6838 {
6853 /* Zero sized array, struct or class. */
6857 }
6884 /* Otherwise figure out the entries of the PARALLEL. */
6886 {
6890 {
6895 /* Merge TImodes on aligned occasions here too. */
6897 tmpmode
6901 else
6903 /* We've requested 24 bytes we
6904 don't have mode for. Use DImode. */
6911 intreg++;
6919 sse_regno++;
6927 sse_regno++;
6932 {
6938 {
6940 i++;
6941 }
6942 else
6955 }
6961 sse_regno++;
6965 }
6966 }
6968 /* Empty aligned struct, union or class. */
6976 }
6978 /* Update the data in CUM to advance over an argument of mode MODE
6979 and data type TYPE. (TYPE is null for libcalls where that information
6980 may not be available.) */
6982 static void
6985 HOST_WIDE_INT words)
6986 {
6988 {
6995 /* FALLTHRU */
7006 {
7009 }
7013 /* OImode shouldn't be used directly. */
7022 /* FALLTHRU */
7038 {
7043 {
7046 }
7047 }
7057 {
7062 {
7065 }
7066 }
7068 }
7069 }
7071 static void
7074 {
7077 /* Unnamed 256bit vector mode parameters are passed on stack. */
7083 {
7088 }
7089 else
7090 {
7094 }
7095 }
7097 static void
7099 HOST_WIDE_INT words)
7100 {
7101 /* Otherwise, this should be passed indirect. */
7106 {
7109 }
7110 }
7112 /* Update the data in CUM to advance over an argument of mode MODE and
7113 data type TYPE. (TYPE is null for libcalls where that information
7114 may not be available.) */
7116 static void
7119 {
7125 else
7136 else
7138 }
7140 /* Define where to put the arguments to a function.
7141 Value is zero to push the argument on the stack,
7142 or a hard register in which to store the argument.
7144 MODE is the argument's machine mode.
7145 TYPE is the data type of the argument (as a tree).
7146 This is null for libcalls where that information may
7147 not be available.
7148 CUM is a variable of type CUMULATIVE_ARGS which gives info about
7149 the preceding args and about the function being called.
7150 NAMED is nonzero if this argument is a named parameter
7151 (otherwise it is an extra parameter matching an ellipsis). */
7153 static rtx
7157 {
7160 /* Avoid the AL settings for the Unix64 ABI. */
7165 {
7172 /* FALLTHRU */
7178 {
7181 /* Fastcall allocates the first two DWORD (SImode) or
7182 smaller arguments to ECX and EDX if it isn't an
7183 aggregate type . */
7185 {
7191 /* ECX not EAX is the first allocated register. */
7194 }
7196 }
7205 /* FALLTHRU */
7207 /* In 32bit, we pass TImode in xmm registers. */
7215 {
7217 {
7221 }
7225 }
7229 /* OImode shouldn't be used directly. */
7239 {
7243 }
7253 {
7255 {
7259 }
7263 }
7265 }
7268 }
7270 static rtx
7273 {
7274 /* Handle a hidden AL argument containing number of registers
7275 for varargs x86-64 functions. */
7279 ? X86_64_SSE_REGPARM_MAX
7284 {
7294 /* Unnamed 256bit vector mode parameters are passed on stack. */
7298 }
7304 }
7306 static rtx
7309 HOST_WIDE_INT bytes)
7310 {
7313 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
7314 We use value of -2 to specify that current function call is MSABI. */
7318 /* If we've run out of registers, it goes on the stack. */
7324 /* Only floating point modes are passed in anything but integer regs. */
7326 {
7329 else
7330 {
7333 /* Unnamed floating parameters are passed in both the
7334 SSE and integer registers. */
7340 }
7341 }
7342 /* Handle aggregated types passed in register. */
7344 {
7349 }
7352 }
7354 /* Return where to put the arguments to a function.
7355 Return zero to push the argument on the stack, or a hard register in which to store the argument.
7357 MODE is the argument's machine mode. TYPE is the data type of the
7358 argument. It is null for libcalls where that information may not be
7359 available. CUM gives information about the preceding args and about
7360 the function being called. NAMED is nonzero if this argument is a
7361 named parameter (otherwise it is an extra parameter matching an
7362 ellipsis). */
7364 static rtx
7367 {
7371 rtx arg;
7375 else
7379 /* To simplify the code below, represent vector types with a vector mode
7380 even if MMX/SSE are not active. */
7388 else
7392 }
7394 /* A C expression that indicates when an argument must be passed by
7395 reference. If nonzero for an argument, a copy of that argument is
7396 made in memory and a pointer to the argument is passed instead of
7397 the argument itself. The pointer is passed in whatever way is
7398 appropriate for passing a pointer to that type. */
7400 static bool
7404 {
7407 /* See Windows x64 Software Convention. */
7409 {
7412 {
7413 /* Arrays are passed by reference. */
7418 {
7419 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
7420 are passed by reference. */
7422 }
7423 }
7425 /* __m128 is passed by reference. */
7431 }
7432 }
7437 }
7439 /* Return true when TYPE should be 128bit aligned for 32bit argument
7440 passing ABI. XXX: This function is obsolete and is only used for
7441 checking psABI compatibility with previous versions of GCC. */
7443 static bool
7445 {
7457 {
7458 /* Walk the aggregates recursively. */
7460 {
7464 {
7465 tree field;
7467 /* Walk all the structure fields. */
7469 {
7473 }
7475 }
7478 /* Just for use if some languages passes arrays by value. */
7485 }
7486 }
7488 }
7490 /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
7491 XXX: This function is obsolete and is only used for checking psABI
7492 compatibility with previous versions of GCC. */
7494 static unsigned int
7497 {
7498 /* In 32bit, only _Decimal128 and __float128 are aligned to their
7499 natural boundaries. */
7501 {
7502 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
7503 make an exception for SSE modes since these require 128bit
7504 alignment.
7506 The handling here differs from field_alignment. ICC aligns MMX
7507 arguments to 4 byte boundaries, while structure fields are aligned
7508 to 8 byte boundaries. */
7510 {
7513 }
7514 else
7515 {
7518 }
7519 }
7523 }
7525 /* Return true when TYPE should be 128bit aligned for 32bit argument
7526 passing ABI. */
7528 static bool
7530 {
7540 {
7541 /* Walk the aggregates recursively. */
7543 {
7547 {
7548 tree field;
7550 /* Walk all the structure fields. */
7552 field;
7554 {
7558 }
7560 }
7563 /* Just for use if some languages passes arrays by value. */
7570 }
7571 }
7572 else
7576 }
7578 /* Gives the alignment boundary, in bits, of an argument with the
7579 specified mode and type. */
7581 static unsigned int
7583 {
7586 {
7587 /* Since the main variant type is used for call, we convert it to
7588 the main variant type. */
7591 }
7592 else
7596 else
7597 {
7602 {
7603 /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
7605 {
7608 }
7614 }
7617 && !warned
7619 saved_align))
7620 {
7623 "The ABI for passing parameters with %d-byte"
7626 }
7627 }
7630 }
7632 /* Return true if N is a possible register number of function value. */
7634 static bool
7636 {
7638 {
7643 /* TODO: The function should depend on current function ABI but
7644 builtins.c would need updating then. Therefore we use the
7645 default ABI. */
7657 }
7660 }
7662 /* Define how to find the value returned by a function.
7663 VALTYPE is the data type of the value (as a tree).
7664 If the precise function being called is known, FUNC is its FUNCTION_DECL;
7665 otherwise, FUNC is 0. */
7667 static rtx
7670 {
7673 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
7674 we normally prevent this case when mmx is not available. However
7675 some ABIs may require the result to be returned like DImode. */
7679 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
7680 we prevent this case when sse is not available. However some ABIs
7681 may require the result to be returned like integer TImode. */
7686 /* 32-byte vector modes in %ymm0. */
7690 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
7693 else
7694 /* Most things go in %eax. */
7697 /* Override FP return register with %xmm0 for local functions when
7698 SSE math is enabled or for functions with sseregparm attribute. */
7700 {
7705 }
7707 /* OImode shouldn't be used directly. */
7711 }
7713 static rtx
7715 const_tree valtype)
7716 {
7717 rtx ret;
7719 /* Handle libcalls, which don't provide a type node. */
7721 {
7725 {
7744 }
7747 }
7749 {
7750 /* Pointers are always returned in word_mode. */
7752 }
7758 /* For zero sized structures, construct_container returns NULL, but we
7759 need to keep rest of compiler happy by returning meaningful value. */
7764 }
7766 static rtx
7768 const_tree valtype)
7769 {
7773 {
7775 {
7794 }
7795 }
7797 }
7799 static rtx
7802 {
7814 else
7816 }
7818 static rtx
7821 {
7827 }
7829 /* Pointer function arguments and return values are promoted to
7830 word_mode. */
7832 static enum machine_mode
7836 {
7838 {
7841 }
7843 for_return);
7844 }
7846 /* Return true if a structure, union or array with MODE containing FIELD
7847 should be accessed using BLKmode. */
7849 static bool
7851 {
7852 /* Union with XFmode must be in BLKmode. */
7856 }
7858 rtx
7860 {
7862 }
7864 /* Return true iff type is returned in memory. */
7866 static bool ATTRIBUTE_UNUSED
7868 {
7869 HOST_WIDE_INT size;
7880 {
7881 /* User-created vectors small enough to fit in EAX. */
7885 /* MMX/3dNow values are returned in MM0,
7886 except when it doesn't exits or the ABI prescribes otherwise. */
7890 /* SSE values are returned in XMM0, except when it doesn't exist. */
7894 /* AVX values are returned in YMM0, except when it doesn't exist. */
7897 }
7905 /* OImode shouldn't be used directly. */
7909 }
7911 static bool ATTRIBUTE_UNUSED
7913 {
7916 }
7918 static bool ATTRIBUTE_UNUSED
7920 {
7923 /* __m128 is returned in xmm0. */
7930 /* Otherwise, the size must be exactly in [1248]. */
7932 }
7934 static bool
7936 {
7937 #ifdef SUBTARGET_RETURN_IN_MEMORY
7939 #else
7943 {
7946 else
7948 }
7949 else
7951 #endif
7952 }
7954 /* When returning SSE vector types, we have a choice of either
7955 (1) being abi incompatible with a -march switch, or
7956 (2) generating an error.
7957 Given no good solution, I think the safest thing is one warning.
7958 The user won't be able to use -Werror, but....
7960 Choose the STRUCT_VALUE_RTX hook because that's (at present) only
7961 called in response to actually generating a caller or callee that
7962 uses such a type. As opposed to TARGET_RETURN_IN_MEMORY, which is called
7963 via aggregate_value_p for general type probing from tree-ssa. */
7965 static rtx
7967 {
7971 {
7972 /* Look at the return type of the function, not the function type. */
7976 {
7979 {
7983 }
7984 }
7987 {
7989 {
7993 }
7994 }
7995 }
7998 }
8000 \f
8001 /* Create the va_list data type. */
8003 /* Returns the calling convention specific va_list date type.
8004 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
8006 static tree
8008 {
8011 /* For i386 we use plain pointer to argument area. */
8021 unsigned_type_node);
8024 unsigned_type_node);
8027 ptr_type_node);
8030 ptr_type_node);
8049 /* The correct type is an array type of one element. */
8051 }
8053 /* Setup the builtin va_list data type and for 64-bit the additional
8054 calling convention specific va_list data types. */
8056 static tree
8058 {
8061 /* Initialize abi specific va_list builtin types. */
8063 {
8064 tree t;
8066 {
8071 }
8072 else
8073 {
8078 }
8080 {
8085 }
8086 else
8087 {
8092 }
8093 }
8096 }
8098 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
8100 static void
8102 {
8104 alias_set_type set;
8107 /* GPR size of varargs save area. */
8110 else
8113 /* FPR size of varargs save area. We don't need it if we don't pass
8114 anything in SSE registers. */
8117 else
8131 {
8139 }
8142 {
8146 /* Now emit code to save SSE registers. The AX parameter contains number
8147 of SSE parameter registers used to call this function, though all we
8148 actually check here is the zero/non-zero status. */
8153 label));
8155 /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
8156 we used movdqa (i.e. TImode) instead? Perhaps even better would
8157 be if we could determine the real mode of the data, via a hook
8158 into pass_stdarg. Ignore all that for now. */
8168 {
8177 }
8180 }
8181 }
8183 static void
8185 {
8189 /* Reset to zero, as there might be a sysv vaarg used
8190 before. */
8195 {
8206 }
8207 }
8209 static void
8213 {
8215 CUMULATIVE_ARGS next_cum;
8216 tree fntype;
8218 /* This argument doesn't appear to be used anymore. Which is good,
8219 because the old code here didn't suppress rtl generation. */
8227 /* For varargs, we do not want to skip the dummy va_dcl argument.
8228 For stdargs, we do want to skip the last named argument. */
8236 else
8238 }
8240 /* Checks if TYPE is of kind va_list char *. */
8242 static bool
8244 {
8245 tree canonic;
8247 /* For 32-bit it is always true. */
8253 }
8255 /* Implement va_start. */
8257 static void
8259 {
8263 tree type;
8264 rtx ovf_rtx;
8268 {
8271 /* When we are splitting the stack, we can't refer to the stack
8272 arguments using internal_arg_pointer, because they may be on
8273 the old stack. The split stack prologue will arrange to
8274 leave a pointer to the old stack arguments in a scratch
8275 register, which we here copy to a pseudo-register. The split
8276 stack prologue can't set the pseudo-register directly because
8277 it (the prologue) runs before any registers have been saved. */
8281 {
8295 }
8296 }
8298 /* Only 64bit target needs something special. */
8300 {
8303 else
8304 {
8313 }
8315 }
8324 /* The following should be folded into the MEM_REF offset. */
8334 /* Count number of gp and fp argument registers used. */
8340 {
8346 }
8349 {
8355 }
8357 /* Find the overflow area. */
8361 else
8371 {
8372 /* Find the register save area.
8373 Prologue of the function save it right above stack frame. */
8381 }
8382 }
8384 /* Implement va_arg. */
8386 static tree
8389 {
8396 rtx container;
8398 tree ptrtype;
8402 /* Only 64bit target needs something special. */
8426 {
8433 /* Unnamed 256bit vector mode parameters are passed on stack. */
8435 {
8438 }
8446 }
8448 /* Pull the value out of the saved registers. */
8453 {
8467 /* In case we are passing structure, verify that it is consecutive block
8468 on the register save area. If not we need to do moves. */
8470 {
8471 /* Verify that all registers are strictly consecutive */
8473 {
8477 {
8482 }
8483 }
8484 else
8485 {
8489 {
8494 }
8495 }
8496 }
8498 {
8501 }
8502 else
8503 {
8506 }
8508 /* First ensure that we fit completely in registers. */
8510 {
8517 }
8519 {
8527 }
8529 /* Compute index to start of area used for integer regs. */
8531 {
8532 /* int_addr = gpr + sav; */
8535 }
8537 {
8538 /* sse_addr = fpr + sav; */
8541 }
8543 {
8547 /* addr = &temp; */
8552 {
8556 tree piece_type;
8557 tree addr_type;
8558 tree daddr_type;
8567 {
8572 }
8576 else
8583 {
8586 }
8587 else
8588 {
8591 }
8598 {
8603 }
8604 else
8605 {
8606 tree copy
8611 }
8613 }
8614 }
8617 {
8621 }
8624 {
8628 }
8633 }
8635 /* ... otherwise out of the overflow area. */
8637 /* When we align parameter on stack for caller, if the parameter
8638 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
8639 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
8640 here with caller. */
8645 /* Care for on-stack alignment if needed. */
8648 else
8649 {
8654 }
8671 }
8672 \f
8673 /* Return true if OPNUM's MEM should be matched
8674 in movabs* patterns. */
8676 bool
8678 {
8690 }
8691 \f
8692 /* Initialize the table of extra 80387 mathematical constants. */
8694 static void
8696 {
8698 {
8704 };
8708 {
8710 /* Ensure each constant is rounded to XFmode precision. */
8713 }
8716 }
8718 /* Return non-zero if the constant is something that
8719 can be loaded with a special instruction. */
8721 int
8723 {
8726 REAL_VALUE_TYPE r;
8738 /* For XFmode constants, try to find a special 80387 instruction when
8739 optimizing for size or on those CPUs that benefit from them. */
8742 {
8751 }
8753 /* Load of the constant -0.0 or -1.0 will be split as
8754 fldz;fchs or fld1;fchs sequence. */
8761 }
8763 /* Return the opcode of the special instruction to be used to load
8764 the constant X. */
8768 {
8770 {
8790 }
8791 }
8793 /* Return the CONST_DOUBLE representing the 80387 constant that is
8794 loaded by the specified special instruction. The argument IDX
8795 matches the return value from standard_80387_constant_p. */
8797 rtx
8799 {
8806 {
8817 }
8820 XFmode);
8821 }
8823 /* Return 1 if X is all 0s and 2 if x is all 1s
8824 in supported SSE/AVX vector mode. */
8826 int
8828 {
8835 {
8850 }
8853 }
8855 /* Return the opcode of the special instruction to be used to load
8856 the constant X. */
8860 {
8862 {
8865 {
8882 }
8887 else
8892 }
8894 }
8896 /* Returns true if OP contains a symbol reference */
8898 bool
8900 {
8909 {
8911 {
8917 }
8921 }
8924 }
8926 /* Return true if it is appropriate to emit `ret' instructions in the
8927 body of a function. Do this only if the epilogue is simple, needing a
8928 couple of insns. Prior to reloading, we can't tell how many registers
8929 must be saved, so return false then. Return false if there is no frame
8930 marker to de-allocate. */
8932 bool
8934 {
8940 /* Don't allow more than 32k pop, since that's all we can do
8941 with one instruction. */
8948 }
8949 \f
8950 /* Value should be nonzero if functions must have frame pointers.
8951 Zero means the frame pointer need not be set up (and parms may
8952 be accessed via the stack pointer) in functions that seem suitable. */
8954 static bool
8956 {
8957 /* If we accessed previous frames, then the generated code expects
8958 to be able to access the saved ebp value in our frame. */
8962 /* Several x86 os'es need a frame pointer for other reasons,
8963 usually pertaining to setjmp. */
8967 /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
8971 /* Win64 SEH, very large frames need a frame-pointer as maximum stack
8972 allocation is 4GB. */
8976 /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
8977 turns off the frame pointer by default. Turn it back on now if
8978 we've not got a leaf function. */
8988 }
8990 /* Record that the current function accesses previous call frames. */
8992 void
8994 {
8996 }
8997 \f
8998 #ifndef USE_HIDDEN_LINKONCE
8999 # if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
9000 # define USE_HIDDEN_LINKONCE 1
9001 # else
9002 # define USE_HIDDEN_LINKONCE 0
9003 # endif
9004 #endif
9008 /* Fills in the label name that should be used for a pc thunk for
9009 the given register. */
9011 static void
9013 {
9018 else
9020 }
9023 /* This function generates code for -fpic that loads %ebx with
9024 the return address of the caller and then returns. */
9026 static void
9028 {
9033 {
9035 tree decl;
9051 #if TARGET_MACHO
9053 {
9062 }
9063 else
9064 #endif
9066 {
9077 }
9078 else
9079 {
9082 }
9088 /* Make sure unwind info is emitted for the thunk if needed. */
9091 /* Pad stack IP move with 4 instructions (two NOPs count
9092 as one instruction). */
9094 {
9099 }
9110 }
9114 }
9116 /* Emit code for the SET_GOT patterns. */
9120 {
9126 {
9127 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
9132 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
9133 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
9134 an unadorned address. */
9139 }
9144 {
9146 /* We don't need a pic base, we're not producing pic. */
9153 }
9154 else
9155 {
9164 #if TARGET_MACHO
9165 /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
9166 This is what will be referenced by the Mach-O PIC subsystem. */
9170 /* When we are restoring the pic base at the site of a nonlocal label,
9171 and we decided to emit the pic base above, we will still output a
9172 local label used for calculating the correction offset (even though
9173 the offset will be 0 in that case). */
9177 #endif
9178 }
9184 }
9186 /* Generate an "push" pattern for input ARG. */
9188 static rtx
9190 {
9203 stack_pointer_rtx)),
9204 arg);
9205 }
9207 /* Generate an "pop" pattern for input ARG. */
9209 static rtx
9211 {
9216 arg,
9219 stack_pointer_rtx)));
9220 }
9222 /* Return >= 0 if there is an unused call-clobbered register available
9223 for the entire function. */
9225 static unsigned int
9227 {
9231 {
9233 /* Can't use the same register for both PIC and DRAP. */
9236 else
9241 }
9244 }
9246 /* Return TRUE if we need to save REGNO. */
9248 static bool
9250 {
9261 {
9264 {
9270 }
9271 }
9280 }
9282 /* Return number of saved general prupose registers. */
9284 static int
9286 {
9292 nregs ++;
9294 }
9296 /* Return number of saved SSE registrers. */
9298 static int
9300 {
9308 nregs ++;
9310 }
9312 /* Given FROM and TO register numbers, say whether this elimination is
9313 allowed. If stack alignment is needed, we can only replace argument
9314 pointer with hard frame pointer, or replace frame pointer with stack
9315 pointer. Otherwise, frame pointer elimination is automatically
9316 handled and all other eliminations are valid. */
9318 static bool
9320 {
9326 else
9328 }
9330 /* Return the offset between two registers, one to be eliminated, and the other
9331 its replacement, at the start of a routine. */
9333 HOST_WIDE_INT
9335 {
9344 else
9345 {
9353 }
9354 }
9356 /* In a dynamically-aligned function, we can't know the offset from
9357 stack pointer to frame pointer, so we must ensure that setjmp
9358 eliminates fp against the hard fp (%ebp) rather than trying to
9359 index from %esp up to the top of the frame across a gap that is
9360 of unknown (at compile-time) size. */
9361 static rtx
9363 {
9365 }
9367 /* When using -fsplit-stack, the allocation routines set a field in
9368 the TCB to the bottom of the stack plus this much space, measured
9369 in bytes. */
9371 #define SPLIT_STACK_AVAILABLE 256
9373 /* Fill structure ix86_frame about frame of currently computed function. */
9375 static void
9377 {
9379 HOST_WIDE_INT offset;
9382 HOST_WIDE_INT to_allocate;
9390 /* 64-bit MS ABI seem to require stack alignment to be always 16 except for
9391 function prologues and leaf. */
9395 {
9400 }
9406 /* For SEH we have to limit the amount of code movement into the prologue.
9407 At present we do this via a BLOCKAGE, at which point there's very little
9408 scheduling that can be done, which means that there's very little point
9409 in doing anything except PUSHs. */
9413 /* During reload iteration the amount of registers saved can change.
9414 Recompute the value as needed. Do not recompute when amount of registers
9415 didn't change as reload does multiple calls to the function and does not
9416 expect the decision to change within single iteration. */
9419 {
9425 /* The fast prologue uses move instead of push to save registers. This
9426 is significantly longer, but also executes faster as modern hardware
9427 can execute the moves in parallel, but can't do that for push/pop.
9429 Be careful about choosing what prologue to emit: When function takes
9430 many instructions to execute we may use slow version as well as in
9431 case function is known to be outside hot spot (this is known with
9432 feedback only). Weight the size of function by number of registers
9433 to save as it is cheap to use one or two push instructions but very
9434 slow to use many of them. */
9438 || (flag_branch_probabilities
9441 else
9444 }
9446 frame->save_regs_using_mov
9448 /* If static stack checking is enabled and done with probes,
9449 the registers need to be saved before allocating the frame. */
9452 /* Skip return address. */
9455 /* Skip pushed static chain. */
9459 /* Skip saved base pointer. */
9464 /* The traditional frame pointer location is at the top of the frame. */
9467 /* Register save area */
9471 /* On SEH target, registers are pushed just before the frame pointer
9472 location. */
9476 /* Align and set SSE register save area. */
9478 {
9479 /* The only ABI that has saved SSE registers (Win64) also has a
9480 16-byte aligned default stack, and thus we don't need to be
9481 within the re-aligned local stack frame to save them. */
9485 }
9488 /* The re-aligned stack starts here. Values before this point are not
9489 directly comparable with values below this point. In order to make
9490 sure that no value happens to be the same before and after, force
9491 the alignment computation below to add a non-zero value. */
9495 /* Va-arg area */
9499 /* Align start of frame for local function. */
9508 /* Frame pointer points here. */
9513 /* Add outgoing arguments area. Can be skipped if we eliminated
9514 all the function calls as dead code.
9515 Skipping is however impossible when function calls alloca. Alloca
9516 expander assumes that last crtl->outgoing_args_size
9517 of stack frame are unused. */
9521 {
9524 }
9525 else
9528 /* Align stack boundary. Only needed if we're calling another function
9529 or using alloca. */
9534 /* We've reached end of stack frame. */
9537 /* Size prologue needs to allocate. */
9548 {
9554 }
9555 else
9559 /* The SEH frame pointer location is near the bottom of the frame.
9560 This is enforced by the fact that the difference between the
9561 stack pointer and the frame pointer is limited to 240 bytes in
9562 the unwind data structure. */
9564 {
9565 HOST_WIDE_INT diff;
9567 /* If we can leave the frame pointer where it is, do so. Also, returns
9568 the establisher frame for __builtin_frame_address (0). */
9573 {
9574 /* Ideally we'd determine what portion of the local stack frame
9575 (within the constraint of the lowest 240) is most heavily used.
9576 But without that complication, simply bias the frame pointer
9577 by 128 bytes so as to maximize the amount of the local stack
9578 frame that is addressable with 8-bit offsets. */
9580 }
9581 }
9582 }
9584 /* This is semi-inlined memory_address_length, but simplified
9585 since we know that we're always dealing with reg+offset, and
9586 to avoid having to create and discard all that rtl. */
9590 {
9594 {
9595 /* EBP and R13 cannot be encoded without an offset. */
9597 }
9601 /* ESP and R12 must be encoded with a SIB byte. */
9603 len++;
9606 }
9608 /* Return an RTX that points to CFA_OFFSET within the stack frame.
9609 The valid base registers are taken from CFUN->MACHINE->FS. */
9611 static rtx
9613 {
9619 {
9620 /* Choose the base register most likely to allow the most scheduling
9621 opportunities. Generally FP is valid throughout the function,
9622 while DRAP must be reloaded within the epilogue. But choose either
9623 over the SP due to increased encoding size. */
9626 {
9629 }
9631 {
9634 }
9636 {
9639 }
9640 }
9641 else
9642 {
9643 HOST_WIDE_INT toffset;
9646 /* Choose the base register with the smallest address encoding.
9647 With a tie, choose FP > DRAP > SP. */
9649 {
9653 }
9655 {
9659 {
9663 }
9664 }
9666 {
9670 {
9674 }
9675 }
9676 }
9680 }
9682 /* Emit code to save registers in the prologue. */
9684 static void
9686 {
9688 rtx insn;
9692 {
9695 }
9696 }
9698 /* Emit a single register save at CFA - CFA_OFFSET. */
9700 static void
9702 HOST_WIDE_INT cfa_offset)
9703 {
9711 /* For SSE saves, we need to indicate the 128-bit alignment. */
9722 /* When saving registers into a re-aligned local stack frame, avoid
9723 any tricky guessing by dwarf2out. */
9725 {
9729 {
9730 /* A bit of a hack. We force the DRAP register to be saved in
9731 the re-aligned stack frame, which provides us with a copy
9732 of the CFA that will last past the prologue. Install it. */
9738 }
9739 else
9740 {
9741 /* The frame pointer is a stable reference within the
9742 aligned frame. Use it. */
9749 }
9750 }
9752 /* The memory may not be relative to the current CFA register,
9753 which means that we may need to generate a new pattern for
9754 use by the unwind info. */
9756 {
9761 }
9762 }
9764 /* Emit code to save registers using MOV insns.
9765 First register is stored at CFA - CFA_OFFSET. */
9766 static void
9768 {
9773 {
9776 }
9777 }
9779 /* Emit code to save SSE registers using MOV insns.
9780 First register is stored at CFA - CFA_OFFSET. */
9781 static void
9783 {
9788 {
9791 }
9792 }
9796 /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
9797 manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
9798 Don't add the note if the previously saved value will be left untouched
9799 within stack red-zone till return, as unwinders can find the same value
9800 in the register and on the stack. */
9802 static void
9804 {
9810 {
9813 }
9814 else
9815 queued_cfa_restores
9817 }
9819 /* Add queued REG_CFA_RESTORE notes if any to INSN. */
9821 static void
9823 {
9824 rtx last;
9828 ;
9833 }
9835 /* Expand prologue or epilogue stack adjustment.
9836 The pattern exist to put a dependency on all ebp-based memory accesses.
9837 STYLE should be negative if instructions should be marked as frame related,
9838 zero if %r11 register is live and cannot be freely used and positive
9839 otherwise. */
9841 static void
9844 {
9846 rtx insn;
9853 else
9854 {
9855 rtx tmp;
9856 /* r11 is used by indirect sibcall return as well, set before the
9857 epilogue and used after the epilogue. */
9860 else
9861 {
9865 }
9871 }
9878 {
9879 rtx r;
9889 }
9891 {
9894 {
9898 }
9899 }
9902 {
9907 {
9910 }
9912 {
9915 }
9916 else
9917 {
9918 /* Else there are two possibilities: SP itself, which we set
9919 up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
9920 taken care of this by hand along the eh_return path. */
9923 }
9927 }
9928 }
9930 /* Find an available register to be used as dynamic realign argument
9931 pointer regsiter. Such a register will be written in prologue and
9932 used in begin of body, so it must not be
9933 1. parameter passing register.
9934 2. GOT pointer.
9935 We reuse static-chain register if it is available. Otherwise, we
9936 use DI for i386 and R13 for x86-64. We chose R13 since it has
9937 shorter encoding.
9939 Return: the regno of chosen register. */
9941 static unsigned int
9943 {
9947 {
9948 /* Use R13 for nested function or function need static chain.
9949 Since function with tail call may use any caller-saved
9950 registers in epilogue, DRAP must not use caller-saved
9951 register in such case. */
9956 }
9957 else
9958 {
9959 /* Use DI for nested function or function need static chain.
9960 Since function with tail call may use any caller-saved
9961 registers in epilogue, DRAP must not use caller-saved
9962 register in such case. */
9966 /* Reuse static chain register if it isn't used for parameter
9967 passing. */
9969 {
9973 }
9975 }
9976 }
9978 /* Return minimum incoming stack alignment. */
9980 static unsigned int
9982 {
9985 /* Prefer the one specified at command line. */
9988 /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
9989 if -mstackrealign is used, it isn't used for sibcall check and
9990 estimated stack alignment is 128bit. */
9992 && !TARGET_64BIT
9993 && ix86_force_align_arg_pointer
9996 else
9999 /* Incoming stack alignment can be changed on individual functions
10000 via force_align_arg_pointer attribute. We use the smallest
10001 incoming stack boundary. */
10007 /* The incoming stack frame has to be aligned at least at
10008 parm_stack_boundary. */
10012 /* Stack at entrance of main is aligned by runtime. We use the
10013 smallest incoming stack boundary. */
10021 }
10023 /* Update incoming stack boundary and estimated stack alignment. */
10025 static void
10027 {
10028 ix86_incoming_stack_boundary
10031 /* x86_64 vararg needs 16byte stack alignment for register save
10032 area. */
10037 }
10039 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
10040 needed or an rtx for DRAP otherwise. */
10042 static rtx
10044 {
10049 {
10050 /* Assign DRAP to vDRAP and returns vDRAP */
10052 rtx drap_vreg;
10053 rtx arg_ptr;
10066 {
10069 }
10071 }
10072 else
10074 }
10076 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
10078 static rtx
10080 {
10082 }
10085 rtx reg;
10087 };
10089 /* Return a short-lived scratch register for use on function entry.
10090 In 32-bit mode, it is valid only after the registers are saved
10091 in the prologue. This register must be released by means of
10092 release_scratch_register_on_entry once it is dead. */
10094 static void
10096 {
10102 {
10103 /* We always use R11 in 64-bit mode. */
10105 }
10106 else
10107 {
10109 bool fastcall_p
10111 bool thiscall_p
10115 int drap_regno
10118 /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
10119 for the static chain register. */
10123 /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
10124 for the static chain register. */
10129 /* ecx is the static chain register. */
10131 && !static_chain_p
10136 /* esi is the static chain register. */
10142 else
10143 {
10146 }
10147 }
10151 {
10154 }
10155 }
10157 /* Release a scratch register obtained from the preceding function. */
10159 static void
10161 {
10163 {
10167 /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */
10173 }
10174 }
10176 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
10178 /* Emit code to adjust the stack pointer by SIZE bytes while probing it. */
10180 static void
10182 {
10183 /* We skip the probe for the first interval + a small dope of 4 words and
10184 probe that many bytes past the specified size to maintain a protection
10185 area at the botton of the stack. */
10189 /* See if we have a constant small number of probes to generate. If so,
10190 that's the easy case. The run-time loop is made up of 11 insns in the
10191 generic case while the compile-time loop is made up of 3+2*(n-1) insns
10192 for n # of intervals. */
10194 {
10198 /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
10199 values of N from 1 until it exceeds SIZE. If only one probe is
10200 needed, this will not generate any code. Then adjust and probe
10201 to PROBE_INTERVAL + SIZE. */
10203 {
10205 {
10208 }
10209 else
10216 }
10220 else
10228 /* Adjust back to account for the additional first interval. */
10232 }
10234 /* Otherwise, do the same as above, but in a loop. Note that we must be
10235 extra careful with variables wrapping around because we might be at
10236 the very top (or the very bottom) of the address space and we have
10237 to be able to handle this case properly; in particular, we use an
10238 equality test for the loop condition. */
10239 else
10240 {
10241 HOST_WIDE_INT rounded_size;
10247 /* Step 1: round SIZE to the previous multiple of the interval. */
10252 /* Step 2: compute initial and final value of the loop counter. */
10254 /* SP = SP_0 + PROBE_INTERVAL. */
10259 /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
10263 stack_pointer_rtx)));
10266 /* Step 3: the loop
10268 while (SP != LAST_ADDR)
10269 {
10270 SP = SP + PROBE_INTERVAL
10271 probe at SP
10272 }
10274 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
10275 values of N from 1 until it is equal to ROUNDED_SIZE. */
10280 /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
10281 assert at compile-time that SIZE is equal to ROUNDED_SIZE. */
10284 {
10289 }
10291 /* Adjust back to account for the additional first interval. */
10297 }
10301 /* Even if the stack pointer isn't the CFA register, we need to correctly
10302 describe the adjustments made to it, in particular differentiate the
10303 frame-related ones from the frame-unrelated ones. */
10305 {
10318 }
10320 /* Make sure nothing is scheduled before we are done. */
10322 }
10324 /* Adjust the stack pointer up to REG while probing it. */
10328 {
10338 /* Jump to END_LAB if SP == LAST_ADDR. */
10346 /* SP = SP + PROBE_INTERVAL. */
10350 /* Probe at SP. */
10361 }
10363 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
10364 inclusive. These are offsets from the current stack pointer. */
10366 static void
10368 {
10369 /* See if we have a constant small number of probes to generate. If so,
10370 that's the easy case. The run-time loop is made up of 7 insns in the
10371 generic case while the compile-time loop is made up of n insns for n #
10372 of intervals. */
10374 {
10375 HOST_WIDE_INT i;
10377 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
10378 it exceeds SIZE. If only one probe is needed, this will not
10379 generate any code. Then probe at FIRST + SIZE. */
10386 }
10388 /* Otherwise, do the same as above, but in a loop. Note that we must be
10389 extra careful with variables wrapping around because we might be at
10390 the very top (or the very bottom) of the address space and we have
10391 to be able to handle this case properly; in particular, we use an
10392 equality test for the loop condition. */
10393 else
10394 {
10401 /* Step 1: round SIZE to the previous multiple of the interval. */
10406 /* Step 2: compute initial and final value of the loop counter. */
10408 /* TEST_OFFSET = FIRST. */
10411 /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
10415 /* Step 3: the loop
10417 while (TEST_ADDR != LAST_ADDR)
10418 {
10419 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
10420 probe at TEST_ADDR
10421 }
10423 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
10424 until it is equal to ROUNDED_SIZE. */
10429 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
10430 that SIZE is equal to ROUNDED_SIZE. */
10435 stack_pointer_rtx,
10440 }
10442 /* Make sure nothing is scheduled before we are done. */
10444 }
10446 /* Probe a range of stack addresses from REG to END, inclusive. These are
10447 offsets from the current stack pointer. */
10451 {
10461 /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */
10469 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
10473 /* Probe at TEST_ADDR. */
10486 }
10488 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
10489 to be generated in correct form. */
10490 static void
10492 {
10493 /* Check if stack realign is really needed after reload, and
10494 stores result in cfun */
10495 unsigned int incoming_stack_boundary
10504 {
10505 /* After stack_realign_needed is finalized, we can't no longer
10506 change it. */
10509 }
10511 /* If the only reason for frame_pointer_needed is that we conservatively
10512 assumed stack realignment might be needed, but in the end nothing that
10513 needed the stack alignment had been spilled, clear frame_pointer_needed
10514 and say we don't need stack realignment. */
10517 && frame_pointer_needed
10519 && flag_omit_frame_pointer
10521 && !ix86_current_function_calls_tls_descriptor
10530 {
10532 basic_block bb;
10539 HARD_FRAME_POINTER_REGNUM);
10541 {
10542 rtx insn;
10546 set_up_by_prologue))
10547 {
10551 }
10552 }
10566 }
10570 }
10572 /* Expand the prologue into a bunch of separate insns. */
10574 void
10576 {
10581 HOST_WIDE_INT allocate;
10587 /* DRAP should not coexist with stack_realign_fp */
10592 /* Initialize CFA state for before the prologue. */
10596 /* Track SP offset to the CFA. We continue tracking this after we've
10597 swapped the CFA register away from SP. In the case of re-alignment
10598 this is fudged; we're interested to offsets within the local frame. */
10605 {
10606 /* We should have already generated an error for any use of
10607 ms_hook on a nested function. */
10610 /* Check if profiling is active and we shall use profiling before
10611 prologue variant. If so sorry. */
10616 /* In ix86_asm_output_function_label we emitted:
10617 8b ff movl.s %edi,%edi
10618 55 push %ebp
10619 8b ec movl.s %esp,%ebp
10621 This matches the hookable function prologue in Win32 API
10622 functions in Microsoft Windows XP Service Pack 2 and newer.
10623 Wine uses this to enable Windows apps to hook the Win32 API
10624 functions provided by Wine.
10626 What that means is that we've already set up the frame pointer. */
10630 {
10633 /* We've decided to use the frame pointer already set up.
10634 Describe this to the unwinder by pretending that both
10635 push and mov insns happen right here.
10637 Putting the unwind info here at the end of the ms_hook
10638 is done so that we can make absolutely certain we get
10639 the required byte sequence at the start of the function,
10640 rather than relying on an assembler that can produce
10641 the exact encoding required.
10643 However it does mean (in the unpatched case) that we have
10644 a 1 insn window where the asynchronous unwind info is
10645 incorrect. However, if we placed the unwind info at
10646 its correct location we would have incorrect unwind info
10647 in the patched case. Which is probably all moot since
10648 I don't expect Wine generates dwarf2 unwind info for the
10649 system libraries that use this feature. */
10655 stack_pointer_rtx);
10663 /* Note that gen_push incremented m->fs.cfa_offset, even
10664 though we didn't emit the push insn here. */
10668 }
10669 else
10670 {
10671 /* The frame pointer is not needed so pop %ebp again.
10672 This leaves us with a pristine state. */
10674 }
10675 }
10677 /* The first insn of a function that accepts its static chain on the
10678 stack is to push the register that would be filled in by a direct
10679 call. This insn will be skipped by the trampoline. */
10681 {
10685 /* We don't want to interpret this push insn as a register save,
10686 only as a stack adjustment. The real copy of the register as
10687 a save will be done later, if needed. */
10692 }
10694 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
10695 of DRAP is needed and stack realignment is really needed after reload */
10697 {
10700 /* Only need to push parameter pointer reg if it is caller saved. */
10702 {
10703 /* Push arg pointer reg */
10706 }
10708 /* Grab the argument pointer. */
10715 /* Align the stack. */
10717 stack_pointer_rtx,
10721 /* Replicate the return address on the stack so that return
10722 address can be reached via (argp - 1) slot. This is needed
10723 to implement macro RETURN_ADDR_RTX and intrinsic function
10724 expand_builtin_return_addr etc. */
10730 /* For the purposes of frame and register save area addressing,
10731 we've started over with a new frame. */
10734 }
10740 {
10741 /* Note: AT&T enter does NOT have reversed args. Enter is probably
10742 slower on all targets. Also sdb doesn't like it. */
10746 /* Push registers now, before setting the frame pointer
10747 on SEH target. */
10749 && TARGET_SEH
10751 {
10755 }
10758 {
10766 }
10767 }
10770 {
10771 /* If saving registers via PUSH, do so now. */
10773 {
10777 }
10779 /* When using red zone we may start register saving before allocating
10780 the stack frame saving one cycle of the prologue. However, avoid
10781 doing this if we have to probe the stack; at least on x86_64 the
10782 stack probe can turn into a call that clobbers a red zone location. */
10784 && (! TARGET_STACK_PROBE
10786 {
10789 }
10790 }
10793 {
10797 /* The computation of the size of the re-aligned stack frame means
10798 that we must allocate the size of the register save area before
10799 performing the actual alignment. Otherwise we cannot guarantee
10800 that there's enough storage above the realignment point. */
10807 /* Align the stack. */
10809 stack_pointer_rtx,
10812 /* For the purposes of register save area addressing, the stack
10813 pointer is no longer valid. As for the value of sp_offset,
10814 see ix86_compute_frame_layout, which we need to match in order
10815 to pass verification of stack_pointer_offset at the end. */
10818 }
10823 {
10824 /* We start to count from ARG_POINTER. */
10827 /* If it was realigned, take into account the fake frame. */
10829 {
10836 /* This over-estimates by 1 minimal-stack-alignment-unit but
10837 mitigates that by counting in the new return address slot. */
10838 current_function_dynamic_stack_size
10840 }
10843 }
10845 /* On SEH target with very large frame size, allocate an area to save
10846 SSE registers (as the very large allocation won't be described). */
10850 {
10851 HOST_WIDE_INT sse_size =
10856 /* No need to do stack checking as the area will be immediately
10857 written. */
10864 }
10866 /* The stack has already been decremented by the instruction calling us
10867 so probe if the size is non-negative to preserve the protection area. */
10869 {
10870 /* We expect the registers to be saved when probes are used. */
10874 {
10877 }
10878 else
10879 {
10887 else
10889 }
10890 }
10893 ;
10896 {
10900 }
10901 else
10902 {
10915 /* Note that SEH directives need to continue tracking the stack
10916 pointer even after the frame pointer has been set up. */
10918 {
10922 {
10926 }
10927 }
10930 {
10935 {
10939 }
10940 }
10945 /* Use the fact that AX still contains ALLOCATE. */
10947 ? gen_pro_epilogue_adjust_stack_di_sub
10954 {
10962 }
10966 {
10973 }
10975 {
10980 }
10981 }
10984 /* If we havn't already set up the frame pointer, do so now. */
10986 {
10998 }
11006 /* We don't use pic-register for pe-coff target. */
11008 && !TARGET_PECOFF
11011 {
11018 }
11021 {
11023 {
11025 {
11035 label));
11039 }
11040 else
11042 }
11043 else
11044 {
11048 }
11049 }
11051 /* In the pic_reg_used case, make sure that the got load isn't deleted
11052 when mcount needs it. Blockage to avoid call movement across mcount
11053 call is emitted in generic code after the NOTE_INSN_PROLOGUE_END
11054 note. */
11059 {
11060 /* vDRAP is setup but after reload it turns out stack realign
11061 isn't necessary, here we will emit prologue to setup DRAP
11062 without stack realign adjustment */
11065 }
11067 /* Prevent instructions from being scheduled into register save push
11068 sequence when access to the redzone area is done through frame pointer.
11069 The offset between the frame pointer and the stack pointer is calculated
11070 relative to the value of the stack pointer at the end of the function
11071 prologue, and moving instructions that access redzone area via frame
11072 pointer inside push sequence violates this assumption. */
11076 /* Emit cld instruction if stringops are used in the function. */
11080 /* SEH requires that the prologue end within 256 bytes of the start of
11081 the function. Prevent instruction schedules that would extend that.
11082 Further, prevent alloca modifications to the stack pointer from being
11083 combined with prologue modifications. */
11086 }
11088 /* Emit code to restore REG using a POP insn. */
11090 static void
11092 {
11101 {
11102 /* Previously we'd represented the CFA as an expression
11103 like *(%ebp - 8). We've just popped that value from
11104 the stack, which means we need to reset the CFA to
11105 the drap register. This will remain until we restore
11106 the stack pointer. */
11110 /* This means that the DRAP register is valid for addressing too. */
11113 }
11116 {
11123 }
11125 /* When the frame pointer is the CFA, and we pop it, we are
11126 swapping back to the stack pointer as the CFA. This happens
11127 for stack frames that don't allocate other data, so we assume
11128 the stack pointer is now pointing at the return address, i.e.
11129 the function entry state, which makes the offset be 1 word. */
11131 {
11134 {
11142 }
11143 }
11144 }
11146 /* Emit code to restore saved registers using POP insns. */
11148 static void
11150 {
11156 }
11158 /* Emit code and notes for the LEAVE instruction. */
11160 static void
11162 {
11174 {
11182 }
11185 }
11187 /* Emit code to restore saved registers using MOV insns.
11188 First register is restored from CFA - CFA_OFFSET. */
11189 static void
11192 {
11198 {
11207 {
11208 /* Previously we'd represented the CFA as an expression
11209 like *(%ebp - 8). We've just popped that value from
11210 the stack, which means we need to reset the CFA to
11211 the drap register. This will remain until we restore
11212 the stack pointer. */
11216 /* This means that the DRAP register is valid for addressing. */
11218 }
11219 else
11223 }
11224 }
11226 /* Emit code to restore saved registers using MOV insns.
11227 First register is restored from CFA - CFA_OFFSET. */
11228 static void
11231 {
11236 {
11238 rtx mem;
11248 }
11249 }
11251 /* Restore function stack, frame, and registers. */
11253 void
11255 {
11271 /* The FP must be valid if the frame pointer is present. */
11276 /* We must have *some* valid pointer to the stack frame. */
11279 /* The DRAP is never valid at this point. */
11282 /* See the comment about red zone and frame
11283 pointer usage in ix86_expand_prologue. */
11290 /* Determine the CFA offset of the end of the red-zone. */
11293 {
11294 /* The red-zone begins below the return address. */
11297 /* When the register save area is in the aligned portion of
11298 the stack, determine the maximum runtime displacement that
11299 matches up with the aligned frame. */
11303 }
11305 /* Special care must be taken for the normal return case of a function
11306 using eh_return: the eax and edx registers are marked as saved, but
11307 not restored along this path. Adjust the save location to match. */
11311 /* EH_RETURN requires the use of moves to function properly. */
11314 /* SEH requires the use of pops to identify the epilogue. */
11317 /* If we're only restoring one register and sp is not valid then
11318 using a move instruction to restore the register since it's
11319 less work than reloading sp and popping the register. */
11332 && TARGET_USE_LEAVE
11336 else
11340 {
11341 /* Ensure that the entire register save area is addressable via
11342 the stack pointer, if we will restore via sp. */
11347 {
11351 style,
11353 }
11354 }
11356 /* If there are any SSE registers to restore, then we have to do it
11357 via moves, since there's obviously no pop for SSE regs. */
11363 {
11364 rtx t;
11369 /* eh_return epilogues need %ecx added to the stack pointer. */
11371 {
11374 /* Stack align doesn't work with eh_return. */
11376 /* Neither does regparm nested functions. */
11380 {
11388 /* Note that we use SA as a temporary CFA, as the return
11389 address is at the proper place relative to it. We
11390 pretend this happens at the FP restore insn because
11391 prior to this insn the FP would be stored at the wrong
11392 offset relative to SA, and after this insn we have no
11393 other reasonable register to use for the CFA. We don't
11394 bother resetting the CFA to the SP for the duration of
11395 the return insn. */
11408 }
11409 else
11410 {
11418 {
11422 UNITS_PER_WORD));
11424 }
11425 }
11428 }
11429 }
11430 else
11431 {
11432 /* SEH requires that the function end with (1) a stack adjustment
11433 if necessary, (2) a sequence of pops, and (3) a return or
11434 jump instruction. Prevent insns from the function body from
11435 being scheduled into this sequence. */
11437 {
11438 /* Prevent a catch region from being adjacent to the standard
11439 epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
11440 several other flags that would be interesting to test are
11441 not yet set up. */
11444 else
11446 }
11448 /* First step is to deallocate the stack frame so that we can
11449 pop the registers. Also do it on SEH target for very large
11450 frame as the emitted instructions aren't allowed by the ABI in
11451 epilogues. */
11453 || (TARGET_SEH
11456 {
11461 }
11463 {
11467 style,
11469 }
11472 }
11474 /* If we used a stack pointer and haven't already got rid of it,
11475 then do so now. */
11477 {
11478 /* If the stack pointer is valid and pointing at the frame
11479 pointer store address, then we only need a pop. */
11482 /* Leave results in shorter dependency chains on CPUs that are
11483 able to grok it fast. */
11488 else
11489 {
11491 hard_frame_pointer_rtx,
11494 }
11495 }
11498 {
11500 rtx insn;
11526 }
11528 /* At this point the stack pointer must be valid, and we must have
11529 restored all of the registers. We may not have deallocated the
11530 entire stack frame. We've delayed this until now because it may
11531 be possible to merge the local stack deallocation with the
11532 deallocation forced by ix86_static_chain_on_stack. */
11537 {
11541 }
11542 else
11545 /* Sibcall epilogues don't want a return instruction. */
11547 {
11550 }
11553 {
11556 /* i386 can only pop 64K bytes. If asked to pop more, pop return
11557 address, do explicit add, and jump indirectly to the caller. */
11560 {
11562 rtx insn;
11564 /* There is no "pascal" calling convention in any 64bit ABI. */
11580 }
11581 else
11583 }
11584 else
11587 /* Restore the state back to the state from the prologue,
11588 so that it's correct for the next epilogue. */
11590 }
11592 /* Reset from the function's potential modifications. */
11594 static void
11596 HOST_WIDE_INT size ATTRIBUTE_UNUSED)
11597 {
11600 #if TARGET_MACHO
11601 /* Mach-O doesn't support labels at the end of objects, so if
11602 it looks like we might want one, insert a NOP. */
11603 {
11609 {
11610 /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
11611 notes only, instead set their CODE_LABEL_NUMBER to -1,
11612 otherwise there would be code generation differences
11613 in between -g and -g0. */
11617 }
11627 }
11628 #endif
11630 }
11632 /* Return a scratch register to use in the split stack prologue. The
11633 split stack prologue is used for -fsplit-stack. It is the first
11634 instructions in the function, even before the regular prologue.
11635 The scratch register can be any caller-saved register which is not
11636 used for parameters or for the static chain. */
11638 static unsigned int
11640 {
11643 else
11644 {
11657 {
11659 {
11663 }
11665 }
11667 {
11671 }
11673 {
11676 else
11677 {
11679 {
11683 }
11685 }
11686 }
11687 else
11688 {
11689 /* FIXME: We could make this work by pushing a register
11690 around the addition and comparison. */
11693 }
11694 }
11695 }
11697 /* A SYMBOL_REF for the function which allocates new stackspace for
11698 -fsplit-stack. */
11702 /* A SYMBOL_REF for the more stack function when using the large
11703 model. */
11707 /* Handle -fsplit-stack. These are the first instructions in the
11708 function, even before the regular prologue. */
11710 void
11712 {
11714 HOST_WIDE_INT allocate;
11719 rtx fn;
11727 /* This is the label we will branch to if we have enough stack
11728 space. We expect the basic block reordering pass to reverse this
11729 branch if optimizing, so that we branch in the unlikely case. */
11732 /* We need to compare the stack pointer minus the frame size with
11733 the stack boundary in the TCB. The stack boundary always gives
11734 us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
11735 can compare directly. Otherwise we need to do an addition. */
11738 UNSPEC_STACK_CHECK);
11743 else
11744 {
11746 rtx offset;
11748 /* We need a scratch register to hold the stack pointer minus
11749 the required frame size. Since this is the very start of the
11750 function, the scratch register can be any caller-saved
11751 register which is not used for parameters. */
11758 {
11759 /* We don't use ix86_gen_add3 in this case because it will
11760 want to split to lea, but when not optimizing the insn
11761 will not be split after this point. */
11764 offset)));
11765 }
11766 else
11767 {
11770 stack_pointer_rtx));
11771 }
11773 }
11779 /* Mark the jump as very likely to be taken. */
11787 /* Get more stack space. We pass in the desired stack space and the
11788 size of the arguments to copy to the new stack. In 32-bit mode
11789 we push the parameters; __morestack will return on a new stack
11790 anyhow. In 64-bit mode we pass the parameters in r10 and
11791 r11. */
11796 {
11802 /* If this function uses a static chain, it will be in %r10.
11803 Preserve it across the call to __morestack. */
11805 {
11806 rtx rax;
11811 }
11815 {
11816 HOST_WIDE_INT argval;
11819 /* When using the large model we need to load the address
11820 into a register, and we've run out of registers. So we
11821 switch to a different calling convention, and we call a
11822 different function: __morestack_large. We pass the
11823 argument size in the upper 32 bits of r10 and pass the
11824 frame size in the lower 32 bits. */
11829 split_stack_fn_large =
11833 {
11843 UNSPEC_GOT);
11849 }
11850 else
11857 }
11858 else
11859 {
11863 }
11866 }
11867 else
11868 {
11871 }
11877 /* In order to make call/return prediction work right, we now need
11878 to execute a return instruction. See
11879 libgcc/config/i386/morestack.S for the details on how this works.
11881 For flow purposes gcc must not see this as a return
11882 instruction--we need control flow to continue at the subsequent
11883 label. Therefore, we use an unspec. */
11887 /* If we are in 64-bit mode and this function uses a static chain,
11888 we saved %r10 in %rax before calling _morestack. */
11893 /* If this function calls va_start, we need to store a pointer to
11894 the arguments on the old stack, because they may not have been
11895 all copied to the new stack. At this point the old stack can be
11896 found at the frame pointer value used by __morestack, because
11897 __morestack has set that up before calling back to us. Here we
11898 store that pointer in a scratch register, and in
11899 ix86_expand_prologue we store the scratch register in a stack
11900 slot. */
11902 {
11904 rtx frame_reg;
11911 /* 64-bit:
11912 fp -> old fp value
11913 return address within this function
11914 return address of caller of this function
11915 stack arguments
11916 So we add three words to get to the stack arguments.
11918 32-bit:
11919 fp -> old fp value
11920 return address within this function
11921 first argument to __morestack
11922 second argument to __morestack
11923 return address of caller of this function
11924 stack arguments
11925 So we add five words to get to the stack arguments.
11926 */
11937 }
11942 /* If this function calls va_start, we now have to set the scratch
11943 register for the case where we do not call __morestack. In this
11944 case we need to set it based on the stack pointer. */
11946 {
11953 }
11954 }
11956 /* We may have to tell the dataflow pass that the split stack prologue
11957 is initializing a scratch register. */
11959 static void
11961 {
11963 {
11966 }
11967 }
11968 \f
11969 /* Determine if op is suitable SUBREG RTX for address. */
11971 static bool
11973 {
11984 /* Don't allow SUBREGs that span more than a word. It can lead to spill
11985 failures when the register is one word out of a two word structure. */
11989 /* Allow only SUBREGs of non-eliminable hard registers. */
11991 }
11993 /* Extract the parts of an RTL expression that is a valid memory address
11994 for an instruction. Return 0 if the structure of the address is
11995 grossly off. Return -1 if the address contains ASHIFT, so it is not
11996 strictly valid, but still used for computing length of lea instruction. */
11998 int
12000 {
12005 rtx tmp;
12009 /* Allow zero-extended SImode addresses,
12010 they will be emitted with addr32 prefix. */
12012 {
12015 {
12019 }
12022 {
12029 }
12030 }
12032 /* Allow SImode subregs of DImode addresses,
12033 they will be emitted with addr32 prefix. */
12035 {
12038 {
12042 }
12043 }
12048 {
12051 else
12053 }
12055 {
12060 do
12061 {
12066 }
12073 {
12076 {
12101 /* FALLTHRU */
12105 && TARGET_TLS_DIRECT_SEG_REFS
12108 else
12115 /* FALLTHRU */
12122 else
12137 }
12138 }
12139 }
12141 {
12144 }
12146 {
12147 /* We're called for lea too, which implements ashift on occasion. */
12157 }
12159 {
12163 /* Constant addresses are sign extended to 64bit, we have to
12164 prevent addresses from 0x80000000 to 0xffffffff in x32 mode. */
12170 }
12171 else
12175 {
12177 ;
12180 ;
12181 else
12183 }
12185 /* Address override works only on the (%reg) part of %fs:(%reg). */
12191 /* Extract the integral value of scale. */
12193 {
12197 }
12202 /* Avoid useless 0 displacement. */
12206 /* Allow arg pointer and stack pointer as index if there is not scaling. */
12211 {
12212 rtx tmp;
12215 }
12217 /* Special case: %ebp cannot be encoded as a base without a displacement.
12218 Similarly %r13. */
12220 && base_reg
12229 /* Special case: on K6, [%esi] makes the instruction vector decoded.
12230 Avoid this by transforming to [%esi+0].
12231 Reload calls address legitimization without cfun defined, so we need
12232 to test cfun for being non-NULL. */
12238 /* Special case: encode reg+reg instead of reg*2. */
12242 /* Special case: scaling cannot be encoded without base or displacement. */
12253 }
12254 \f
12255 /* Return cost of the memory address x.
12256 For i386, it is better to use a complex address than let gcc copy
12257 the address into a reg and make a new pseudo. But not if the address
12258 requires to two regs - that would mean more pseudos with longer
12259 lifetimes. */
12260 static int
12262 addr_space_t as ATTRIBUTE_UNUSED,
12264 {
12276 /* Attempt to minimize number of registers in the address. */
12282 cost++;
12289 cost++;
12291 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
12292 since it's predecode logic can't detect the length of instructions
12293 and it degenerates to vector decoded. Increase cost of such
12294 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
12295 to split such addresses or even refuse such addresses at all.
12297 Following addressing modes are affected:
12298 [base+scale*index]
12299 [scale*index+disp]
12300 [base+index]
12302 The first and last case may be avoidable by explicitly coding the zero in
12303 memory address, but I don't have AMD-K6 machine handy to check this
12304 theory. */
12313 }
12314 \f
12315 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
12316 this is used for to form addresses to local data when -fPIC is in
12317 use. */
12319 static bool
12321 {
12324 }
12326 /* Determine if a given RTX is a valid constant. We already know this
12327 satisfies CONSTANT_P. */
12329 static bool
12331 {
12333 {
12338 {
12342 }
12347 /* Only some unspecs are valid as "constants". */
12350 {
12366 }
12368 /* We must have drilled down to a symbol. */
12373 /* FALLTHRU */
12376 /* TLS symbols are never valid. */
12380 /* DLLIMPORT symbols are never valid. */
12385 #if TARGET_MACHO
12386 /* mdynamic-no-pic */
12389 #endif
12405 }
12407 /* Otherwise we handle everything else in the move patterns. */
12409 }
12411 /* Determine if it's legal to put X into the constant pool. This
12412 is not possible for the address of thread-local symbols, which
12413 is checked above. */
12415 static bool
12417 {
12418 /* We can always put integral constants and vectors in memory. */
12420 {
12428 }
12430 }
12432 /* Nonzero if the symbol is marked as dllimport, or as stub-variable,
12433 otherwise zero. */
12435 static bool
12437 {
12443 }
12446 /* Nonzero if the constant value X is a legitimate general operand
12447 when generating PIC code. It is given that flag_pic is on and
12448 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
12450 bool
12452 {
12453 rtx inner;
12456 {
12463 /* Only some unspecs are valid as "constants". */
12466 {
12479 }
12480 /* FALLTHRU */
12488 }
12489 }
12491 /* Determine if a given CONST RTX is a valid memory displacement
12492 in PIC mode. */
12494 bool
12496 {
12499 /* In 64bit mode we can allow direct addresses of symbols and labels
12500 when they are not dynamic symbols. */
12502 {
12506 {
12530 /* FALLTHRU */
12533 /* TLS references should always be enclosed in UNSPEC.
12534 The dllimported symbol needs always to be resolved. */
12540 {
12548 /* Function-symbols need to be resolved only for
12549 large-model.
12550 For the small-model we don't need to resolve anything
12551 here. */
12556 /* Non-external symbols don't need to be resolved for
12557 large, and medium-model. */
12562 }
12571 }
12572 }
12578 {
12579 /* We are unsafe to allow PLUS expressions. This limit allowed distance
12580 of GOT tables. We should not need these anyway. */
12592 }
12596 {
12601 }
12610 {
12614 /* We need to check for both symbols and labels because VxWorks loads
12615 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
12616 details. */
12620 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
12621 While ABI specify also 32bit relocation but we don't produce it in
12622 small PIC model at all. */
12644 }
12647 }
12649 /* Our implementation of LEGITIMIZE_RELOAD_ADDRESS. Returns a value to
12650 replace the input X, or the original X if no replacement is called for.
12651 The output parameter *WIN is 1 if the calling macro should goto WIN,
12652 0 if it should not. */
12654 bool
12659 {
12660 /* Reload can generate:
12662 (plus:DI (plus:DI (unspec:DI [(const_int 0 [0])] UNSPEC_TP)
12663 (reg:DI 97))
12664 (reg:DI 2 cx))
12666 This RTX is rejected from ix86_legitimate_address_p due to
12667 non-strictness of base register 97. Following this rejection,
12668 reload pushes all three components into separate registers,
12669 creating invalid memory address RTX.
12671 Following code reloads only the invalid part of the
12672 memory address RTX. */
12678 {
12684 {
12689 }
12693 {
12698 }
12702 }
12705 }
12707 /* Recognizes RTL expressions that are valid memory addresses for an
12708 instruction. The MODE argument is the machine mode for the MEM
12709 expression that wants to use this address.
12711 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
12712 convert common non-canonical forms to canonical form so that they will
12713 be recognized. */
12715 static bool
12718 {
12721 HOST_WIDE_INT scale;
12724 /* Decomposition failed. */
12732 /* Validate base register. */
12734 {
12735 rtx reg;
12741 else
12742 /* Base is not a register. */
12750 /* Base is not valid. */
12752 }
12754 /* Validate index register. */
12756 {
12757 rtx reg;
12763 else
12764 /* Index is not a register. */
12772 /* Index is not valid. */
12774 }
12776 /* Index and base should have the same mode. */
12781 /* Validate scale factor. */
12783 {
12785 /* Scale without index. */
12789 /* Scale is not a valid multiplier. */
12791 }
12793 /* Validate displacement. */
12795 {
12800 {
12801 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
12802 used. While ABI specify also 32bit relocations, we don't produce
12803 them at all and use IP relative instead. */
12810 /* 64bit address unspec. */
12830 /* Invalid address unspec. */
12832 }
12835 && (flag_pic
12836 || (TARGET_MACHO
12837 #if TARGET_MACHO
12838 && MACHOPIC_INDIRECT
12840 #endif
12841 )))
12842 {
12844 is_legitimate_pic:
12846 {
12847 /* foo@dtpoff(%rX) is ok. */
12854 /* Non-constant pic memory reference. */
12856 }
12859 /* Displacement is an invalid pic construct. */
12861 #if TARGET_MACHO
12864 /* displacment must be referenced via non_lazy_pointer */
12866 #endif
12868 /* This code used to verify that a symbolic pic displacement
12869 includes the pic_offset_table_rtx register.
12871 While this is good idea, unfortunately these constructs may
12872 be created by "adds using lea" optimization for incorrect
12873 code like:
12875 int a;
12876 int foo(int i)
12877 {
12878 return *(&a+i);
12879 }
12881 This code is nonsensical, but results in addressing
12882 GOT table with pic_offset_table_rtx base. We can't
12883 just refuse it easily, since it gets matched by
12884 "addsi3" pattern, that later gets split to lea in the
12885 case output register differs from input. While this
12886 can be handled by separate addsi pattern for this case
12887 that never results in lea, this seems to be easier and
12888 correct fix for crash to disable this test. */
12889 }
12896 /* Displacement is not constant. */
12900 /* Displacement is out of range. */
12902 }
12904 /* Everything looks valid. */
12906 }
12908 /* Determine if a given RTX is a valid constant address. */
12910 bool
12912 {
12914 }
12915 \f
12916 /* Return a unique alias set for the GOT. */
12918 static alias_set_type
12920 {
12925 }
12927 /* Return a legitimate reference for ORIG (an address) using the
12928 register REG. If REG is 0, a new pseudo is generated.
12930 There are two types of references that must be handled:
12932 1. Global data references must load the address from the GOT, via
12933 the PIC reg. An insn is emitted to do this load, and the reg is
12934 returned.
12936 2. Static data references, constant pool addresses, and code labels
12937 compute the address as an offset from the GOT, whose base is in
12938 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
12939 differentiate them from global data objects. The returned
12940 address is the PIC reg + an unspec constant.
12942 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
12943 reg also appears in the address. */
12945 static rtx
12947 {
12951 #if TARGET_MACHO
12953 {
12956 /* Use the generic Mach-O PIC machinery. */
12958 }
12959 #endif
12962 {
12966 }
12972 {
12973 rtx tmpreg;
12974 /* This symbol may be referenced via a displacement from the PIC
12975 base address (@GOTOFF). */
12982 {
12984 UNSPEC_GOTOFF);
12986 }
12987 else
12992 else
12997 {
13001 }
13002 else
13004 }
13006 {
13007 /* This symbol may be referenced via a displacement from the PIC
13008 base address (@GOTOFF). */
13015 {
13017 UNSPEC_GOTOFF);
13019 }
13020 else
13026 {
13029 }
13030 }
13032 /* We can't use @GOTOFF for text labels on VxWorks;
13033 see gotoff_operand. */
13035 {
13040 /* For x64 PE-COFF there is no GOT table. So we use address
13041 directly. */
13043 {
13051 }
13053 {
13061 /* Use directly gen_movsi, otherwise the address is loaded
13062 into register for CSE. We don't want to CSE this addresses,
13063 instead we CSE addresses from the GOT table, so skip this. */
13066 }
13067 else
13068 {
13069 /* This symbol must be referenced via a load from the
13070 Global Offset Table (@GOT). */
13086 }
13087 }
13088 else
13089 {
13092 {
13094 {
13097 }
13098 else
13100 }
13102 {
13105 /* We must match stuff we generate before. Assume the only
13106 unspecs that can get here are ours. Not that we could do
13107 anything with them anyway.... */
13113 }
13115 {
13118 /* Check first to see if this is a constant offset from a @GOTOFF
13119 symbol reference. */
13122 {
13124 {
13128 UNSPEC_GOTOFF);
13134 {
13137 }
13138 }
13139 else
13140 {
13143 {
13147 }
13148 }
13149 }
13150 else
13151 {
13154 new_rtx
13158 {
13161 {
13164 new_rtx
13166 }
13167 else
13169 }
13170 else
13171 {
13174 {
13177 }
13179 }
13180 }
13181 }
13182 }
13184 }
13185 \f
13186 /* Load the thread pointer. If TO_REG is true, force it into a register. */
13188 static rtx
13190 {
13194 {
13199 }
13205 }
13207 /* Construct the SYMBOL_REF for the tls_get_addr function. */
13211 static rtx
13213 {
13215 {
13221 }
13224 }
13226 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
13230 rtx
13232 {
13234 {
13235 ix86_tls_module_base_symbol
13240 }
13243 }
13245 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
13246 false if we expect this to be used for a memory address and true if
13247 we expect to load the address into a register. */
13249 static rtx
13251 {
13258 {
13263 {
13266 else
13267 {
13270 }
13271 }
13274 {
13277 else
13287 }
13288 else
13289 {
13293 {
13295 rtx insns;
13298 emit_call_insn
13308 }
13309 else
13311 }
13318 {
13321 else
13322 {
13325 }
13326 }
13329 {
13334 else
13340 }
13341 else
13342 {
13346 {
13351 emit_call_insn
13356 /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
13357 share the LD_BASE result with other LD model accesses. */
13359 UNSPEC_TLS_LD_BASE);
13363 }
13364 else
13366 }
13374 {
13381 }
13386 {
13388 {
13389 /* The Sun linker took the AMD64 TLS spec literally
13390 and can only handle %rax as destination of the
13391 initial executable code sequence. */
13396 }
13398 /* Generate DImode references to avoid %fs:(%reg32)
13399 problems and linker IE->LE relaxation bug. */
13403 }
13405 {
13410 }
13412 {
13416 }
13417 else
13418 {
13421 }
13431 {
13436 }
13437 else
13438 {
13442 }
13452 {
13456 }
13457 else
13458 {
13462 }
13467 }
13470 }
13472 /* Create or return the unique __imp_DECL dllimport symbol corresponding
13473 to symbol DECL if BEIMPORT is true. Otherwise create or return the
13474 unique refptr-DECL symbol corresponding to symbol DECL. */
13477 htab_t dllimport_map;
13479 static tree
13481 {
13488 tree to;
13489 rtx rtl;
13516 else
13529 {
13531 #ifdef SUB_TARGET_RECORD_STUB
13533 #endif
13534 }
13543 }
13545 /* Expand SYMBOL into its corresponding far-addresse symbol.
13546 WANT_REG is true if we require the result be a register. */
13548 static rtx
13550 {
13551 tree imp_decl;
13552 rtx x;
13561 }
13563 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
13564 true if we require the result be a register. */
13566 static rtx
13568 {
13569 tree imp_decl;
13570 rtx x;
13579 }
13581 /* Expand SYMBOL into its corresponding dllimport or refptr symbol. WANT_REG
13582 is true if we require the result be a register. */
13584 static rtx
13586 {
13591 {
13598 {
13601 }
13602 }
13618 {
13621 }
13623 }
13625 /* Try machine-dependent ways of modifying an illegitimate address
13626 to be legitimate. If we find one, return the new, valid address.
13627 This macro is used in only one place: `memory_address' in explow.c.
13629 OLDX is the address as it was before break_out_memory_refs was called.
13630 In some cases it is useful to look at this to decide what needs to be done.
13632 It is always safe for this macro to do nothing. It exists to recognize
13633 opportunities to optimize the output.
13635 For the 80386, we handle X+REG by loading X into a register R and
13636 using R+REG. R will go in a general reg and indexing will be used.
13637 However, if REG is a broken-out memory address or multiplication,
13638 nothing needs to be done because REG can certainly go in a general reg.
13640 When -fpic is used, special handling is needed for symbolic references.
13641 See comments by legitimize_pic_address in i386.c for details. */
13643 static rtx
13646 {
13657 {
13661 }
13664 {
13668 }
13673 #if TARGET_MACHO
13676 #endif
13678 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
13682 {
13687 }
13690 {
13691 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
13696 {
13702 }
13707 {
13713 }
13715 /* Put multiply first if it isn't already. */
13717 {
13722 }
13724 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
13725 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
13726 created by virtual register instantiation, register elimination, and
13727 similar optimizations. */
13729 {
13735 }
13737 /* Canonicalize
13738 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
13739 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
13744 {
13745 rtx constant;
13749 {
13752 }
13754 {
13757 }
13758 else
13762 {
13769 }
13770 }
13776 {
13779 }
13782 {
13785 }
13793 {
13796 }
13802 {
13806 {
13809 }
13813 }
13816 {
13820 {
13823 }
13827 }
13828 }
13831 }
13832 \f
13833 /* Print an integer constant expression in assembler syntax. Addition
13834 and subtraction are the only arithmetic that may appear in these
13835 expressions. FILE is the stdio stream to write to, X is the rtx, and
13836 CODE is the operand print code from the output string. */
13838 static void
13840 {
13844 {
13853 else
13854 {
13857 /* Mark the decl as referenced so that cgraph will
13858 output the function. */
13862 #if TARGET_MACHO
13866 #endif
13868 }
13876 /* FALLTHRU */
13887 /* This used to output parentheses around the expression,
13888 but that does not work on the 386 (either ATT or BSD assembler). */
13894 {
13895 /* We can use %d if the number is <32 bits and positive. */
13900 else
13902 }
13903 else
13904 /* We can't handle floating point constants;
13905 TARGET_PRINT_OPERAND must handle them. */
13910 /* Some assemblers need integer constants to appear first. */
13912 {
13916 }
13917 else
13918 {
13923 }
13938 {
13942 }
13947 {
13966 /* FIXME: This might be @TPOFF in Sun ld too. */
13975 else
13985 else
13991 #if TARGET_MACHO
13996 #endif
14000 }
14005 }
14006 }
14008 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
14009 We need to emit DTP-relative relocations. */
14011 static void ATTRIBUTE_UNUSED
14013 {
14018 {
14026 }
14027 }
14029 /* Return true if X is a representation of the PIC register. This copes
14030 with calls from ix86_find_base_term, where the register might have
14031 been replaced by a cselib value. */
14033 static bool
14035 {
14039 else
14041 }
14043 /* Helper function for ix86_delegitimize_address.
14044 Attempt to delegitimize TLS local-exec accesses. */
14046 static rtx
14048 {
14073 {
14078 }
14084 }
14086 /* In the name of slightly smaller debug output, and to cater to
14087 general assembler lossage, recognize PIC+GOTOFF and turn it back
14088 into a direct symbol reference.
14090 On Darwin, this is necessary to avoid a crash, because Darwin
14091 has a different PIC label for each routine but the DWARF debugging
14092 information is not associated with any particular routine, so it's
14093 necessary to remove references to the PIC label from RTL stored by
14094 the DWARF output code. */
14096 static rtx
14098 {
14100 /* addend is NULL or some rtx if x is something+GOTOFF where
14101 something doesn't include the PIC register. */
14103 /* reg_addend is NULL or a multiple of some register. */
14105 /* const_addend is NULL or a const_int. */
14107 /* This is the result, or NULL. */
14116 {
14123 {
14129 }
14138 {
14143 }
14145 }
14152 /* %ebx + GOT/GOTOFF */
14153 ;
14155 {
14156 /* %ebx + %reg * scale + GOT/GOTOFF */
14162 else
14163 {
14166 }
14167 }
14168 else
14174 {
14177 }
14196 {
14197 /* If the rest of original X doesn't involve the PIC register, add
14198 addend and subtract pic_offset_table_rtx. This can happen e.g.
14199 for code like:
14200 leal (%ebx, %ecx, 4), %ecx
14201 ...
14202 movl foo@GOTOFF(%ecx), %edx
14203 in which case we return (%ecx - %ebx) + foo. */
14206 pic_offset_table_rtx),
14207 result);
14208 else
14210 }
14212 {
14216 }
14218 }
14220 /* If X is a machine specific address (i.e. a symbol or label being
14221 referenced as a displacement from the GOT implemented using an
14222 UNSPEC), then return the base term. Otherwise return X. */
14224 rtx
14226 {
14227 rtx term;
14230 {
14244 }
14247 }
14248 \f
14249 static void
14252 {
14256 {
14259 }
14264 {
14267 {
14286 }
14290 {
14309 }
14316 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
14317 Those same assemblers have the same but opposite lossage on cmov. */
14322 else
14327 {
14340 }
14348 {
14361 }
14364 /* ??? As above. */
14373 /* ??? As above. */
14378 else
14389 }
14391 }
14393 /* Print the name of register X to FILE based on its machine mode and number.
14394 If CODE is 'w', pretend the mode is HImode.
14395 If CODE is 'b', pretend the mode is QImode.
14396 If CODE is 'k', pretend the mode is SImode.
14397 If CODE is 'q', pretend the mode is DImode.
14398 If CODE is 'x', pretend the mode is V4SFmode.
14399 If CODE is 't', pretend the mode is V8SFmode.
14400 If CODE is 'h', pretend the reg is the 'high' byte register.
14401 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
14402 If CODE is 'd', duplicate the operand for AVX instruction.
14403 */
14405 void
14407 {
14416 {
14420 }
14445 else
14448 /* Irritatingly, AMD extended registers use different naming convention
14449 from the normal registers: "r%d[bwd]" */
14451 {
14456 {
14470 /* no suffix */
14475 }
14477 }
14481 {
14484 {
14487 }
14488 /* FALLTHRU */
14494 /* FALLTHRU */
14497 normal:
14512 {
14517 }
14521 }
14525 {
14528 else
14530 }
14531 }
14533 /* Locate some local-dynamic symbol still in use by this function
14534 so that we can print its name in some tls_local_dynamic_base
14535 pattern. */
14537 static int
14539 {
14544 {
14547 }
14550 }
14554 {
14555 rtx insn;
14566 }
14568 /* Meaning of CODE:
14569 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
14570 C -- print opcode suffix for set/cmov insn.
14571 c -- like C, but print reversed condition
14572 F,f -- likewise, but for floating-point.
14573 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
14574 otherwise nothing
14575 R -- print the prefix for register names.
14576 z -- print the opcode suffix for the size of the current operand.
14577 Z -- likewise, with special suffixes for x87 instructions.
14578 * -- print a star (in certain assembler syntax)
14579 A -- print an absolute memory reference.
14580 E -- print address with DImode register names if TARGET_64BIT.
14581 w -- print the operand as if it's a "word" (HImode) even if it isn't.
14582 s -- print a shift double count, followed by the assemblers argument
14583 delimiter.
14584 b -- print the QImode name of the register for the indicated operand.
14585 %b0 would print %al if operands[0] is reg 0.
14586 w -- likewise, print the HImode name of the register.
14587 k -- likewise, print the SImode name of the register.
14588 q -- likewise, print the DImode name of the register.
14589 x -- likewise, print the V4SFmode name of the register.
14590 t -- likewise, print the V8SFmode name of the register.
14591 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
14592 y -- print "st(0)" instead of "st" as a register.
14593 d -- print duplicated register operand for AVX instruction.
14594 D -- print condition for SSE cmp instruction.
14595 P -- if PIC, print an @PLT suffix.
14596 p -- print raw symbol name.
14597 X -- don't print any sort of PIC '@' suffix for a symbol.
14598 & -- print some in-use local-dynamic symbol name.
14599 H -- print a memory address offset by 8; used for sse high-parts
14600 Y -- print condition for XOP pcom* instruction.
14601 + -- print a branch hint as 'cs' or 'ds' prefix
14602 ; -- print a semicolon (after prefixes due to bug in older gas).
14603 ~ -- print "i" if TARGET_AVX2, "f" otherwise.
14604 @ -- print a segment register of thread base pointer load
14605 ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
14606 */
14608 void
14610 {
14612 {
14614 {
14617 {
14623 /* Intel syntax. For absolute addresses, registers should not
14624 be surrounded by braces. */
14626 {
14631 }
14636 }
14642 /* Wrap address in an UNSPEC to declare special handling. */
14680 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14685 {
14699 output_operand_lossage
14702 }
14705 #endif
14710 {
14711 /* Opcodes don't get size suffixes if using Intel opcodes. */
14716 {
14734 output_operand_lossage
14737 }
14738 }
14741 warning
14743 /* FALLTHRU */
14746 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
14751 {
14753 {
14755 #ifdef HAVE_AS_IX86_FILDS
14757 #endif
14765 #ifdef HAVE_AS_IX86_FILDQ
14767 #else
14769 #endif
14774 }
14775 }
14777 {
14778 /* 387 opcodes don't get size suffixes
14779 if the operands are registers. */
14784 {
14800 }
14801 }
14802 else
14803 {
14804 output_operand_lossage
14807 }
14809 output_operand_lossage
14829 {
14832 }
14837 {
14888 }
14892 /* Little bit of braindamage here. The SSE compare instructions
14893 does use completely different names for the comparisons that the
14894 fp conditional moves. */
14896 {
14899 {
14902 }
14908 {
14911 }
14917 {
14920 }
14929 {
14932 }
14938 {
14941 }
14947 {
14950 }
14961 }
14966 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14969 #endif
14974 {
14978 }
14982 file);
14987 {
14991 }
14992 /* It doesn't actually matter what mode we use here, as we're
14993 only going to use this for printing. */
14995 /* Output 'qword ptr' for intel assembler dialect. */
15004 #ifdef HAVE_AS_IX86_HLE
15006 #else
15008 #endif
15010 #ifdef HAVE_AS_IX86_HLE
15012 #else
15014 #endif
15015 /* We do not want to print value of the operand. */
15024 {
15029 else
15032 }
15035 {
15036 rtx x;
15045 {
15050 {
15052 bool cputaken
15055 /* Emit hints only in the case default branch prediction
15056 heuristics would fail. */
15058 {
15059 /* We use 3e (DS) prefix for taken branches and
15060 2e (CS) prefix for not taken branches. */
15063 else
15065 }
15066 }
15067 }
15069 }
15072 #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
15074 #endif
15081 /* The kernel uses a different segment register for performance
15082 reasons; a system call would not have to trash the userspace
15083 segment register, which would be expensive. */
15086 else
15101 }
15102 }
15108 {
15109 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
15112 {
15115 {
15124 else
15130 }
15132 /* Check for explicit size override (codes 'b', 'w', 'k',
15133 'q' and 'x') */
15147 }
15150 /* Avoid (%rip) for call operands. */
15156 else
15158 }
15161 {
15162 REAL_VALUE_TYPE r;
15170 /* Sign extend 32bit SFmode immediate to 8 bytes. */
15174 else
15176 }
15179 {
15180 REAL_VALUE_TYPE r;
15189 }
15191 /* These float cases don't actually occur as immediate operands. */
15193 {
15198 }
15200 else
15201 {
15202 /* We have patterns that allow zero sets of memory, for instance.
15203 In 64-bit mode, we should probably support all 8-byte vectors,
15204 since we can in fact encode that into an immediate. */
15206 {
15209 }
15212 {
15214 {
15217 }
15220 {
15223 else
15225 }
15226 }
15231 else
15233 }
15234 }
15236 static bool
15238 {
15241 }
15242 \f
15243 /* Print a memory operand whose address is ADDR. */
15245 static void
15247 {
15256 {
15263 }
15265 {
15269 }
15270 else
15281 {
15292 }
15294 /* Use one byte shorter RIP relative addressing for 64bit mode. */
15296 {
15308 }
15310 {
15311 /* Displacement only requires special attention. */
15314 {
15318 }
15321 else
15323 }
15324 else
15325 {
15326 /* Print SImode register names to force addr32 prefix. */
15328 {
15329 #ifdef ENABLE_CHECKING
15332 {
15343 }
15344 #endif
15347 }
15349 && TARGET_X32
15350 && disp
15353 {
15354 /* X32 runs in 64-bit mode, where displacement, DISP, in
15355 address DISP(%r64), is encoded as 32-bit immediate sign-
15356 extended from 32-bit to 64-bit. For -0x40000300(%r64),
15357 address is %r64 + 0xffffffffbffffd00. When %r64 <
15358 0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
15359 which is invalid for x32. The correct address is %r64
15360 - 0x40000300 == 0xf7ffdd64. To properly encode
15361 -0x40000300(%r64) for x32, we zero-extend negative
15362 displacement by forcing addr32 prefix which truncates
15363 0xfffffffff7ffdd64 to 0xf7ffdd64. In theory, we should
15364 zero-extend all negative displacements, including -1(%rsp).
15365 However, for small negative displacements, sign-extension
15366 won't cause overflow. We only zero-extend negative
15367 displacements if they < -16*1024*1024, which is also used
15368 to check legitimate address displacements for PIC. */
15370 }
15373 {
15375 {
15380 else
15382 }
15388 {
15393 }
15395 }
15396 else
15397 {
15401 {
15402 /* Pull out the offset of a symbol; print any symbol itself. */
15406 {
15410 }
15418 else
15420 }
15424 {
15427 {
15431 }
15432 }
15435 else
15439 {
15444 }
15446 }
15447 }
15448 }
15450 /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
15452 static bool
15454 {
15455 rtx op;
15462 {
15465 /* FIXME: This might be @TPOFF in Sun ld. */
15476 else
15488 else
15495 #if TARGET_MACHO
15501 #endif
15504 {
15509 #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
15511 #else
15513 #endif
15516 }
15521 }
15524 }
15525 \f
15526 /* Split one or more double-mode RTL references into pairs of half-mode
15527 references. The RTL can be REG, offsettable MEM, integer constant, or
15528 CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
15529 split and "num" is its length. lo_half and hi_half are output arrays
15530 that parallel "operands". */
15532 void
15535 {
15540 {
15549 }
15554 {
15557 /* simplify_subreg refuse to split volatile memory addresses,
15558 but we still have to handle it. */
15560 {
15563 }
15564 else
15565 {
15572 }
15573 }
15574 }
15575 \f
15576 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
15577 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
15578 is the expression of the binary operation. The output may either be
15579 emitted here, or returned to the caller, like all output_* functions.
15581 There is no guarantee that the operands are the same mode, as they
15582 might be within FLOAT or FLOAT_EXTEND expressions. */
15584 #ifndef SYSV386_COMPAT
15585 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
15586 wants to fix the assemblers because that causes incompatibility
15587 with gcc. No-one wants to fix gcc because that causes
15588 incompatibility with assemblers... You can use the option of
15589 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
15590 #define SYSV386_COMPAT 1
15591 #endif
15595 {
15601 #ifdef ENABLE_CHECKING
15602 /* Even if we do not want to check the inputs, this documents input
15603 constraints. Which helps in understanding the following code. */
15613 else
15615 #endif
15618 {
15623 else
15632 else
15641 else
15650 else
15657 }
15660 {
15662 {
15666 else
15668 }
15669 else
15670 {
15674 else
15676 }
15678 }
15682 {
15686 {
15690 }
15692 /* know operands[0] == operands[1]. */
15695 {
15698 }
15701 {
15703 /* How is it that we are storing to a dead operand[2]?
15704 Well, presumably operands[1] is dead too. We can't
15705 store the result to st(0) as st(0) gets popped on this
15706 instruction. Instead store to operands[2] (which I
15707 think has to be st(1)). st(1) will be popped later.
15708 gcc <= 2.8.1 didn't have this check and generated
15709 assembly code that the Unixware assembler rejected. */
15711 else
15714 }
15718 else
15725 {
15728 }
15731 {
15734 }
15737 {
15738 #if SYSV386_COMPAT
15739 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
15740 derived assemblers, confusingly reverse the direction of
15741 the operation for fsub{r} and fdiv{r} when the
15742 destination register is not st(0). The Intel assembler
15743 doesn't have this brain damage. Read !SYSV386_COMPAT to
15744 figure out what the hardware really does. */
15747 else
15749 #else
15751 /* As above for fmul/fadd, we can't store to st(0). */
15753 else
15755 #endif
15757 }
15760 {
15761 #if SYSV386_COMPAT
15764 else
15766 #else
15769 else
15771 #endif
15773 }
15776 {
15779 else
15782 }
15784 {
15785 #if SYSV386_COMPAT
15787 #else
15789 #endif
15790 }
15791 else
15792 {
15793 #if SYSV386_COMPAT
15795 #else
15797 #endif
15798 }
15803 }
15807 }
15809 /* Check if a 256bit AVX register is referenced inside of EXP. */
15811 static int
15813 {
15824 }
15826 /* Return needed mode for entity in optimize_mode_switching pass. */
15828 static int
15830 {
15832 {
15833 rtx link;
15835 /* Needed mode is set to AVX_U128_CLEAN if there are
15836 no 256bit modes used in function arguments. */
15838 link;
15840 {
15842 {
15847 }
15848 }
15851 }
15853 /* Require DIRTY mode if a 256bit AVX register is referenced. Hardware
15854 changes state only when a 256bit register is written to, but we need
15855 to prevent the compiler from moving optimal insertion point above
15856 eventual read from 256bit register. */
15861 }
15863 /* Return mode that i387 must be switched into
15864 prior to the execution of insn. */
15866 static int
15868 {
15871 /* The mode UNINITIALIZED is used to store control word after a
15872 function call or ASM pattern. The mode ANY specify that function
15873 has no requirements on the control word and make no changes in the
15874 bits we are interested in. */
15888 {
15911 }
15914 }
15916 /* Return mode that entity must be switched into
15917 prior to the execution of insn. */
15919 int
15921 {
15923 {
15933 }
15935 }
15937 /* Check if a 256bit AVX register is referenced in stores. */
15939 static void
15941 {
15943 {
15946 }
15947 }
15949 /* Calculate mode of upper 128bit AVX registers after the insn. */
15951 static int
15953 {
15960 /* We know that state is clean after CALL insn if there are no
15961 256bit registers used in the function return register. */
15963 {
15968 }
15970 /* Otherwise, return current mode. Remember that if insn
15971 references AVX 256bit registers, the mode was already changed
15972 to DIRTY from MODE_NEEDED. */
15974 }
15976 /* Return the mode that an insn results in. */
15978 int
15980 {
15982 {
15992 }
15993 }
15995 static int
15997 {
15998 tree arg;
16000 /* Entry mode is set to AVX_U128_DIRTY if there are
16001 256bit modes used in function arguments. */
16004 {
16009 }
16012 }
16014 /* Return a mode that ENTITY is assumed to be
16015 switched to at function entry. */
16017 int
16019 {
16021 {
16031 }
16032 }
16034 static int
16036 {
16039 /* Exit mode is set to AVX_U128_DIRTY if there are
16040 256bit modes used in the function return register. */
16045 }
16047 /* Return a mode that ENTITY is assumed to be
16048 switched to at function exit. */
16050 int
16052 {
16054 {
16064 }
16065 }
16067 /* Output code to initialize control word copies used by trunc?f?i and
16068 rounding patterns. CURRENT_MODE is set to current control word,
16069 while NEW_MODE is set to new control word. */
16071 static void
16073 {
16075 rtx new_mode;
16086 {
16088 {
16090 /* round toward zero (truncate) */
16096 /* round down toward -oo */
16103 /* round up toward +oo */
16110 /* mask precision exception for nearbyint() */
16117 }
16118 }
16119 else
16120 {
16122 {
16124 /* round toward zero (truncate) */
16130 /* round down toward -oo */
16136 /* round up toward +oo */
16142 /* mask precision exception for nearbyint() */
16149 }
16150 }
16156 }
16158 /* Emit vzeroupper. */
16160 void
16162 {
16165 /* Cancel automatic vzeroupper insertion if there are
16166 live call-saved SSE registers at the insertion point. */
16178 }
16180 /* Generate one or more insns to set ENTITY to MODE. */
16182 void
16184 {
16186 {
16201 }
16202 }
16204 /* Output code for INSN to convert a float to a signed int. OPERANDS
16205 are the insn operands. The output may be [HSD]Imode and the input
16206 operand may be [SDX]Fmode. */
16210 {
16215 /* Jump through a hoop or two for DImode, since the hardware has no
16216 non-popping instruction. We used to do this a different way, but
16217 that was somewhat fragile and broke with post-reload splitters. */
16227 else
16228 {
16233 else
16237 }
16240 }
16242 /* Output code for x87 ffreep insn. The OPNO argument, which may only
16243 have the values zero or one, indicates the ffreep insn's operand
16244 from the OPERANDS array. */
16248 {
16250 #ifdef HAVE_AS_IX86_FFREEP
16252 #else
16253 {
16263 }
16264 #endif
16267 }
16270 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
16271 should be used. UNORDERED_P is true when fucom should be used. */
16275 {
16281 {
16284 }
16285 else
16286 {
16289 }
16292 {
16296 else
16298 else
16301 else
16303 }
16310 {
16312 {
16315 }
16316 else
16318 }
16321 && stack_top_dies
16324 {
16325 /* If both the top of the 387 stack dies, and the other operand
16326 is also a stack register that dies, then this must be a
16327 `fcompp' float compare */
16330 {
16331 /* There is no double popping fcomi variant. Fortunately,
16332 eflags is immune from the fstp's cc clobbering. */
16335 else
16338 }
16339 else
16340 {
16343 else
16345 }
16346 }
16347 else
16348 {
16349 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
16352 {
16360 NULL,
16361 NULL,
16368 NULL,
16369 NULL,
16370 NULL,
16371 NULL
16372 };
16387 }
16388 }
16390 void
16392 {
16395 #ifdef ASM_QUAD
16398 #else
16400 #endif
16403 }
16405 void
16407 {
16410 #ifdef ASM_QUAD
16413 #else
16415 #endif
16416 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
16422 #if TARGET_MACHO
16424 {
16428 }
16429 #endif
16430 else
16433 }
16434 \f
16435 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
16436 for the target. */
16438 void
16440 {
16441 rtx tmp;
16443 /* We play register width games, which are only valid after reload. */
16446 /* Avoid HImode and its attendant prefix byte. */
16451 /* This predicate should match that for movsi_xor and movdi_xor_rex64. */
16453 {
16456 }
16459 }
16461 /* X is an unchanging MEM. If it is a constant pool reference, return
16462 the constant pool rtx, else NULL. */
16464 rtx
16466 {
16473 }
16475 void
16477 {
16485 {
16486 rtx tmp;
16490 {
16496 }
16499 }
16503 {
16506 rtx tmp;
16511 else
16515 {
16522 }
16523 }
16527 {
16529 {
16530 #if TARGET_MACHO
16531 /* dynamic-no-pic */
16533 {
16534 rtx temp = ((reload_in_progress
16542 }
16544 {
16548 }
16551 else
16552 {
16560 }
16561 /* dynamic-no-pic */
16562 #endif
16563 }
16564 else
16565 {
16569 {
16575 }
16576 }
16577 }
16578 else
16579 {
16590 /* Force large constants in 64bit compilation into register
16591 to get them CSEed. */
16603 {
16604 /* If we are loading a floating point constant to a register,
16605 force the value to memory now, since we'll get better code
16606 out the back end. */
16610 {
16615 }
16616 }
16617 }
16620 }
16622 void
16624 {
16628 /* Force constants other than zero into memory. We do not know how
16629 the instructions used to build constants modify the upper 64 bits
16630 of the register, once we have that information we may be able
16631 to handle some of them more efficiently. */
16640 /* We need to check memory alignment for SSE mode since attribute
16641 can make operands unaligned. */
16646 {
16649 /* ix86_expand_vector_move_misalign() does not like constants ... */
16655 /* ... nor both arguments in memory. */
16663 }
16665 /* Make operand1 a register if it isn't already. */
16669 {
16672 }
16675 }
16677 /* Split 32-byte AVX unaligned load and store if needed. */
16679 static void
16681 {
16682 rtx m;
16689 {
16710 }
16713 {
16715 {
16722 }
16723 else
16725 }
16727 {
16729 {
16734 }
16735 else
16737 }
16738 else
16740 }
16742 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
16743 straight to ix86_expand_vector_move. */
16744 /* Code generation for scalar reg-reg moves of single and double precision data:
16745 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
16746 movaps reg, reg
16747 else
16748 movss reg, reg
16749 if (x86_sse_partial_reg_dependency == true)
16750 movapd reg, reg
16751 else
16752 movsd reg, reg
16754 Code generation for scalar loads of double precision data:
16755 if (x86_sse_split_regs == true)
16756 movlpd mem, reg (gas syntax)
16757 else
16758 movsd mem, reg
16760 Code generation for unaligned packed loads of single precision data
16761 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
16762 if (x86_sse_unaligned_move_optimal)
16763 movups mem, reg
16765 if (x86_sse_partial_reg_dependency == true)
16766 {
16767 xorps reg, reg
16768 movlps mem, reg
16769 movhps mem+8, reg
16770 }
16771 else
16772 {
16773 movlps mem, reg
16774 movhps mem+8, reg
16775 }
16777 Code generation for unaligned packed loads of double precision data
16778 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
16779 if (x86_sse_unaligned_move_optimal)
16780 movupd mem, reg
16782 if (x86_sse_split_regs == true)
16783 {
16784 movlpd mem, reg
16785 movhpd mem+8, reg
16786 }
16787 else
16788 {
16789 movsd mem, reg
16790 movhpd mem+8, reg
16791 }
16792 */
16794 void
16796 {
16804 {
16806 {
16811 /* FALLTHRU */
16819 }
16822 }
16825 {
16826 /* ??? If we have typed data, then it would appear that using
16827 movdqu is the only way to get unaligned data loaded with
16828 integer type. */
16830 {
16833 /* We will eventually emit movups based on insn attributes. */
16835 }
16837 {
16838 rtx zero;
16841 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16842 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16844 {
16845 /* We will eventually emit movups based on insn attributes. */
16848 }
16850 /* When SSE registers are split into halves, we can avoid
16851 writing to the top half twice. */
16853 {
16856 }
16857 else
16858 {
16859 /* ??? Not sure about the best option for the Intel chips.
16860 The following would seem to satisfy; the register is
16861 entirely cleared, breaking the dependency chain. We
16862 then store to the upper half, with a dependency depth
16863 of one. A rumor has it that Intel recommends two movsd
16864 followed by an unpacklpd, but this is unconfirmed. And
16865 given that the dependency depth of the unpacklpd would
16866 still be one, I'm not sure why this would be better. */
16868 }
16874 }
16875 else
16876 {
16878 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16879 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16881 {
16886 }
16890 else
16900 }
16901 }
16903 {
16905 {
16908 /* We will eventually emit movups based on insn attributes. */
16910 }
16912 {
16914 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
16915 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16917 /* We will eventually emit movups based on insn attributes. */
16919 else
16920 {
16925 }
16926 }
16927 else
16928 {
16933 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
16934 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16936 {
16939 }
16940 else
16941 {
16946 }
16947 }
16948 }
16949 else
16951 }
16953 /* Expand a push in MODE. This is some mode for which we do not support
16954 proper push instructions, at least from the registers that we expect
16955 the value to live in. */
16957 void
16959 {
16960 rtx tmp;
16970 /* When we push an operand onto stack, it has to be aligned at least
16971 at the function argument boundary. However since we don't have
16972 the argument type, we can't determine the actual argument
16973 boundary. */
16975 }
16977 /* Helper function of ix86_fixup_binary_operands to canonicalize
16978 operand order. Returns true if the operands should be swapped. */
16980 static bool
16982 rtx operands[])
16983 {
16988 /* If the operation is not commutative, we can't do anything. */
16992 /* Highest priority is that src1 should match dst. */
16998 /* Next highest priority is that immediate constants come second. */
17004 /* Lowest priority is that memory references should come second. */
17011 }
17014 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
17015 destination to use for the operation. If different from the true
17016 destination in operands[0], a copy operation will be required. */
17018 rtx
17020 rtx operands[])
17021 {
17026 /* Canonicalize operand order. */
17028 {
17029 rtx temp;
17031 /* It is invalid to swap operands of different modes. */
17037 }
17039 /* Both source operands cannot be in memory. */
17041 {
17042 /* Optimization: Only read from memory once. */
17044 {
17047 }
17048 else
17050 }
17052 /* If the destination is memory, and we do not have matching source
17053 operands, do things in registers. */
17057 /* Source 1 cannot be a constant. */
17061 /* Source 1 cannot be a non-matching memory. */
17065 /* Improve address combine. */
17074 }
17076 /* Similarly, but assume that the destination has already been
17077 set up properly. */
17079 void
17082 {
17085 }
17087 /* Attempt to expand a binary operator. Make the expansion closer to the
17088 actual machine, then just general_operand, which will allow 3 separate
17089 memory references (one output, two input) in a single insn. */
17091 void
17093 rtx operands[])
17094 {
17101 /* Emit the instruction. */
17105 {
17106 /* Reload doesn't know about the flags register, and doesn't know that
17107 it doesn't want to clobber it. We can only do this with PLUS. */
17110 }
17114 {
17115 /* This is going to be an LEA; avoid splitting it later. */
17117 }
17118 else
17119 {
17122 }
17124 /* Fix up the destination if needed. */
17127 }
17129 /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
17130 the given OPERANDS. */
17132 void
17134 rtx operands[])
17135 {
17138 {
17141 }
17143 {
17146 }
17147 /* Optimize (__m128i) d | (__m128i) e and similar code
17148 when d and e are float vectors into float vector logical
17149 insn. In C/C++ without using intrinsics there is no other way
17150 to express vector logical operation on float vectors than
17151 to cast them temporarily to integer vectors. */
17153 && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17162 {
17163 rtx dst;
17165 {
17172 {
17175 }
17176 else
17177 {
17182 }
17193 }
17194 }
17203 }
17205 /* Return TRUE or FALSE depending on whether the binary operator meets the
17206 appropriate constraints. */
17208 bool
17211 {
17216 /* Both source operands cannot be in memory. */
17220 /* Canonicalize operand order for commutative operators. */
17222 {
17226 }
17228 /* If the destination is memory, we must have a matching source operand. */
17232 /* Source 1 cannot be a constant. */
17236 /* Source 1 cannot be a non-matching memory. */
17238 /* Support "andhi/andsi/anddi" as a zero-extending move. */
17246 }
17248 /* Attempt to expand a unary operator. Make the expansion closer to the
17249 actual machine, then just general_operand, which will allow 2 separate
17250 memory references (one output, one input) in a single insn. */
17252 void
17254 rtx operands[])
17255 {
17262 /* If the destination is memory, and we do not have matching source
17263 operands, do things in registers. */
17266 {
17269 else
17271 }
17273 /* When source operand is memory, destination must match. */
17277 /* Emit the instruction. */
17281 {
17282 /* Reload doesn't know about the flags register, and doesn't know that
17283 it doesn't want to clobber it. */
17286 }
17287 else
17288 {
17291 }
17293 /* Fix up the destination if needed. */
17296 }
17298 /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
17299 divisor are within the range [0-255]. */
17301 void
17304 {
17313 {
17326 }
17333 /* Use 8bit unsigned divimod if dividend and divisor are within
17334 the range [0-255]. */
17343 pc_rtx);
17348 /* Generate original signed/unsigned divimod. */
17353 /* Branch to the end. */
17357 /* Generate 8bit unsigned divide. */
17359 /* Don't use operands[0] for result of 8bit divide since not all
17360 registers support QImode ZERO_EXTRACT. */
17367 {
17370 }
17371 else
17372 {
17375 }
17377 /* Extract remainder from AH. */
17381 else
17382 {
17383 /* Need a new scratch register since the old one has result
17384 of 8bit divide. */
17388 }
17391 /* Zero extend quotient from AL. */
17397 }
17399 #define LEA_MAX_STALL (3)
17400 #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
17402 /* Increase given DISTANCE in half-cycles according to
17403 dependencies between PREV and NEXT instructions.
17404 Add 1 half-cycle if there is no dependency and
17405 go to next cycle if there is some dependecy. */
17407 static unsigned int
17409 {
17426 }
17428 /* Function checks if instruction INSN defines register number
17429 REGNO1 or REGNO2. */
17431 static bool
17433 rtx insn)
17434 {
17442 {
17444 }
17447 }
17449 /* Function checks if instruction INSN uses register number
17450 REGNO as a part of address expression. */
17452 static bool
17454 {
17462 }
17464 /* Search backward for non-agu definition of register number REGNO1
17465 or register number REGNO2 in basic block starting from instruction
17466 START up to head of basic block or instruction INSN.
17468 Function puts true value into *FOUND var if definition was found
17469 and false otherwise.
17471 Distance in half-cycles between START and found instruction or head
17472 of BB is added to DISTANCE and returned. */
17474 static int
17478 {
17488 {
17490 {
17493 {
17496 {
17499 }
17500 }
17503 }
17508 }
17511 }
17513 /* Search backward for non-agu definition of register number REGNO1
17514 or register number REGNO2 in INSN's basic block until
17515 1. Pass LEA_SEARCH_THRESHOLD instructions, or
17516 2. Reach neighbour BBs boundary, or
17517 3. Reach agu definition.
17518 Returns the distance between the non-agu definition point and INSN.
17519 If no definition point, returns -1. */
17521 static int
17523 rtx insn)
17524 {
17535 {
17536 edge e;
17537 edge_iterator ei;
17542 {
17545 }
17551 else
17552 {
17557 {
17558 int bb_dist
17564 {
17571 }
17572 }
17575 }
17576 }
17578 /* get_attr_type may modify recog data. We want to make sure
17579 that recog data is valid for instruction INSN, on which
17580 distance_non_agu_define is called. INSN is unchanged here. */
17587 }
17589 /* Return the distance in half-cycles between INSN and the next
17590 insn that uses register number REGNO in memory address added
17591 to DISTANCE. Return -1 if REGNO0 is set.
17593 Put true value into *FOUND if register usage was found and
17594 false otherwise.
17595 Put true value into *REDEFINED if register redefinition was
17596 found and false otherwise. */
17598 static int
17602 {
17613 {
17615 {
17618 {
17619 /* Return DISTANCE if OP0 is used in memory
17620 address in NEXT. */
17623 }
17626 {
17627 /* Return -1 if OP0 is set in NEXT. */
17630 }
17633 }
17639 }
17642 }
17644 /* Return the distance between INSN and the next insn that uses
17645 register number REGNO0 in memory address. Return -1 if no such
17646 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
17648 static int
17650 {
17662 {
17663 edge e;
17664 edge_iterator ei;
17669 {
17672 }
17678 else
17679 {
17685 {
17686 int bb_dist
17691 {
17698 }
17699 }
17702 }
17703 }
17709 }
17711 /* Define this macro to tune LEA priority vs ADD, it take effect when
17712 there is a dilemma of choicing LEA or ADD
17713 Negative value: ADD is more preferred than LEA
17714 Zero: Netrual
17715 Positive value: LEA is more preferred than ADD*/
17716 #define IX86_LEA_PRIORITY 0
17718 /* Return true if usage of lea INSN has performance advantage
17719 over a sequence of instructions. Instructions sequence has
17720 SPLIT_COST cycles higher latency than lea latency. */
17722 static bool
17725 {
17728 /* For Silvermont if using a 2-source or 3-source LEA for
17729 non-destructive destination purposes, or due to wanting
17730 ability to use SCALE, the use of LEA is justified. */
17732 {
17740 }
17746 {
17747 /* If there is no non AGU operand definition, no AGU
17748 operand usage and split cost is 0 then both lea
17749 and non lea variants have same priority. Currently
17750 we prefer lea for 64 bit code and non lea on 32 bit
17751 code. */
17754 else
17756 }
17758 /* With longer definitions distance lea is more preferable.
17759 Here we change it to take into account splitting cost and
17760 lea priority. */
17763 /* If there is no use in memory addess then we just check
17764 that split cost exceeds AGU stall. */
17768 /* If this insn has both backward non-agu dependence and forward
17769 agu dependence, the one with short distance takes effect. */
17771 }
17773 /* Return true if it is legal to clobber flags by INSN and
17774 false otherwise. */
17776 static bool
17778 {
17781 bitmap live;
17784 {
17786 {
17793 }
17799 }
17803 }
17805 /* Return true if we need to split op0 = op1 + op2 into a sequence of
17806 move and add to avoid AGU stalls. */
17808 bool
17810 {
17813 /* Check if we need to optimize. */
17817 /* Check it is correct to split here. */
17825 /* We need to split only adds with non destructive
17826 destination operand. */
17829 else
17831 }
17833 /* Return true if we should emit lea instruction instead of mov
17834 instruction. */
17836 bool
17838 {
17841 /* Check if we need to optimize. */
17845 /* Use lea for reg to reg moves only. */
17853 }
17855 /* Return true if we need to split lea into a sequence of
17856 instructions to avoid AGU stalls. */
17858 bool
17860 {
17866 /* Check we need to optimize. */
17870 /* Check it is correct to split here. */
17877 /* There should be at least two components in the address. */
17882 /* We should not split into add if non legitimate pic
17883 operand is used as displacement. */
17898 /* Compute how many cycles we will add to execution time
17899 if split lea into a sequence of instructions. */
17901 {
17902 /* Have to use mov instruction if non desctructive
17903 destination form is used. */
17907 /* Have to add index to base if both exist. */
17911 /* Have to use shift and adds if scale is 2 or greater. */
17913 {
17918 else
17920 }
17922 /* Have to use add instruction with immediate if
17923 disp is non zero. */
17927 /* Subtract the price of lea. */
17929 }
17933 }
17935 /* Emit x86 binary operand CODE in mode MODE, where the first operand
17936 matches destination. RTX includes clobber of FLAGS_REG. */
17938 static void
17941 {
17948 }
17950 /* Return true if regno1 def is nearest to the insn. */
17952 static bool
17954 {
17961 {
17963 {
17966 }
17972 }
17974 /* None of the regs is defined in the bb. */
17976 }
17978 /* Split lea instructions into a sequence of instructions
17979 which are executed on ALU to avoid AGU stalls.
17980 It is assumed that it is allowed to clobber flags register
17981 at lea position. */
17983 void
17985 {
18001 {
18004 }
18007 {
18010 }
18016 {
18017 /* Case r1 = r1 + ... */
18019 {
18020 /* If we have a case r1 = r1 + C * r1 then we
18021 should use multiplication which is very
18022 expensive. Assume cost model is wrong if we
18023 have such case here. */
18028 }
18029 else
18030 {
18031 /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
18035 /* Use shift for scaling. */
18044 }
18045 }
18047 {
18050 }
18051 else
18052 {
18054 {
18057 }
18059 {
18062 }
18063 else
18064 {
18069 else
18070 {
18071 rtx tmp1;
18073 /* Find better operand for SET instruction, depending
18074 on which definition is farther from the insn. */
18077 else
18087 }
18090 }
18094 }
18095 }
18097 /* Return true if it is ok to optimize an ADD operation to LEA
18098 operation to avoid flag register consumation. For most processors,
18099 ADD is faster than LEA. For the processors like ATOM, if the
18100 destination register of LEA holds an actual address which will be
18101 used soon, LEA is better and otherwise ADD is better. */
18103 bool
18105 {
18110 /* If a = b + c, (a!=b && a!=c), must use lea form. */
18118 }
18120 /* Return true if destination reg of SET_BODY is shift count of
18121 USE_BODY. */
18123 static bool
18125 {
18126 rtx set_dest;
18127 rtx shift_rtx;
18130 /* Retrieve destination of SET_BODY. */
18132 {
18141 use_body))
18146 }
18148 /* Retrieve shift count of USE_BODY. */
18150 {
18162 }
18170 {
18173 /* Return true if shift count is dest of SET_BODY. */
18175 {
18176 /* Add check since it can be invoked before register
18177 allocation in pre-reload schedule. */
18183 }
18184 }
18187 }
18189 /* Return true if destination reg of SET_INSN is shift count of
18190 USE_INSN. */
18192 bool
18194 {
18197 }
18199 /* Return TRUE or FALSE depending on whether the unary operator meets the
18200 appropriate constraints. */
18202 bool
18206 {
18207 /* If one of operands is memory, source and destination must match. */
18213 }
18215 /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
18216 are ok, keeping in mind the possible movddup alternative. */
18218 bool
18220 {
18226 }
18228 /* Post-reload splitter for converting an SF or DFmode value in an
18229 SSE register into an unsigned SImode. */
18231 void
18233 {
18244 /* Load up the value into the low element. We must ensure that the other
18245 elements are valid floats -- zero is the easiest such value. */
18247 {
18250 else
18252 }
18253 else
18254 {
18259 else
18261 }
18281 else
18286 }
18288 /* Convert an unsigned DImode value into a DFmode, using only SSE.
18289 Expects the 64-bit DImode to be supplied in a pair of integral
18290 registers. Requires SSE2; will use SSE3 if available. For x86_32,
18291 -mfpmath=sse, !optimize_size only. */
18293 void
18295 {
18299 rtx x;
18305 {
18308 }
18309 else
18310 {
18314 }
18322 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
18325 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
18326 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
18327 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
18328 (0x1.0p84 + double(fp_value_hi_xmm)).
18329 Note these exponents differ by 32. */
18333 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
18334 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
18343 /* Add the upper and lower DFmode values together. */
18346 else
18347 {
18351 }
18354 }
18356 /* Not used, but eases macroization of patterns. */
18357 void
18359 rtx input ATTRIBUTE_UNUSED)
18360 {
18362 }
18364 /* Convert an unsigned SImode value into a DFmode. Only currently used
18365 for SSE, but applicable anywhere. */
18367 void
18369 {
18370 REAL_VALUE_TYPE TWO31r;
18385 }
18387 /* Convert a signed DImode value into a DFmode. Only used for SSE in
18388 32-bit mode; otherwise we have a direct convert instruction. */
18390 void
18392 {
18393 REAL_VALUE_TYPE TWO32r;
18411 }
18413 /* Convert an unsigned SImode value into a SFmode, using only SSE.
18414 For x86_32, -mfpmath=sse, !optimize_size only. */
18415 void
18417 {
18418 REAL_VALUE_TYPE ONE16r;
18437 }
18439 /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
18440 a vector of unsigned ints VAL to vector of floats TARGET. */
18442 void
18444 {
18446 REAL_VALUE_TYPE TWO16r;
18453 else
18458 OPTAB_DIRECT);
18469 OPTAB_DIRECT);
18471 OPTAB_DIRECT);
18474 }
18476 /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
18477 pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
18478 This is done by doing just signed conversion if < 0x1p31, and otherwise by
18479 subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
18481 rtx
18483 {
18484 REAL_VALUE_TYPE TWO31r;
18499 {
18505 }
18514 OPTAB_DIRECT);
18515 else
18516 {
18522 OPTAB_DIRECT);
18523 }
18526 }
18528 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
18529 then replicate the value for all elements of the vector
18530 register. */
18532 rtx
18534 {
18536 rtvec v;
18540 {
18567 }
18568 }
18570 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
18571 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
18572 for an SSE register. If VECT is true, then replicate the mask for
18573 all elements of the vector register. If INVERT is true, then create
18574 a mask excluding the sign bit. */
18576 rtx
18578 {
18582 rtx v;
18583 rtx mask;
18585 /* Find the sign bit, sign extended to 2*HWI. */
18587 {
18607 else
18615 {
18618 }
18619 else
18620 {
18621 rtvec vec;
18627 {
18630 }
18631 else
18641 }
18646 }
18651 /* Force this value into the low part of a fp vector constant. */
18660 }
18662 /* Generate code for floating point ABS or NEG. */
18664 void
18666 rtx operands[])
18667 {
18678 {
18684 }
18686 /* NEG and ABS performed with SSE use bitwise mask operations.
18687 Create the appropriate mask now. */
18690 else
18700 {
18702 rtvec par;
18707 else
18708 {
18711 }
18713 }
18714 else
18716 }
18718 /* Expand a copysign operation. Special case operand 0 being a constant. */
18720 void
18722 {
18736 else
18740 {
18747 {
18750 else
18751 {
18755 }
18756 }
18766 else
18770 }
18771 else
18772 {
18782 else
18786 }
18787 }
18789 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
18790 be a constant, and so has already been expanded into a vector constant. */
18792 void
18794 {
18810 {
18813 }
18814 }
18816 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
18817 so we have to do two masks. */
18819 void
18821 {
18836 {
18837 /* Shouldn't happen often (it's useless, obviously), but when it does
18838 we'd generate incorrect code if we continue below. */
18841 }
18844 {
18855 }
18856 else
18857 {
18859 {
18861 }
18863 {
18867 }
18871 {
18874 }
18876 {
18881 }
18883 }
18887 }
18889 /* Return TRUE or FALSE depending on whether the first SET in INSN
18890 has source and destination with matching CC modes, and that the
18891 CC mode is at least as constrained as REQ_MODE. */
18893 bool
18895 {
18896 rtx set;
18907 {
18917 /* FALLTHRU */
18921 /* FALLTHRU */
18925 /* FALLTHRU */
18939 }
18942 }
18944 /* Generate insn patterns to do an integer compare of OPERANDS. */
18946 static rtx
18948 {
18955 /* This is very simple, but making the interface the same as in the
18956 FP case makes the rest of the code easier. */
18960 /* Return the test that should be put into the flags user, i.e.
18961 the bcc, scc, or cmov instruction. */
18963 }
18965 /* Figure out whether to use ordered or unordered fp comparisons.
18966 Return the appropriate mode to use. */
18968 enum machine_mode
18970 {
18971 /* ??? In order to make all comparisons reversible, we do all comparisons
18972 non-trapping when compiling for IEEE. Once gcc is able to distinguish
18973 all forms trapping and nontrapping comparisons, we can make inequality
18974 comparisons trapping again, since it results in better code when using
18975 FCOM based compares. */
18977 }
18979 enum machine_mode
18981 {
18985 {
18988 }
18991 {
18992 /* Only zero flag is needed. */
18996 /* Codes needing carry flag. */
18999 /* Detect overflow checks. They need just the carry flag. */
19003 else
19007 /* Detect overflow checks. They need just the carry flag. */
19011 else
19013 /* Codes possibly doable only with sign flag when
19014 comparing against zero. */
19019 else
19020 /* For other cases Carry flag is not required. */
19022 /* Codes doable only with sign flag when comparing
19023 against zero, but we miss jump instruction for it
19024 so we need to use relational tests against overflow
19025 that thus needs to be zero. */
19030 else
19032 /* strcmp pattern do (use flags) and combine may ask us for proper
19033 mode. */
19038 }
19039 }
19041 /* Return the fixed registers used for condition codes. */
19043 static bool
19045 {
19049 }
19051 /* If two condition code modes are compatible, return a condition code
19052 mode which is compatible with both. Otherwise, return
19053 VOIDmode. */
19055 static enum machine_mode
19057 {
19074 {
19088 {
19102 }
19106 /* These are only compatible with themselves, which we already
19107 checked above. */
19109 }
19110 }
19113 /* Return a comparison we can do and that it is equivalent to
19114 swap_condition (code) apart possibly from orderedness.
19115 But, never change orderedness if TARGET_IEEE_FP, returning
19116 UNKNOWN in that case if necessary. */
19118 static enum rtx_code
19120 {
19122 {
19133 }
19134 }
19136 /* Return cost of comparison CODE using the best strategy for performance.
19137 All following functions do use number of instructions as a cost metrics.
19138 In future this should be tweaked to compute bytes for optimize_size and
19139 take into account performance of various instructions on various CPUs. */
19141 static int
19143 {
19146 /* The cost of code using bit-twiddling on %ah. */
19148 {
19171 }
19174 {
19181 }
19182 }
19184 /* Return strategy to use for floating-point. We assume that fcomi is always
19185 preferrable where available, since that is also true when looking at size
19186 (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
19188 enum ix86_fpcmp_strategy
19190 {
19191 /* Do fcomi/sahf based test when profitable. */
19200 }
19202 /* Swap, force into registers, or otherwise massage the two operands
19203 to a fp comparison. The operands are updated in place; the new
19204 comparison code is returned. */
19206 static enum rtx_code
19208 {
19214 /* All of the unordered compare instructions only work on registers.
19215 The same is true of the fcomi compare instructions. The XFmode
19216 compare instructions require registers except when comparing
19217 against zero or when converting operand 1 from fixed point to
19218 floating point. */
19227 {
19230 }
19231 else
19232 {
19233 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
19234 things around if they appear profitable, otherwise force op0
19235 into a register. */
19241 {
19244 {
19245 rtx tmp;
19248 }
19249 }
19255 {
19260 {
19263 }
19264 else
19266 }
19267 }
19269 /* Try to rearrange the comparison to make it cheaper. */
19273 {
19274 rtx tmp;
19279 }
19284 }
19286 /* Convert comparison codes we use to represent FP comparison to integer
19287 code that will result in proper branch. Return UNKNOWN if no such code
19288 is available. */
19290 enum rtx_code
19292 {
19294 {
19317 }
19318 }
19320 /* Generate insn patterns to do a floating point compare of OPERANDS. */
19322 static rtx
19324 {
19331 /* Do fcomi/sahf based test when profitable. */
19333 {
19338 tmp);
19346 tmp);
19355 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
19362 /* In the unordered case, we have to check C2 for NaN's, which
19363 doesn't happen to work out to anything nice combination-wise.
19364 So do some bit twiddling on the value we've got in AH to come
19365 up with an appropriate set of condition codes. */
19369 {
19373 {
19376 }
19377 else
19378 {
19384 }
19389 {
19394 }
19395 else
19396 {
19399 }
19404 {
19407 }
19408 else
19409 {
19413 }
19418 {
19424 }
19425 else
19426 {
19429 }
19434 {
19439 }
19440 else
19441 {
19444 }
19449 {
19454 }
19455 else
19456 {
19459 }
19473 }
19478 }
19480 /* Return the test that should be put into the flags user, i.e.
19481 the bcc, scc, or cmov instruction. */
19484 const0_rtx);
19485 }
19487 static rtx
19489 {
19490 rtx ret;
19496 {
19499 }
19500 else
19504 }
19506 void
19508 {
19510 rtx tmp;
19513 {
19520 simple:
19524 pc_rtx);
19532 /* Expand DImode branch into multiple compare+branch. */
19533 {
19539 {
19542 }
19549 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
19550 avoid two branches. This costs one extra insn, so disable when
19551 optimizing for size. */
19556 {
19574 }
19576 /* Otherwise, if we are doing less-than or greater-or-equal-than,
19577 op1 is a constant and the low word is zero, then we can just
19578 examine the high word. Similarly for low word -1 and
19579 less-or-equal-than or greater-than. */
19583 {
19586 {
19589 }
19593 {
19596 }
19600 }
19602 /* Otherwise, we need two or three jumps. */
19611 {
19625 }
19627 /*
19628 * a < b =>
19629 * if (hi(a) < hi(b)) goto true;
19630 * if (hi(a) > hi(b)) goto false;
19631 * if (lo(a) < lo(b)) goto true;
19632 * false:
19633 */
19645 }
19650 }
19651 }
19653 /* Split branch based on floating point condition. */
19654 void
19657 {
19658 rtx condition;
19659 rtx i;
19662 {
19667 }
19670 tmp);
19672 /* Remove pushed operand from stack. */
19682 }
19684 void
19686 {
19687 rtx ret;
19694 }
19696 /* Expand comparison setting or clearing carry flag. Return true when
19697 successful and set pop for the operation. */
19698 static bool
19700 {
19704 /* Do not handle double-mode compares that go through special path. */
19709 {
19714 /* Shortcut: following common codes never translate
19715 into carry flag compares. */
19720 /* These comparisons require zero flag; swap operands so they won't. */
19723 {
19728 }
19730 /* Try to expand the comparison and verify that we end up with
19731 carry flag based comparison. This fails to be true only when
19732 we decide to expand comparison using arithmetic that is not
19733 too common scenario. */
19742 else
19751 }
19757 {
19762 /* Convert a==0 into (unsigned)a<1. */
19771 /* Convert a>b into b<a or a>=b-1. */
19775 {
19777 /* Bail out on overflow. We still can swap operands but that
19778 would force loading of the constant into register. */
19783 }
19784 else
19785 {
19790 }
19793 /* Convert a>=0 into (unsigned)a<0x80000000. */
19811 }
19812 /* Swapping operands may cause constant to appear as first operand. */
19814 {
19818 }
19822 }
19824 bool
19826 {
19850 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
19851 HImode insns, we'd be swallowed in word prefix ops. */
19857 {
19861 HOST_WIDE_INT diff;
19864 /* Sign bit compares are better done using shifts than we do by using
19865 sbb. */
19868 {
19869 /* Detect overlap between destination and compare sources. */
19873 {
19874 rtx flags;
19883 {
19885 compare_code
19887 }
19889 /* To simplify rest of code, restrict to the GEU case. */
19891 {
19897 }
19898 else
19899 {
19902 reverse_condition_maybe_unordered
19904 else
19907 }
19916 else
19919 }
19920 else
19921 {
19924 else
19925 {
19930 }
19932 }
19935 {
19936 /*
19937 * cmpl op0,op1
19938 * sbbl dest,dest
19939 * [addl dest, ct]
19940 *
19941 * Size 5 - 8.
19942 */
19947 }
19949 {
19950 /*
19951 * cmpl op0,op1
19952 * sbbl dest,dest
19953 * orl $ct, dest
19954 *
19955 * Size 8.
19956 */
19960 }
19962 {
19963 /*
19964 * cmpl op0,op1
19965 * sbbl dest,dest
19966 * notl dest
19967 * [addl dest, cf]
19968 *
19969 * Size 8 - 11.
19970 */
19976 }
19977 else
19978 {
19979 /*
19980 * cmpl op0,op1
19981 * sbbl dest,dest
19982 * [notl dest]
19983 * andl cf - ct, dest
19984 * [addl dest, ct]
19985 *
19986 * Size 8 - 11.
19987 */
19990 {
19994 }
20004 }
20010 }
20013 {
20016 HOST_WIDE_INT tmp;
20021 {
20024 /* We may be reversing unordered compare to normal compare, that
20025 is not valid in general (we may convert non-trapping condition
20026 to trapping one), however on i386 we currently emit all
20027 comparisons unordered. */
20030 }
20031 else
20032 {
20035 }
20036 }
20041 {
20046 {
20051 }
20052 }
20054 /* Optimize dest = (op0 < 0) ? -1 : cf. */
20058 {
20059 /* If lea code below could be used, only optimize
20060 if it results in a 2 insn sequence. */
20066 {
20067 /*
20068 * notl op1 (if necessary)
20069 * sarl $31, op1
20070 * orl cf, op1
20071 */
20073 {
20077 }
20088 }
20089 }
20097 {
20098 /*
20099 * xorl dest,dest
20100 * cmpl op1,op2
20101 * setcc dest
20102 * lea cf(dest*(ct-cf)),dest
20103 *
20104 * Size 14.
20105 *
20106 * This also catches the degenerate setcc-only case.
20107 */
20109 rtx tmp;
20115 /* On x86_64 the lea instruction operates on Pmode, so we need
20116 to get arithmetics done in proper mode to match. */
20119 else
20120 {
20121 rtx out1;
20124 nops++;
20126 {
20128 nops++;
20129 }
20130 }
20132 {
20134 nops++;
20135 }
20137 {
20140 else
20142 }
20147 }
20149 /*
20150 * General case: Jumpful:
20151 * xorl dest,dest cmpl op1, op2
20152 * cmpl op1, op2 movl ct, dest
20153 * setcc dest jcc 1f
20154 * decl dest movl cf, dest
20155 * andl (cf-ct),dest 1:
20156 * addl ct,dest
20157 *
20158 * Size 20. Size 14.
20159 *
20160 * This is reasonably steep, but branch mispredict costs are
20161 * high on modern cpus, so consider failing only if optimizing
20162 * for space.
20163 */
20168 {
20170 {
20177 {
20180 /* We may be reversing unordered compare to normal compare,
20181 that is not valid in general (we may convert non-trapping
20182 condition to trapping one), however on i386 we currently
20183 emit all comparisons unordered. */
20185 }
20186 else
20187 {
20191 }
20192 }
20195 {
20196 /* notl op1 (if needed)
20197 sarl $31, op1
20198 andl (cf-ct), op1
20199 addl ct, op1
20201 For x < 0 (resp. x <= -1) there will be no notl,
20202 so if possible swap the constants to get rid of the
20203 complement.
20204 True/false will be -1/0 while code below (store flag
20205 followed by decrement) is 0/-1, so the constants need
20206 to be exchanged once more. */
20209 {
20212 }
20213 else
20214 {
20218 }
20221 }
20222 else
20223 {
20227 constm1_rtx,
20229 }
20241 }
20242 }
20245 {
20246 /* Try a few things more with specific constants and a variable. */
20248 optab op;
20254 /* If one of the two operands is an interesting constant, load a
20255 constant with the above and mask it in with a logical operation. */
20258 {
20264 else
20266 }
20268 {
20274 else
20276 }
20277 else
20284 /* Recurse to get the constant loaded. */
20288 /* Mask in the interesting variable. */
20290 OPTAB_WIDEN);
20295 }
20297 /*
20298 * For comparison with above,
20299 *
20300 * movl cf,dest
20301 * movl ct,tmp
20302 * cmpl op1,op2
20303 * cmovcc tmp,dest
20304 *
20305 * Size 15.
20306 */
20328 }
20330 /* Swap, force into registers, or otherwise massage the two operands
20331 to an sse comparison with a mask result. Thus we differ a bit from
20332 ix86_prepare_fp_compare_args which expects to produce a flags result.
20334 The DEST operand exists to help determine whether to commute commutative
20335 operators. The POP0/POP1 operands are updated in place. The new
20336 comparison code is returned, or UNKNOWN if not implementable. */
20338 static enum rtx_code
20341 {
20342 rtx tmp;
20345 {
20348 /* AVX supports all the needed comparisons. */
20351 /* We have no LTGT as an operator. We could implement it with
20352 NE & ORDERED, but this requires an extra temporary. It's
20353 not clear that it's worth it. */
20360 /* These are supported directly. */
20367 /* AVX has 3 operand comparisons, no need to swap anything. */
20370 /* For commutative operators, try to canonicalize the destination
20371 operand to be first in the comparison - this helps reload to
20372 avoid extra moves. */
20375 /* FALLTHRU */
20381 /* These are not supported directly before AVX, and furthermore
20382 ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
20383 comparison operands to transform into something that is
20384 supported. */
20393 }
20396 }
20398 /* Detect conditional moves that exactly match min/max operational
20399 semantics. Note that this is IEEE safe, as long as we don't
20400 interchange the operands.
20402 Returns FALSE if this conditional move doesn't match a MIN/MAX,
20403 and TRUE if the operation is successful and instructions are emitted. */
20405 static bool
20408 {
20411 rtx tmp;
20414 ;
20416 {
20420 }
20421 else
20428 else
20433 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
20434 but MODE may be a vector mode and thus not appropriate. */
20436 {
20438 rtvec v;
20443 }
20444 else
20445 {
20448 }
20452 }
20454 /* Expand an sse vector comparison. Return the register with the result. */
20456 static rtx
20459 {
20462 rtx x;
20475 {
20478 }
20479 else
20483 }
20485 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
20486 operations. This is used for both scalar and vector conditional moves. */
20488 static void
20490 {
20496 {
20498 }
20500 {
20504 }
20506 {
20511 }
20513 {
20517 }
20519 {
20527 op_true,
20528 op_false)));
20529 }
20530 else
20531 {
20540 {
20554 {
20560 }
20575 {
20581 }
20585 }
20589 else
20590 {
20596 else
20608 }
20609 }
20610 }
20612 /* Expand a floating-point conditional move. Return true if successful. */
20614 bool
20616 {
20624 {
20627 /* Since we've no cmove for sse registers, don't force bad register
20628 allocation just to gain access to it. Deny movcc when the
20629 comparison mode doesn't match the move mode. */
20648 }
20655 /* The floating point conditional move instructions don't directly
20656 support conditions resulting from a signed integer comparison. */
20660 {
20665 }
20672 }
20674 /* Expand a floating-point vector conditional move; a vcond operation
20675 rather than a movcc operation. */
20677 bool
20679 {
20681 rtx cmp;
20686 {
20687 rtx temp;
20689 {
20706 }
20708 OPTAB_DIRECT);
20711 }
20721 }
20723 /* Expand a signed/unsigned integral vector conditional move. */
20725 bool
20727 {
20737 /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
20738 and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
20747 {
20751 {
20757 }
20760 {
20766 }
20767 }
20776 /* XOP supports all of the comparisons on all 128-bit vector int types. */
20780 ;
20781 else
20782 {
20783 /* Canonicalize the comparison to EQ, GT, GTU. */
20785 {
20802 /* FALLTHRU */
20812 }
20814 /* Only SSE4.1/SSE4.2 supports V2DImode. */
20816 {
20818 {
20820 /* SSE4.1 supports EQ. */
20827 /* SSE4.2 supports GT/GTU. */
20834 }
20835 }
20837 /* Unsigned parallel compare is not supported by the hardware.
20838 Play some tricks to turn this into a signed comparison
20839 against 0. */
20841 {
20845 {
20850 {
20855 {
20862 }
20863 /* Subtract (-(INT MAX) - 1) from both operands to make
20864 them signed. */
20875 }
20882 /* Perform a parallel unsigned saturating subtraction. */
20895 }
20896 }
20897 }
20899 /* Allow the comparison to be done in one mode, but the movcc to
20900 happen in another mode. */
20902 {
20905 }
20906 else
20907 {
20913 }
20918 }
20920 /* Expand a variable vector permutation. */
20922 void
20924 {
20935 /* Number of elements in the vector. */
20941 {
20943 {
20944 /* Unfortunately, the VPERMQ and VPERMPD instructions only support
20945 an constant shuffle operand. With a tiny bit of effort we can
20946 use VPERMD instead. A re-interpretation stall for V4DFmode is
20947 unfortunate but there's no avoiding it.
20948 Similarly for V16HImode we don't have instructions for variable
20949 shuffling, while for V32QImode we can use after preparing suitable
20950 masks vpshufb; vpshufb; vpermq; vpor. */
20953 {
20957 }
20958 else
20959 {
20963 }
20966 /* Replicate the low bits of the V4DImode mask into V8SImode:
20967 mask = { A B C D }
20968 t1 = { A A B B C C D D }. */
20976 else
20979 /* Multiply the shuffle indicies by two. */
20981 OPTAB_DIRECT);
20983 /* Add one to the odd shuffle indicies:
20984 t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
20986 {
20989 }
20993 OPTAB_DIRECT);
20995 /* Continue as if V8SImode (resp. V32QImode) was used initially. */
21000 }
21003 {
21005 /* The VPERMD and VPERMPS instructions already properly ignore
21006 the high bits of the shuffle elements. No need for us to
21007 perform an AND ourselves. */
21010 else
21011 {
21017 }
21024 else
21025 {
21031 }
21035 /* By combining the two 128-bit input vectors into one 256-bit
21036 input vector, we can use VPERMD and VPERMPS for the full
21037 two-operand shuffle. */
21069 /* From mask create two adjusted masks, which contain the same
21070 bits as mask in the low 7 bits of each vector element.
21071 The first mask will have the most significant bit clear
21072 if it requests element from the same 128-bit lane
21073 and MSB set if it requests element from the other 128-bit lane.
21074 The second mask will have the opposite values of the MSB,
21075 and additionally will have its 128-bit lanes swapped.
21076 E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
21077 t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
21078 t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
21079 stands for other 12 bytes. */
21080 /* The bit whether element is from the same lane or the other
21081 lane is bit 4, so shift it up by 3 to the MSB position. */
21085 /* Clear MSB bits from the mask just in case it had them set. */
21087 /* After this t1 will have MSB set for elements from other lane. */
21089 /* Clear bits other than MSB. */
21091 /* Or in the lower bits from mask into t3. */
21093 /* And invert MSB bits in t1, so MSB is set for elements from the same
21094 lane. */
21096 /* Swap 128-bit lanes in t3. */
21101 /* And or in the lower bits from mask into t1. */
21104 {
21105 /* Each of these shuffles will put 0s in places where
21106 element from the other 128-bit lane is needed, otherwise
21107 will shuffle in the requested value. */
21110 /* For t3 the 128-bit lanes are swapped again. */
21115 /* And oring both together leads to the result. */
21118 }
21121 /* Similarly to the above one_operand_shuffle code,
21122 just for repeated twice for each operand. merge_two:
21123 code will merge the two results together. */
21145 }
21146 }
21149 {
21150 /* The XOP VPPERM insn supports three inputs. By ignoring the
21151 one_operand_shuffle special case, we avoid creating another
21152 set of constant vectors in memory. */
21155 /* mask = mask & {2*w-1, ...} */
21157 }
21158 else
21159 {
21160 /* mask = mask & {w-1, ...} */
21162 }
21170 /* For non-QImode operations, convert the word permutation control
21171 into a byte permutation control. */
21173 {
21178 /* Convert mask to vector of chars. */
21181 /* Replicate each of the input bytes into byte positions:
21182 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
21183 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
21184 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
21191 else
21194 /* Convert it into the byte positions by doing
21195 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
21201 }
21203 /* The actual shuffle operations all operate on V16QImode. */
21209 {
21211 }
21213 {
21215 }
21216 else
21217 {
21221 /* Shuffle the two input vectors independently. */
21227 merge_two:
21228 /* Then merge them together. The key is whether any given control
21229 element contained a bit set that indicates the second word. */
21233 {
21234 /* Without SSE4.1, we don't have V2DImode EQ. Perform one
21235 more shuffle to convert the V2DI input mask into a V4SI
21236 input mask. At which point the masking that expand_int_vcond
21237 will work as desired. */
21245 }
21262 }
21263 }
21265 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
21266 true if we should do zero extension, else sign extension. HIGH_P is
21267 true if we want the N/2 high elements, else the low elements. */
21269 void
21271 {
21273 rtx tmp;
21276 {
21282 {
21286 else
21289 extract
21295 else
21298 extract
21304 else
21307 extract
21313 else
21319 else
21325 else
21330 }
21333 {
21336 }
21338 {
21339 /* Shift higher 8 bytes to lower 8 bytes. */
21344 }
21345 else
21349 }
21350 else
21351 {
21355 {
21359 else
21365 else
21371 else
21376 }
21380 else
21385 }
21386 }
21388 /* Expand conditional increment or decrement using adb/sbb instructions.
21389 The default case using setcc followed by the conditional move can be
21390 done by generic code. */
21391 bool
21393 {
21395 rtx flags;
21397 rtx compare_op;
21415 {
21418 }
21421 {
21425 reverse_condition_maybe_unordered
21427 else
21429 }
21433 /* Construct either adc or sbb insn. */
21435 {
21437 {
21452 }
21453 }
21454 else
21455 {
21457 {
21472 }
21473 }
21477 }
21480 /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
21481 but works for floating pointer parameters and nonoffsetable memories.
21482 For pushes, it returns just stack offsets; the values will be saved
21483 in the right order. Maximally three parts are generated. */
21485 static int
21487 {
21492 else
21498 /* Optimize constant pool reference to immediates. This is used by fp
21499 moves, that force all constants to memory to allow combining. */
21501 {
21505 }
21508 {
21509 /* The only non-offsetable memories we handle are pushes. */
21518 }
21521 {
21523 /* Caution: if we looked through a constant pool memory above,
21524 the operand may actually have a different mode now. That's
21525 ok, since we want to pun this all the way back to an integer. */
21529 }
21532 {
21535 else
21536 {
21540 {
21544 }
21546 {
21551 }
21553 {
21554 REAL_VALUE_TYPE r;
21559 {
21566 /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
21567 long double may not be 80-bit. */
21576 }
21579 }
21580 else
21582 }
21583 }
21584 else
21585 {
21589 {
21592 {
21596 }
21598 {
21602 }
21604 {
21605 REAL_VALUE_TYPE r;
21611 /* Do not use shift by 32 to avoid warning on 32bit systems. */
21614 = gen_int_mode
21617 DImode);
21618 else
21625 = gen_int_mode
21628 DImode);
21629 else
21631 }
21632 else
21634 }
21635 }
21638 }
21640 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
21641 Return false when normal moves are needed; true when all required
21642 insns have been emitted. Operands 2-4 contain the input values
21643 int the correct order; operands 5-7 contain the output values. */
21645 void
21647 {
21655 /* The DFmode expanders may ask us to move double.
21656 For 64bit target this is single move. By hiding the fact
21657 here we simplify i386.md splitters. */
21659 {
21660 /* Optimize constant pool reference to immediates. This is used by
21661 fp moves, that force all constants to memory to allow combining. */
21668 {
21671 }
21672 else
21677 }
21679 /* The only non-offsettable memory we handle is push. */
21682 else
21689 /* When emitting push, take care for source operands on the stack. */
21692 {
21695 /* Compensate for the stack decrement by 4. */
21700 /* src_base refers to the stack pointer and is
21701 automatically decreased by emitted push. */
21705 }
21707 /* We need to do copy in the right order in case an address register
21708 of the source overlaps the destination. */
21710 {
21711 rtx tmp;
21714 {
21718 collisions++;
21719 }
21721 /* Collision in the middle part can be handled by reordering. */
21723 {
21726 }
21730 {
21732 {
21735 }
21736 else
21737 {
21740 }
21741 }
21743 /* If there are more collisions, we can't handle it by reordering.
21744 Do an lea to the last part and use only one colliding move. */
21746 {
21747 rtx base;
21753 /* Handle the case when the last part isn't valid for lea.
21754 Happens in 64-bit mode storing the 12-byte XFmode. */
21761 {
21764 }
21765 }
21766 }
21769 {
21771 {
21773 {
21778 }
21780 {
21783 }
21784 }
21785 else
21786 {
21787 /* In 64bit mode we don't have 32bit push available. In case this is
21788 register, it is OK - we will just use larger counterpart. We also
21789 retype memory - these comes from attempt to avoid REX prefix on
21790 moving of second half of TFmode value. */
21792 {
21794 {
21805 }
21809 }
21810 }
21814 }
21816 /* Choose correct order to not overwrite the source before it is copied. */
21826 {
21828 {
21831 }
21832 }
21833 else
21834 {
21836 {
21839 }
21840 }
21842 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
21844 {
21853 }
21859 }
21861 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
21862 left shift by a constant, either using a single shift or
21863 a sequence of add instructions. */
21865 static void
21867 {
21873 {
21877 }
21878 else
21879 {
21882 }
21883 }
21885 void
21887 {
21896 {
21901 {
21907 }
21908 else
21909 {
21917 }
21919 }
21926 {
21927 /* Assuming we've chosen a QImode capable registers, then 1 << N
21928 can be done with two 32/64-bit shifts, no branches, no cmoves. */
21930 {
21946 }
21948 /* Otherwise, we can get the same results by manually performing
21949 a bit extract operation on bit 5/6, and then performing the two
21950 shifts. The two methods of getting 0/1 into low/high are exactly
21951 the same size. Avoiding the shift in the bit extract case helps
21952 pentium4 a bit; no one else seems to care much either way. */
21953 else
21954 {
21959 HOST_WIDE_INT bits;
21960 rtx x;
21963 {
21969 }
21970 else
21971 {
21977 }
21981 else
21989 }
21994 }
21997 {
21998 /* For -1 << N, we can avoid the shld instruction, because we
21999 know that we're shifting 0...31/63 ones into a -1. */
22003 else
22005 }
22006 else
22007 {
22015 }
22020 {
22026 }
22027 else
22028 {
22033 }
22034 }
22036 void
22038 {
22048 {
22053 {
22059 }
22061 {
22070 }
22071 else
22072 {
22080 }
22081 }
22082 else
22083 {
22095 {
22103 scratch));
22104 }
22105 else
22106 {
22111 }
22112 }
22113 }
22115 void
22117 {
22127 {
22132 {
22139 }
22140 else
22141 {
22149 }
22150 }
22151 else
22152 {
22164 {
22170 scratch));
22171 }
22172 else
22173 {
22178 }
22179 }
22180 }
22182 /* Predict just emitted jump instruction to be taken with probability PROB. */
22183 static void
22185 {
22189 }
22191 /* Helper function for the string operations below. Dest VARIABLE whether
22192 it is aligned to VALUE bytes. If true, jump to the label. */
22193 static rtx
22195 {
22200 else
22206 else
22209 }
22211 /* Adjust COUNTER by the VALUE. */
22212 static void
22214 {
22219 }
22221 /* Zero extend possibly SImode EXP to Pmode register. */
22222 rtx
22224 {
22226 }
22228 /* Divide COUNTREG by SCALE. */
22229 static rtx
22231 {
22232 rtx sc;
22244 }
22246 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
22247 DImode for constant loop counts. */
22249 static enum machine_mode
22251 {
22259 }
22261 /* When SRCPTR is non-NULL, output simple loop to move memory
22262 pointer to SRCPTR to DESTPTR via chunks of MODE unrolled UNROLL times,
22263 overall size is COUNT specified in bytes. When SRCPTR is NULL, output the
22264 equivalent loop to set memory by VALUE (supposed to be in MODE).
22266 The size is rounded down to whole number of chunk size moved at once.
22267 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
22270 static void
22275 {
22281 rtx size;
22290 /* Those two should combine. */
22292 {
22296 }
22303 /* This assert could be relaxed - in this case we'll need to compute
22304 smallest power of two, containing in PIECE_SIZE_N and pass it to
22305 offset_address. */
22311 {
22315 /* When unrolling for chips that reorder memory reads and writes,
22316 we can save registers by using single temporary.
22317 Also using 4 temporaries is overkill in 32bit mode. */
22319 {
22321 {
22323 {
22324 destmem =
22326 srcmem =
22328 }
22330 }
22331 }
22332 else
22333 {
22337 {
22340 {
22341 srcmem =
22343 }
22345 }
22347 {
22349 {
22350 destmem =
22352 }
22354 }
22355 }
22356 }
22357 else
22359 {
22361 destmem =
22364 }
22374 {
22380 else
22382 }
22383 else
22391 {
22396 }
22398 }
22400 /* Output "rep; mov" instruction.
22401 Arguments have same meaning as for previous function */
22402 static void
22405 rtx count,
22407 {
22408 rtx destexp;
22409 rtx srcexp;
22410 rtx countreg;
22411 HOST_WIDE_INT rounded_count;
22413 /* If the size is known, it is shorter to use rep movs. */
22424 {
22431 }
22432 else
22433 {
22436 }
22438 {
22445 }
22446 else
22447 {
22452 }
22455 }
22457 /* Output "rep; stos" instruction.
22458 Arguments have same meaning as for previous function */
22459 static void
22462 rtx orig_value)
22463 {
22464 rtx destexp;
22465 rtx countreg;
22466 HOST_WIDE_INT rounded_count;
22473 {
22477 }
22478 else
22481 {
22486 }
22490 }
22492 /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
22493 DESTMEM.
22494 SRC is passed by pointer to be updated on return.
22495 Return value is updated DST. */
22496 static rtx
22498 HOST_WIDE_INT size_to_move)
22499 {
22505 /* Find the widest mode in which we could perform moves.
22506 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
22507 it until move of such size is supported. */
22512 {
22516 }
22518 /* Find the corresponding vector mode with the same size as MOVE_MODE.
22519 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
22521 {
22526 {
22530 }
22531 }
22537 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
22541 {
22542 /* We move from memory to memory, so we'll need to do it via
22543 a temporary register. */
22554 piece_size);
22556 piece_size);
22557 }
22559 /* Update DST and SRC rtx. */
22562 }
22564 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
22565 static void
22568 {
22571 {
22576 /* For now MAX_SIZE should be a power of 2. This assert could be
22577 relaxed, but it'll require a bit more complicated epilogue
22578 expanding. */
22581 {
22584 }
22586 }
22588 {
22594 }
22596 /* When there are stringops, we can cheaply increase dest and src pointers.
22597 Otherwise we save code size by maintaining offset (zero is readily
22598 available from preceding rep operation) and using x86 addressing modes.
22599 */
22601 {
22603 {
22610 }
22612 {
22619 }
22621 {
22628 }
22629 }
22630 else
22631 {
22633 rtx tmp;
22636 {
22647 }
22649 {
22662 }
22664 {
22673 }
22674 }
22675 }
22677 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
22678 static void
22681 {
22682 count =
22688 }
22690 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
22691 static void
22693 {
22694 rtx dest;
22697 {
22702 {
22704 {
22709 }
22710 else
22713 }
22715 {
22717 {
22720 }
22721 else
22722 {
22727 }
22729 }
22731 {
22735 }
22737 {
22741 }
22743 {
22747 }
22749 }
22751 {
22754 }
22756 {
22759 {
22763 }
22764 else
22765 {
22771 }
22774 }
22776 {
22779 {
22782 }
22783 else
22784 {
22788 }
22791 }
22793 {
22799 }
22801 {
22807 }
22809 {
22815 }
22816 }
22818 /* Copy enough from DEST to SRC to align DEST known to by aligned by ALIGN to
22819 DESIRED_ALIGNMENT.
22820 Return value is updated DESTMEM. */
22821 static rtx
22825 {
22828 {
22830 {
22837 }
22838 }
22840 }
22842 /* Copy enough from DST to SRC to align DST known to DESIRED_ALIGN.
22843 ALIGN_BYTES is how many bytes need to be copied.
22844 The function updates DST and SRC, namely, it sets proper alignment.
22845 DST is returned via return value, SRC is updated via pointer SRCP. */
22846 static rtx
22849 {
22862 {
22864 {
22867 }
22868 }
22873 {
22876 {
22880 }
22885 }
22892 }
22894 /* Set enough from DEST to align DEST known to by aligned by ALIGN to
22895 DESIRED_ALIGNMENT. */
22896 static void
22899 {
22901 {
22908 }
22910 {
22917 }
22919 {
22926 }
22928 }
22930 /* Set enough from DST to align DST known to by aligned by ALIGN to
22931 DESIRED_ALIGN. ALIGN_BYTES is how many bytes need to be stored. */
22932 static rtx
22935 {
22939 {
22944 }
22946 {
22953 }
22955 {
22962 }
22969 }
22971 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
22972 static enum stringop_alg
22975 {
22978 /* Algorithms using the rep prefix want at least edi and ecx;
22979 additionally, memset wants eax and memcpy wants esi. Don't
22980 consider such algorithms if the user has appropriated those
22981 registers for their own purposes. */
22983 || (memset
22987 #define ALG_USABLE_P(alg) (rep_prefix_usable \
22988 || (alg != rep_prefix_1_byte \
22989 && alg != rep_prefix_4_byte \
22990 && alg != rep_prefix_8_byte))
22993 /* Even if the string operation call is cold, we still might spend a lot
22994 of time processing large blocks. */
22999 else
23007 else
23011 /* rep; movq or rep; movl is the smallest variant. */
23013 {
23016 else
23018 }
23019 /* Very tiny blocks are best handled via the loop, REP is expensive to setup.
23020 */
23024 {
23028 {
23029 /* We get here if the algorithms that were not libcall-based
23030 were rep-prefix based and we are unable to use rep prefixes
23031 based on global register usage. Break out of the loop and
23032 use the heuristic below. */
23036 {
23041 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
23042 last non-libcall inline algorithm. */
23044 {
23045 /* When the current size is best to be copied by a libcall,
23046 but we are still forced to inline, run the heuristic below
23047 that will pick code for medium sized blocks. */
23051 }
23053 {
23056 }
23057 }
23058 }
23060 }
23061 /* When asked to inline the call anyway, try to pick meaningful choice.
23062 We look for maximal size of block that is faster to copy by hand and
23063 take blocks of at most of that size guessing that average size will
23064 be roughly half of the block.
23066 If this turns out to be bad, we might simply specify the preferred
23067 choice in ix86_costs. */
23070 {
23077 {
23084 }
23085 /* If there aren't any usable algorithms, then recursing on
23086 smaller sizes isn't going to find anything. Just return the
23087 simple byte-at-a-time copy loop. */
23089 {
23090 /* Pick something reasonable. */
23094 }
23103 }
23105 #undef ALG_USABLE_P
23106 }
23108 /* Decide on alignment. We know that the operand is already aligned to ALIGN
23109 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
23110 static int
23115 {
23126 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
23127 copying whole cacheline at once. */
23140 }
23142 /* Expand string move (memcpy) operation. Use i386 string operations
23143 when profitable. expand_setmem contains similar code. The code
23144 depends upon architecture, block size and alignment, but always has
23145 the same overall structure:
23147 1) Prologue guard: Conditional that jumps up to epilogues for small
23148 blocks that can be handled by epilogue alone. This is faster
23149 but also needed for correctness, since prologue assume the block
23150 is larger than the desired alignment.
23152 Optional dynamic check for size and libcall for large
23153 blocks is emitted here too, with -minline-stringops-dynamically.
23155 2) Prologue: copy first few bytes in order to get destination
23156 aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
23157 than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
23158 copied. We emit either a jump tree on power of two sized
23159 blocks, or a byte loop.
23161 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
23162 with specified algorithm.
23164 4) Epilogue: code copying tail of the block that is too small to be
23165 handled by main body (or up to size guarded by prologue guard). */
23167 bool
23170 {
23171 rtx destreg;
23172 rtx srcreg;
23174 rtx tmp;
23190 /* i386 can do misaligned access on reasonably increased cost. */
23194 /* ALIGN is the minimum of destination and source alignment, but we care here
23195 just about destination alignment. */
23204 /* Make sure we don't need to care about overflow later on. */
23208 /* Step 0: Decide on preferred algorithm, desired alignment and
23209 size of chunks to be copied by main loop. */
23223 {
23242 /* Find the widest supported mode. */
23248 /* Find the corresponding vector mode with the same size as MOVE_MODE.
23249 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
23251 {
23256 }
23268 }
23276 /* Step 1: Prologue guard. */
23278 /* Alignment code needs count to be in register. */
23280 {
23283 {
23284 align_bytes
23288 else
23290 }
23293 }
23296 /* Ensure that alignment prologue won't copy past end of block. */
23298 {
23300 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
23301 Make sure it is power of 2. */
23305 {
23307 {
23308 /* If main algorithm works on QImode, no epilogue is needed.
23309 For small sizes just don't align anything. */
23312 else
23314 }
23315 }
23316 else
23317 {
23324 else
23326 }
23327 }
23329 /* Emit code to decide on runtime whether library call or inline should be
23330 used. */
23332 {
23334 {
23336 {
23340 }
23341 }
23342 else
23343 {
23352 }
23353 }
23355 /* Step 2: Alignment prologue. */
23358 {
23360 {
23361 /* Except for the first move in epilogue, we no longer know
23362 constant offset in aliasing info. It don't seems to worth
23363 the pain to maintain it for the first move, so throw away
23364 the info early. */
23368 desired_align);
23369 }
23370 else
23371 {
23372 /* If we know how many bytes need to be stored before dst is
23373 sufficiently aligned, maintain aliasing info accurately. */
23379 }
23385 {
23386 /* It is possible that we copied enough so the main loop will not
23387 execute. */
23397 else
23399 }
23400 }
23402 {
23407 }
23411 /* Step 3: Main loop. */
23414 {
23425 expected_size);
23431 move_mode);
23433 }
23434 /* Adjust properly the offset of src and dest memory for aliasing. */
23436 {
23441 }
23442 else
23443 {
23446 }
23448 /* Step 4: Epilogue to copy the remaining bytes. */
23449 epilogue:
23451 {
23452 /* When the main loop is done, COUNT_EXP might hold original count,
23453 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
23454 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
23455 bytes. Compensate if needed. */
23458 {
23459 tmp =
23462 OPTAB_DIRECT);
23465 }
23468 }
23472 size_needed);
23476 }
23478 /* Helper function for memcpy. For QImode value 0xXY produce
23479 0xXYXYXYXY of wide specified by MODE. This is essentially
23480 a * 0x10101010, but we can do slightly better than
23481 synth_mult by unwinding the sequence by hand on CPUs with
23482 slow multiply. */
23483 static rtx
23485 {
23487 rtx tmp;
23494 {
23502 }
23511 nops--;
23516 {
23520 OPTAB_DIRECT);
23521 }
23522 else
23523 {
23529 else
23531 else
23532 {
23535 reg =
23537 }
23547 }
23548 }
23550 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
23551 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
23552 alignment from ALIGN to DESIRED_ALIGN. */
23553 static rtx
23555 {
23556 rtx promoted_val;
23565 else
23569 }
23571 /* Expand string clear operation (bzero). Use i386 string operations when
23572 profitable. See expand_movmem comment for explanation of individual
23573 steps performed. */
23574 bool
23577 {
23578 rtx destreg;
23580 rtx tmp;
23598 /* i386 can do misaligned access on reasonably increased cost. */
23607 /* Make sure we don't need to care about overflow later on. */
23611 /* Step 0: Decide on preferred algorithm, desired alignment and
23612 size of chunks to be copied by main loop. */
23626 {
23652 }
23660 /* Step 1: Prologue guard. */
23662 /* Alignment code needs count to be in register. */
23664 {
23667 {
23668 align_bytes
23672 else
23674 }
23676 {
23681 }
23682 }
23683 /* Do the cheap promotion to allow better CSE across the
23684 main loop and epilogue (ie one load of the big constant in the
23685 front of all code. */
23689 /* Ensure that alignment prologue won't copy past end of block. */
23691 {
23693 /* Epilogue always copies COUNT_EXP & (EPILOGUE_SIZE_NEEDED - 1) bytes.
23694 Make sure it is power of 2. */
23697 /* To improve performance of small blocks, we jump around the VAL
23698 promoting mode. This mean that if the promoted VAL is not constant,
23699 we might not use it in the epilogue and have to use byte
23700 loop variant. */
23704 {
23706 {
23707 /* If main algorithm works on QImode, no epilogue is needed.
23708 For small sizes just don't align anything. */
23711 else
23713 }
23714 }
23715 else
23716 {
23723 else
23725 }
23726 }
23728 {
23737 }
23739 /* Step 2: Alignment prologue. */
23741 /* Do the expensive promotion once we branched off the small blocks. */
23748 {
23750 {
23751 /* Except for the first move in epilogue, we no longer know
23752 constant offset in aliasing info. It don't seems to worth
23753 the pain to maintain it for the first move, so throw away
23754 the info early. */
23757 desired_align);
23758 }
23759 else
23760 {
23761 /* If we know how many bytes need to be stored before dst is
23762 sufficiently aligned, maintain aliasing info accurately. */
23768 }
23774 {
23775 /* It is possible that we copied enough so the main loop will not
23776 execute. */
23786 else
23788 }
23789 }
23791 {
23797 }
23801 /* Step 3: Main loop. */
23804 {
23815 expected_size);
23829 }
23830 /* Adjust properly the offset of src and dest memory for aliasing. */
23834 else
23837 /* Step 4: Epilogue to copy the remaining bytes. */
23840 {
23841 /* When the main loop is done, COUNT_EXP might hold original count,
23842 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
23843 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
23844 bytes. Compensate if needed. */
23847 {
23848 tmp =
23851 OPTAB_DIRECT);
23854 }
23857 }
23858 epilogue:
23860 {
23863 epilogue_size_needed);
23864 else
23866 epilogue_size_needed);
23867 }
23871 }
23873 /* Expand the appropriate insns for doing strlen if not just doing
23874 repnz; scasb
23876 out = result, initialized with the start address
23877 align_rtx = alignment of the address.
23878 scratch = scratch register, initialized with the startaddress when
23879 not aligned, otherwise undefined
23881 This is just the body. It needs the initializations mentioned above and
23882 some address computing at the end. These things are done in i386.md. */
23884 static void
23886 {
23888 rtx tmp;
23893 rtx mem;
23896 rtx cmp;
23902 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
23904 /* Is there a known alignment and is it less than 4? */
23906 {
23909 /* Is there a known alignment and is it not 2? */
23911 {
23915 /* Leave just the 3 lower bits. */
23925 }
23926 else
23927 {
23928 /* Since the alignment is 2, we have to check 2 or 0 bytes;
23929 check if is aligned to 4 - byte. */
23936 }
23940 /* Now compare the bytes. */
23942 /* Compare the first n unaligned byte on a byte per byte basis. */
23946 /* Increment the address. */
23949 /* Not needed with an alignment of 2 */
23951 {
23955 end_0_label);
23960 }
23963 end_0_label);
23966 }
23968 /* Generate loop to check 4 bytes at a time. It is not a good idea to
23969 align this loop. It gives only huge programs, but does not help to
23970 speed up. */
23977 /* This formula yields a nonzero result iff one of the bytes is zero.
23978 This saves three branches inside loop and many cycles. */
23986 align_4_label);
23989 {
23995 /* If zero is not in the first two bytes, move two bytes forward. */
24001 reg,
24002 tmpreg)));
24003 /* Emit lea manually to avoid clobbering of flags. */
24011 reg2,
24012 out)));
24013 }
24014 else
24015 {
24017 /* Is zero in the first two bytes? */
24024 pc_rtx);
24028 /* Not in the first two. Move two bytes forward. */
24034 }
24036 /* Avoid branch in fixing the byte. */
24044 }
24046 /* Expand strlen. */
24048 bool
24050 {
24053 /* The generic case of strlen expander is long. Avoid it's
24054 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
24057 && !TARGET_INLINE_ALL_STRINGOPS
24067 {
24068 /* Well it seems that some optimizer does not combine a call like
24069 foo(strlen(bar), strlen(bar));
24070 when the move and the subtraction is done here. It does calculate
24071 the length just once when these instructions are done inside of
24072 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
24073 often used and I use one fewer register for the lifetime of
24074 output_strlen_unroll() this is better. */
24080 /* strlensi_unroll_1 returns the address of the zero at the end of
24081 the string, like memchr(), so compute the length by subtracting
24082 the start address. */
24084 }
24085 else
24086 {
24087 rtx unspec;
24089 /* Can't use this if the user has appropriated eax, ecx, or edi. */
24102 /* If .md starts supporting :P, this can be done in .md. */
24108 }
24110 }
24112 /* For given symbol (function) construct code to compute address of it's PLT
24113 entry in large x86-64 PIC model. */
24114 static rtx
24116 {
24129 }
24131 rtx
24133 rtx callarg2,
24135 {
24136 unsigned int const cregs_size
24147 {
24148 #if TARGET_MACHO
24151 #endif
24152 }
24153 else
24154 {
24155 /* Static functions and indirect calls don't need the pic register. */
24157 && (!TARGET_64BIT
24163 }
24166 {
24170 }
24173 && !TARGET_PECOFF
24181 {
24184 }
24192 {
24196 }
24200 {
24204 UNSPEC_MS_TO_SYSV_CALL);
24207 {
24213 }
24214 }
24223 }
24225 /* Output the assembly for a call instruction. */
24229 {
24235 {
24238 /* SEH epilogue detection requires the indirect branch case
24239 to include REX.W. */
24242 else
24247 }
24249 /* SEH unwinding can require an extra nop to be emitted in several
24250 circumstances. Determine if we have one of those. */
24252 {
24253 rtx i;
24256 {
24257 /* If we get to another real insn, we don't need the nop. */
24261 /* If we get to the epilogue note, prevent a catch region from
24262 being adjacent to the standard epilogue sequence. If non-
24263 call-exceptions, we'll have done this during epilogue emission. */
24265 && !flag_non_call_exceptions
24267 {
24270 }
24271 }
24273 /* If we didn't find a real insn following the call, prevent the
24274 unwinder from looking into the next function. */
24277 }
24281 else
24290 }
24291 \f
24292 /* Clear stack slot assignments remembered from previous functions.
24293 This is called from INIT_EXPANDERS once before RTL is emitted for each
24294 function. */
24298 {
24306 }
24308 /* Return a MEM corresponding to a stack slot with mode MODE.
24309 Allocate a new slot if necessary.
24311 The RTL for a function can have several slots available: N is
24312 which slot to use. */
24314 rtx
24316 {
24333 }
24335 static void
24337 {
24343 }
24344 \f
24345 /* Calculate the length of the memory address in the instruction encoding.
24346 Includes addr32 prefix, does not include the one-byte modrm, opcode,
24347 or other prefixes. We never generate addr32 prefix for LEA insn. */
24349 int
24351 {
24368 /* If this is not LEA instruction, add the length of addr32 prefix. */
24373 len++;
24387 /* Rule of thumb:
24388 - esp as the base always wants an index,
24389 - ebp as the base always wants a displacement,
24390 - r12 as the base always wants an index,
24391 - r13 as the base always wants a displacement. */
24393 /* Register Indirect. */
24395 {
24396 /* esp (for its index) and ebp (for its displacement) need
24397 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
24398 code. */
24405 len++;
24406 }
24408 /* Direct Addressing. In 64-bit mode mod 00 r/m 5
24409 is not disp32, but disp32(%rip), so for disp32
24410 SIB byte is needed, unless print_operand_address
24411 optimizes it into disp32(%rip) or (%rip) is implied
24412 by UNSPEC. */
24414 {
24417 {
24433 len++;
24434 }
24435 }
24436 else
24437 {
24438 /* Find the length of the displacement constant. */
24440 {
24443 else
24445 }
24446 /* ebp always wants a displacement. Similarly r13. */
24448 len++;
24450 /* An index requires the two-byte modrm form.... */
24452 /* ...like esp (or r12), which always wants an index. */
24456 len++;
24457 }
24460 }
24462 /* Compute default value for "length_immediate" attribute. When SHORTFORM
24463 is set, expect that insn have 8bit immediate alternative. */
24464 int
24466 {
24472 {
24477 {
24480 {
24492 }
24494 {
24497 }
24498 }
24500 {
24510 /* Immediates for DImode instructions are encoded
24511 as 32bit sign extended values. */
24517 }
24518 }
24520 }
24522 /* Compute default value for "length_address" attribute. */
24523 int
24525 {
24529 {
24540 }
24545 {
24548 {
24553 constraints++;
24556 ;
24557 /* Skip ignored operands. */
24560 }
24562 }
24564 }
24566 /* Compute default value for "length_vex" attribute. It includes
24567 2 or 3 byte VEX prefix and 1 opcode byte. */
24569 int
24571 {
24574 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
24575 byte VEX prefix. */
24579 /* We can always use 2 byte VEX prefix in 32bit. */
24587 {
24588 /* REX.W bit uses 3 byte VEX prefix. */
24592 }
24593 else
24594 {
24595 /* REX.X or REX.B bits use 3 byte VEX prefix. */
24599 }
24602 }
24603 \f
24604 /* Return the maximum number of instructions a cpu can issue. */
24606 static int
24608 {
24610 {
24637 }
24638 }
24640 /* A subroutine of ix86_adjust_cost -- return TRUE iff INSN reads flags set
24641 by DEP_INSN and nothing set by DEP_INSN. */
24643 static bool
24645 {
24648 /* Simplify the test for uninteresting insns. */
24656 {
24659 }
24664 {
24667 }
24668 else
24674 /* This test is true if the dependent insn reads the flags but
24675 not any other potentially set register. */
24683 }
24685 /* Return true iff USE_INSN has a memory address with operands set by
24686 SET_INSN. */
24688 bool
24690 {
24695 {
24698 }
24700 }
24702 /* Helper function for exact_store_load_dependency.
24703 Return true if addr is found in insn. */
24704 static bool
24706 {
24713 {
24719 CASE_CONST_ANY:
24728 }
24732 {
24734 {
24744 }
24745 }
24747 }
24749 /* Return true if there exists exact dependency for store & load, i.e.
24750 the same memory address is used in them. */
24751 static bool
24753 {
24767 }
24769 static int
24771 {
24777 /* Anti and output dependencies have zero cost on all CPUs. */
24783 /* If we can't recognize the insns, we can't really do anything. */
24791 {
24793 /* Address Generation Interlock adds a cycle of latency. */
24795 {
24806 }
24810 /* ??? Compares pair with jump/setcc. */
24814 /* Floating point stores require value to be ready one cycle earlier. */
24824 /* INT->FP conversion is expensive. */
24828 /* There is one cycle extra latency between an FP op and a store. */
24836 /* Show ability of reorder buffer to hide latency of load by executing
24837 in parallel with previous instruction in case
24838 previous instruction is not needed to compute the address. */
24841 {
24842 /* Claim moves to take one cycle, as core can issue one load
24843 at time and the next load can start cycle later. */
24848 cost--;
24849 }
24855 /* The esp dependency is resolved before the instruction is really
24856 finished. */
24861 /* INT->FP conversion is expensive. */
24865 /* Show ability of reorder buffer to hide latency of load by executing
24866 in parallel with previous instruction in case
24867 previous instruction is not needed to compute the address. */
24870 {
24871 /* Claim moves to take one cycle, as core can issue one load
24872 at time and the next load can start cycle later. */
24878 else
24880 }
24896 /* Show ability of reorder buffer to hide latency of load by executing
24897 in parallel with previous instruction in case
24898 previous instruction is not needed to compute the address. */
24901 {
24905 /* Because of the difference between the length of integer and
24906 floating unit pipeline preparation stages, the memory operands
24907 for floating point are cheaper.
24909 ??? For Athlon it the difference is most probably 2. */
24912 else
24917 else
24919 }
24926 /* Increase cost of integer loads. */
24929 {
24932 {
24935 else
24936 {
24937 /* Increase cost of ld/st for short int types only
24938 because of store forwarding issue. */
24942 {
24943 /* Increase cost of store/load insn if exact
24944 dependence exists and it is load insn. */
24949 }
24950 }
24951 }
24952 }
24956 }
24959 }
24961 /* How many alternative schedules to try. This should be as wide as the
24962 scheduling freedom in the DFA, but no wider. Making this value too
24963 large results extra work for the scheduler. */
24965 static int
24967 {
24969 {
24982 /* Generally, we want haifa-sched:max_issue() to look ahead as far
24983 as many instructions can be executed on a cycle, i.e.,
24984 issue_rate. I wonder why tuning for many CPUs does not do this. */
24987 /* Don't use lookahead for pre-reload schedule to save compile time. */
24992 }
24993 }
24995 /* Try to reorder ready list to take advantage of Atom pipelined IMUL
24996 execution. It is applied if
24997 (1) IMUL instruction is on the top of list;
24998 (2) There exists the only producer of independent IMUL instruction in
24999 ready list.
25000 Return index of IMUL producer if it was found and -1 otherwise. */
25001 static int
25003 {
25005 sd_iterator_def sd_it;
25006 dep_t dep;
25013 /* Check that IMUL instruction is on the top of ready list. */
25022 /* Search for producer of independent IMUL instruction. */
25024 {
25028 /* Skip IMUL instruction. */
25038 {
25039 rtx con;
25050 {
25051 sd_iterator_def sd_it1;
25052 dep_t dep1;
25053 /* Check if there is no other dependee for IMUL. */
25056 {
25057 rtx pro;
25063 }
25066 }
25067 }
25070 }
25072 }
25074 /* Try to find the best candidate on the top of ready list if two insns
25075 have the same priority - candidate is best if its dependees were
25076 scheduled earlier. Applied for Silvermont only.
25077 Return true if top 2 insns must be interchanged. */
25078 static bool
25080 {
25083 rtx set;
25084 sd_iterator_def sd_it;
25085 dep_t dep;
25088 #define INSN_TICK(INSN) (HID (INSN)->tick)
25109 {
25112 /* Determine winner more precise. */
25114 {
25115 rtx pro;
25121 }
25123 {
25124 rtx pro;
25130 }
25133 {
25134 /* Determine winner - load must win. */
25140 }
25142 }
25144 #undef INSN_TICK
25145 }
25147 /* Perform possible reodering of ready list for Atom/Silvermont only.
25148 Return issue rate. */
25149 static int
25152 {
25156 rtx insn;
25159 /* Set up issue rate. */
25162 /* Do reodering for Atom/SLM only. */
25166 /* Nothing to do if ready list contains only 1 instruction. */
25170 /* Do reodering for post-reload scheduler only. */
25175 {
25180 /* Put IMUL producer (ready[index]) at the top of ready list. */
25186 }
25188 {
25192 /* Swap 2 top elements of ready list. */
25196 }
25198 }
25200 static bool
25203 /* Returns true if lhs of insn is HW function argument register and set up
25204 is_spilled to true if it is likely spilled HW register. */
25205 static bool
25207 {
25208 rtx dst;
25212 /* Call instructions are not movable, ignore it. */
25223 {
25224 /* Is it likely spilled HW register? */
25229 }
25231 }
25233 /* Add output dependencies for chain of function adjacent arguments if only
25234 there is a move to likely spilled HW register. Return first argument
25235 if at least one dependence was added or NULL otherwise. */
25236 static rtx
25238 {
25239 rtx insn;
25246 /* Find nearest to call argument passing instruction. */
25248 {
25257 }
25261 {
25268 {
25271 }
25273 {
25274 /* Add output depdendence between two function arguments if chain
25275 of output arguments contains likely spilled HW registers. */
25279 }
25280 else
25282 }
25286 }
25288 /* Add output or anti dependency from insn to first_arg to restrict its code
25289 motion. */
25290 static void
25292 {
25293 rtx set;
25294 rtx tmp;
25301 {
25302 /* Add output dependency to the first function argument. */
25305 }
25306 /* Add anti dependency. */
25308 }
25310 /* Avoid cross block motion of function argument through adding dependency
25311 from the first non-jump instruction in bb. */
25312 static void
25314 {
25318 {
25320 {
25323 {
25326 }
25327 }
25331 }
25332 }
25334 /* Hook for pre-reload schedule - avoid motion of function arguments
25335 passed in likely spilled HW registers. */
25336 static void
25338 {
25339 rtx insn;
25347 {
25350 {
25351 /* Add dependee for first argument to predecessors if only
25352 region contains more than one block. */
25356 /* Skip trivial regions and region head blocks that can have
25357 predecessors outside of region. */
25359 {
25360 edge e;
25361 edge_iterator ei;
25362 /* Assume that region is SCC, i.e. all immediate predecessors
25363 of non-head block are in the same region. */
25365 {
25366 /* Avoid creating of loop-carried dependencies through
25367 using topological odering in region. */
25370 }
25371 }
25375 }
25376 }
25379 }
25381 /* Hook for pre-reload schedule - set priority of moves from likely spilled
25382 HW registers to maximum, to schedule them at soon as possible. These are
25383 moves from function argument registers at the top of the function entry
25384 and moves from function return value registers after call. */
25385 static int
25387 {
25388 rtx set;
25398 {
25405 }
25408 }
25410 /* Model decoder of Core 2/i7.
25411 Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
25412 track the instruction fetch block boundaries and make sure that long
25413 (9+ bytes) instructions are assigned to D0. */
25415 /* Maximum length of an insn that can be handled by
25416 a secondary decoder unit. '8' for Core 2/i7. */
25419 /* Ifetch block size, i.e., number of bytes decoder reads per cycle.
25420 '16' for Core 2/i7. */
25423 /* Maximum number of instructions decoder can handle per cycle.
25424 '6' for Core 2/i7. */
25428 ix86_first_cycle_multipass_data_t;
25430 const_ix86_first_cycle_multipass_data_t;
25432 /* A variable to store target state across calls to max_issue within
25433 one cycle. */
25437 /* Initialize DATA. */
25438 static void
25440 {
25441 ix86_first_cycle_multipass_data_t data
25448 }
25450 /* Advancing the cycle; reset ifetch block counts. */
25451 static void
25453 {
25460 }
25464 /* Filter out insns from ready_try that the core will not be able to issue
25465 on current cycle due to decoder. */
25466 static void
25467 core2i7_first_cycle_multipass_filter_ready_try
25470 {
25472 {
25473 rtx insn;
25483 (!first_cycle_insn_p
25485 /* ... or it would not fit into the ifetch block ... */
25487 /* ... or the decoder is full already ... */
25489 /* ... mask the insn out. */
25490 {
25495 }
25496 }
25497 }
25499 /* Prepare for a new round of multipass lookahead scheduling. */
25500 static void
25503 {
25504 ix86_first_cycle_multipass_data_t data
25506 const_ix86_first_cycle_multipass_data_t prev_data
25509 /* Restore the state from the end of the previous round. */
25513 /* Filter instructions that cannot be issued on current cycle due to
25514 decoder restrictions. */
25516 first_cycle_insn_p);
25517 }
25519 /* INSN is being issued in current solution. Account for its impact on
25520 the decoder model. */
25521 static void
25524 {
25525 ix86_first_cycle_multipass_data_t data
25527 const_ix86_first_cycle_multipass_data_t prev_data
25537 /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
25539 {
25542 }
25544 {
25548 }
25551 /* Filter out insns from ready_try that the core will not be able to issue
25552 on current cycle due to decoder. */
25555 }
25557 /* Revert the effect on ready_try. */
25558 static void
25562 {
25563 const_ix86_first_cycle_multipass_data_t data
25566 sbitmap_iterator sbi;
25570 {
25572 }
25573 }
25575 /* Save the result of multipass lookahead scheduling for the next round. */
25576 static void
25578 {
25579 const_ix86_first_cycle_multipass_data_t data
25581 ix86_first_cycle_multipass_data_t next_data
25585 {
25588 }
25589 }
25591 /* Deallocate target data. */
25592 static void
25594 {
25595 ix86_first_cycle_multipass_data_t data
25599 {
25603 }
25604 }
25606 /* Prepare for scheduling pass. */
25607 static void
25611 {
25612 /* Install scheduling hooks for current CPU. Some of these hooks are used
25613 in time-critical parts of the scheduler, so we only set them up when
25614 they are actually used. */
25616 {
25620 /* Do not perform multipass scheduling for pre-reload schedule
25621 to save compile time. */
25623 {
25639 /* Set decoder parameters. */
25644 }
25645 /* ... Fall through ... */
25655 }
25656 }
25658 \f
25659 /* Compute the alignment given to a constant that is being placed in memory.
25660 EXP is the constant and ALIGN is the alignment that the object would
25661 ordinarily have.
25662 The value of this function is used instead of that alignment to align
25663 the object. */
25665 int
25667 {
25670 {
25675 }
25681 }
25683 /* Compute the alignment for a static variable.
25684 TYPE is the data type, and ALIGN is the alignment that
25685 the object would ordinarily have. The value of this function is used
25686 instead of that alignment to align the object. */
25688 int
25690 {
25702 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
25703 to 16byte boundary. */
25705 {
25712 }
25718 {
25723 }
25725 {
25732 }
25737 {
25742 }
25745 {
25750 }
25753 }
25755 /* Compute the alignment for a local variable or a stack slot. EXP is
25756 the data type or decl itself, MODE is the widest mode available and
25757 ALIGN is the alignment that the object would ordinarily have. The
25758 value of this macro is used instead of that alignment to align the
25759 object. */
25761 unsigned int
25764 {
25768 {
25771 }
25772 else
25773 {
25776 }
25778 /* Don't do dynamic stack realignment for long long objects with
25779 -mpreferred-stack-boundary=2. */
25788 /* If TYPE is NULL, we are allocating a stack slot for caller-save
25789 register in MODE. We will return the largest alignment of XF
25790 and DF. */
25792 {
25796 }
25798 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
25799 to 16byte boundary. Exact wording is:
25801 An array uses the same alignment as its elements, except that a local or
25802 global array variable of length at least 16 bytes or
25803 a C99 variable-length array variable always has alignment of at least 16 bytes.
25805 This was added to allow use of aligned SSE instructions at arrays. This
25806 rule is meant for static storage (where compiler can not do the analysis
25807 by itself). We follow it for automatic variables only when convenient.
25808 We fully control everything in the function compiled and functions from
25809 other unit can not rely on the alignment.
25811 Exclude va_list type. It is the common case of local array where
25812 we can not benefit from the alignment.
25814 TODO: Probably one should optimize for size only when var is not escaping. */
25817 {
25827 }
25829 {
25834 }
25836 {
25842 }
25847 {
25852 }
25855 {
25861 }
25863 }
25865 /* Compute the minimum required alignment for dynamic stack realignment
25866 purposes for a local variable, parameter or a stack slot. EXP is
25867 the data type or decl itself, MODE is its mode and ALIGN is the
25868 alignment that the object would ordinarily have. */
25870 unsigned int
25873 {
25877 {
25880 }
25881 else
25882 {
25885 }
25890 /* Don't do dynamic stack realignment for long long objects with
25891 -mpreferred-stack-boundary=2. */
25898 }
25899 \f
25900 /* Find a location for the static chain incoming to a nested function.
25901 This is a register, unless all free registers are used by arguments. */
25903 static rtx
25905 {
25912 {
25913 /* We always use R10 in 64-bit mode. */
25915 }
25916 else
25917 {
25918 tree fntype;
25921 /* By default in 32-bit mode we use ECX to pass the static chain. */
25927 {
25928 /* Fastcall functions use ecx/edx for arguments, which leaves
25929 us with EAX for the static chain.
25930 Thiscall functions use ecx for arguments, which also
25931 leaves us with EAX for the static chain. */
25933 }
25935 {
25936 /* Thiscall functions use ecx for arguments, which leaves
25937 us with EAX and EDX for the static chain.
25938 We are using for abi-compatibility EAX. */
25940 }
25942 {
25943 /* For regparm 3, we have no free call-clobbered registers in
25944 which to store the static chain. In order to implement this,
25945 we have the trampoline push the static chain to the stack.
25946 However, we can't push a value below the return address when
25947 we call the nested function directly, so we have to use an
25948 alternate entry point. For this we use ESI, and have the
25949 alternate entry point push ESI, so that things appear the
25950 same once we're executing the nested function. */
25952 {
25958 }
25960 }
25961 }
25964 }
25966 /* Emit RTL insns to initialize the variable parts of a trampoline.
25967 FNDECL is the decl of the target address; M_TRAMP is a MEM for
25968 the trampoline, and CHAIN_VALUE is an RTX for the static chain
25969 to be passed to the target function. */
25971 static void
25973 {
25981 {
25984 /* Load the function address to r11. Try to load address using
25985 the shorter movl instead of movabs. We may want to support
25986 movq for kernel mode, but kernel does not use trampolines at
25987 the moment. FNADDR is a 32bit address and may not be in
25988 DImode when ptr_mode == SImode. Always use movl in this
25989 case. */
25992 {
26001 }
26002 else
26003 {
26010 }
26012 /* Load static chain using movabs to r10. Use the shorter movl
26013 instead of movabs when ptr_mode == SImode. */
26015 {
26018 }
26019 else
26020 {
26023 }
26032 /* Jump to r11; the last (unused) byte is a nop, only there to
26033 pad the write out to a single 32-bit store. */
26037 }
26038 else
26039 {
26042 /* Depending on the static chain location, either load a register
26043 with a constant, or push the constant to the stack. All of the
26044 instructions are the same size. */
26047 {
26049 {
26056 }
26057 }
26058 else
26073 /* Compute offset from the end of the jmp to the target function.
26074 In the case in which the trampoline stores the static chain on
26075 the stack, we need to skip the first insn which pushes the
26076 (call-saved) register static chain; this push is 1 byte. */
26083 }
26087 #ifdef HAVE_ENABLE_EXECUTE_STACK
26088 #ifdef CHECK_EXECUTE_STACK_ENABLED
26090 #endif
26093 #endif
26094 }
26095 \f
26096 /* The following file contains several enumerations and data structures
26097 built from the definitions in i386-builtin-types.def. */
26101 /* Table for the ix86 builtin non-function types. */
26104 /* Retrieve an element from the above table, building some of
26105 the types lazily. */
26107 static tree
26109 {
26121 {
26129 }
26130 else
26131 {
26137 else
26145 }
26149 }
26151 /* Table for the ix86 builtin function types. */
26154 /* Retrieve an element from the above table, building some of
26155 the types lazily. */
26157 static tree
26159 {
26160 tree type;
26169 {
26177 {
26180 }
26183 }
26184 else
26185 {
26191 }
26195 }
26198 /* Codes for all the SSE/MMX builtins. */
26199 enum ix86_builtins
26200 {
26201 IX86_BUILTIN_ADDPS,
26202 IX86_BUILTIN_ADDSS,
26203 IX86_BUILTIN_DIVPS,
26204 IX86_BUILTIN_DIVSS,
26205 IX86_BUILTIN_MULPS,
26206 IX86_BUILTIN_MULSS,
26207 IX86_BUILTIN_SUBPS,
26208 IX86_BUILTIN_SUBSS,
26210 IX86_BUILTIN_CMPEQPS,
26211 IX86_BUILTIN_CMPLTPS,
26212 IX86_BUILTIN_CMPLEPS,
26213 IX86_BUILTIN_CMPGTPS,
26214 IX86_BUILTIN_CMPGEPS,
26215 IX86_BUILTIN_CMPNEQPS,
26216 IX86_BUILTIN_CMPNLTPS,
26217 IX86_BUILTIN_CMPNLEPS,
26218 IX86_BUILTIN_CMPNGTPS,
26219 IX86_BUILTIN_CMPNGEPS,
26220 IX86_BUILTIN_CMPORDPS,
26221 IX86_BUILTIN_CMPUNORDPS,
26222 IX86_BUILTIN_CMPEQSS,
26223 IX86_BUILTIN_CMPLTSS,
26224 IX86_BUILTIN_CMPLESS,
26225 IX86_BUILTIN_CMPNEQSS,
26226 IX86_BUILTIN_CMPNLTSS,
26227 IX86_BUILTIN_CMPNLESS,
26228 IX86_BUILTIN_CMPORDSS,
26229 IX86_BUILTIN_CMPUNORDSS,
26231 IX86_BUILTIN_COMIEQSS,
26232 IX86_BUILTIN_COMILTSS,
26233 IX86_BUILTIN_COMILESS,
26234 IX86_BUILTIN_COMIGTSS,
26235 IX86_BUILTIN_COMIGESS,
26236 IX86_BUILTIN_COMINEQSS,
26237 IX86_BUILTIN_UCOMIEQSS,
26238 IX86_BUILTIN_UCOMILTSS,
26239 IX86_BUILTIN_UCOMILESS,
26240 IX86_BUILTIN_UCOMIGTSS,
26241 IX86_BUILTIN_UCOMIGESS,
26242 IX86_BUILTIN_UCOMINEQSS,
26244 IX86_BUILTIN_CVTPI2PS,
26245 IX86_BUILTIN_CVTPS2PI,
26246 IX86_BUILTIN_CVTSI2SS,
26247 IX86_BUILTIN_CVTSI642SS,
26248 IX86_BUILTIN_CVTSS2SI,
26249 IX86_BUILTIN_CVTSS2SI64,
26250 IX86_BUILTIN_CVTTPS2PI,
26251 IX86_BUILTIN_CVTTSS2SI,
26252 IX86_BUILTIN_CVTTSS2SI64,
26254 IX86_BUILTIN_MAXPS,
26255 IX86_BUILTIN_MAXSS,
26256 IX86_BUILTIN_MINPS,
26257 IX86_BUILTIN_MINSS,
26259 IX86_BUILTIN_LOADUPS,
26260 IX86_BUILTIN_STOREUPS,
26261 IX86_BUILTIN_MOVSS,
26263 IX86_BUILTIN_MOVHLPS,
26264 IX86_BUILTIN_MOVLHPS,
26265 IX86_BUILTIN_LOADHPS,
26266 IX86_BUILTIN_LOADLPS,
26267 IX86_BUILTIN_STOREHPS,
26268 IX86_BUILTIN_STORELPS,
26270 IX86_BUILTIN_MASKMOVQ,
26271 IX86_BUILTIN_MOVMSKPS,
26272 IX86_BUILTIN_PMOVMSKB,
26274 IX86_BUILTIN_MOVNTPS,
26275 IX86_BUILTIN_MOVNTQ,
26277 IX86_BUILTIN_LOADDQU,
26278 IX86_BUILTIN_STOREDQU,
26280 IX86_BUILTIN_PACKSSWB,
26281 IX86_BUILTIN_PACKSSDW,
26282 IX86_BUILTIN_PACKUSWB,
26284 IX86_BUILTIN_PADDB,
26285 IX86_BUILTIN_PADDW,
26286 IX86_BUILTIN_PADDD,
26287 IX86_BUILTIN_PADDQ,
26288 IX86_BUILTIN_PADDSB,
26289 IX86_BUILTIN_PADDSW,
26290 IX86_BUILTIN_PADDUSB,
26291 IX86_BUILTIN_PADDUSW,
26292 IX86_BUILTIN_PSUBB,
26293 IX86_BUILTIN_PSUBW,
26294 IX86_BUILTIN_PSUBD,
26295 IX86_BUILTIN_PSUBQ,
26296 IX86_BUILTIN_PSUBSB,
26297 IX86_BUILTIN_PSUBSW,
26298 IX86_BUILTIN_PSUBUSB,
26299 IX86_BUILTIN_PSUBUSW,
26301 IX86_BUILTIN_PAND,
26302 IX86_BUILTIN_PANDN,
26303 IX86_BUILTIN_POR,
26304 IX86_BUILTIN_PXOR,
26306 IX86_BUILTIN_PAVGB,
26307 IX86_BUILTIN_PAVGW,
26309 IX86_BUILTIN_PCMPEQB,
26310 IX86_BUILTIN_PCMPEQW,
26311 IX86_BUILTIN_PCMPEQD,
26312 IX86_BUILTIN_PCMPGTB,
26313 IX86_BUILTIN_PCMPGTW,
26314 IX86_BUILTIN_PCMPGTD,
26316 IX86_BUILTIN_PMADDWD,
26318 IX86_BUILTIN_PMAXSW,
26319 IX86_BUILTIN_PMAXUB,
26320 IX86_BUILTIN_PMINSW,
26321 IX86_BUILTIN_PMINUB,
26323 IX86_BUILTIN_PMULHUW,
26324 IX86_BUILTIN_PMULHW,
26325 IX86_BUILTIN_PMULLW,
26327 IX86_BUILTIN_PSADBW,
26328 IX86_BUILTIN_PSHUFW,
26330 IX86_BUILTIN_PSLLW,
26331 IX86_BUILTIN_PSLLD,
26332 IX86_BUILTIN_PSLLQ,
26333 IX86_BUILTIN_PSRAW,
26334 IX86_BUILTIN_PSRAD,
26335 IX86_BUILTIN_PSRLW,
26336 IX86_BUILTIN_PSRLD,
26337 IX86_BUILTIN_PSRLQ,
26338 IX86_BUILTIN_PSLLWI,
26339 IX86_BUILTIN_PSLLDI,
26340 IX86_BUILTIN_PSLLQI,
26341 IX86_BUILTIN_PSRAWI,
26342 IX86_BUILTIN_PSRADI,
26343 IX86_BUILTIN_PSRLWI,
26344 IX86_BUILTIN_PSRLDI,
26345 IX86_BUILTIN_PSRLQI,
26347 IX86_BUILTIN_PUNPCKHBW,
26348 IX86_BUILTIN_PUNPCKHWD,
26349 IX86_BUILTIN_PUNPCKHDQ,
26350 IX86_BUILTIN_PUNPCKLBW,
26351 IX86_BUILTIN_PUNPCKLWD,
26352 IX86_BUILTIN_PUNPCKLDQ,
26354 IX86_BUILTIN_SHUFPS,
26356 IX86_BUILTIN_RCPPS,
26357 IX86_BUILTIN_RCPSS,
26358 IX86_BUILTIN_RSQRTPS,
26359 IX86_BUILTIN_RSQRTPS_NR,
26360 IX86_BUILTIN_RSQRTSS,
26361 IX86_BUILTIN_RSQRTF,
26362 IX86_BUILTIN_SQRTPS,
26363 IX86_BUILTIN_SQRTPS_NR,
26364 IX86_BUILTIN_SQRTSS,
26366 IX86_BUILTIN_UNPCKHPS,
26367 IX86_BUILTIN_UNPCKLPS,
26369 IX86_BUILTIN_ANDPS,
26370 IX86_BUILTIN_ANDNPS,
26371 IX86_BUILTIN_ORPS,
26372 IX86_BUILTIN_XORPS,
26374 IX86_BUILTIN_EMMS,
26375 IX86_BUILTIN_LDMXCSR,
26376 IX86_BUILTIN_STMXCSR,
26377 IX86_BUILTIN_SFENCE,
26379 IX86_BUILTIN_FXSAVE,
26380 IX86_BUILTIN_FXRSTOR,
26381 IX86_BUILTIN_FXSAVE64,
26382 IX86_BUILTIN_FXRSTOR64,
26384 IX86_BUILTIN_XSAVE,
26385 IX86_BUILTIN_XRSTOR,
26386 IX86_BUILTIN_XSAVE64,
26387 IX86_BUILTIN_XRSTOR64,
26389 IX86_BUILTIN_XSAVEOPT,
26390 IX86_BUILTIN_XSAVEOPT64,
26392 /* 3DNow! Original */
26393 IX86_BUILTIN_FEMMS,
26394 IX86_BUILTIN_PAVGUSB,
26395 IX86_BUILTIN_PF2ID,
26396 IX86_BUILTIN_PFACC,
26397 IX86_BUILTIN_PFADD,
26398 IX86_BUILTIN_PFCMPEQ,
26399 IX86_BUILTIN_PFCMPGE,
26400 IX86_BUILTIN_PFCMPGT,
26401 IX86_BUILTIN_PFMAX,
26402 IX86_BUILTIN_PFMIN,
26403 IX86_BUILTIN_PFMUL,
26404 IX86_BUILTIN_PFRCP,
26405 IX86_BUILTIN_PFRCPIT1,
26406 IX86_BUILTIN_PFRCPIT2,
26407 IX86_BUILTIN_PFRSQIT1,
26408 IX86_BUILTIN_PFRSQRT,
26409 IX86_BUILTIN_PFSUB,
26410 IX86_BUILTIN_PFSUBR,
26411 IX86_BUILTIN_PI2FD,
26412 IX86_BUILTIN_PMULHRW,
26414 /* 3DNow! Athlon Extensions */
26415 IX86_BUILTIN_PF2IW,
26416 IX86_BUILTIN_PFNACC,
26417 IX86_BUILTIN_PFPNACC,
26418 IX86_BUILTIN_PI2FW,
26419 IX86_BUILTIN_PSWAPDSI,
26420 IX86_BUILTIN_PSWAPDSF,
26422 /* SSE2 */
26423 IX86_BUILTIN_ADDPD,
26424 IX86_BUILTIN_ADDSD,
26425 IX86_BUILTIN_DIVPD,
26426 IX86_BUILTIN_DIVSD,
26427 IX86_BUILTIN_MULPD,
26428 IX86_BUILTIN_MULSD,
26429 IX86_BUILTIN_SUBPD,
26430 IX86_BUILTIN_SUBSD,
26432 IX86_BUILTIN_CMPEQPD,
26433 IX86_BUILTIN_CMPLTPD,
26434 IX86_BUILTIN_CMPLEPD,
26435 IX86_BUILTIN_CMPGTPD,
26436 IX86_BUILTIN_CMPGEPD,
26437 IX86_BUILTIN_CMPNEQPD,
26438 IX86_BUILTIN_CMPNLTPD,
26439 IX86_BUILTIN_CMPNLEPD,
26440 IX86_BUILTIN_CMPNGTPD,
26441 IX86_BUILTIN_CMPNGEPD,
26442 IX86_BUILTIN_CMPORDPD,
26443 IX86_BUILTIN_CMPUNORDPD,
26444 IX86_BUILTIN_CMPEQSD,
26445 IX86_BUILTIN_CMPLTSD,
26446 IX86_BUILTIN_CMPLESD,
26447 IX86_BUILTIN_CMPNEQSD,
26448 IX86_BUILTIN_CMPNLTSD,
26449 IX86_BUILTIN_CMPNLESD,
26450 IX86_BUILTIN_CMPORDSD,
26451 IX86_BUILTIN_CMPUNORDSD,
26453 IX86_BUILTIN_COMIEQSD,
26454 IX86_BUILTIN_COMILTSD,
26455 IX86_BUILTIN_COMILESD,
26456 IX86_BUILTIN_COMIGTSD,
26457 IX86_BUILTIN_COMIGESD,
26458 IX86_BUILTIN_COMINEQSD,
26459 IX86_BUILTIN_UCOMIEQSD,
26460 IX86_BUILTIN_UCOMILTSD,
26461 IX86_BUILTIN_UCOMILESD,
26462 IX86_BUILTIN_UCOMIGTSD,
26463 IX86_BUILTIN_UCOMIGESD,
26464 IX86_BUILTIN_UCOMINEQSD,
26466 IX86_BUILTIN_MAXPD,
26467 IX86_BUILTIN_MAXSD,
26468 IX86_BUILTIN_MINPD,
26469 IX86_BUILTIN_MINSD,
26471 IX86_BUILTIN_ANDPD,
26472 IX86_BUILTIN_ANDNPD,
26473 IX86_BUILTIN_ORPD,
26474 IX86_BUILTIN_XORPD,
26476 IX86_BUILTIN_SQRTPD,
26477 IX86_BUILTIN_SQRTSD,
26479 IX86_BUILTIN_UNPCKHPD,
26480 IX86_BUILTIN_UNPCKLPD,
26482 IX86_BUILTIN_SHUFPD,
26484 IX86_BUILTIN_LOADUPD,
26485 IX86_BUILTIN_STOREUPD,
26486 IX86_BUILTIN_MOVSD,
26488 IX86_BUILTIN_LOADHPD,
26489 IX86_BUILTIN_LOADLPD,
26491 IX86_BUILTIN_CVTDQ2PD,
26492 IX86_BUILTIN_CVTDQ2PS,
26494 IX86_BUILTIN_CVTPD2DQ,
26495 IX86_BUILTIN_CVTPD2PI,
26496 IX86_BUILTIN_CVTPD2PS,
26497 IX86_BUILTIN_CVTTPD2DQ,
26498 IX86_BUILTIN_CVTTPD2PI,
26500 IX86_BUILTIN_CVTPI2PD,
26501 IX86_BUILTIN_CVTSI2SD,
26502 IX86_BUILTIN_CVTSI642SD,
26504 IX86_BUILTIN_CVTSD2SI,
26505 IX86_BUILTIN_CVTSD2SI64,
26506 IX86_BUILTIN_CVTSD2SS,
26507 IX86_BUILTIN_CVTSS2SD,
26508 IX86_BUILTIN_CVTTSD2SI,
26509 IX86_BUILTIN_CVTTSD2SI64,
26511 IX86_BUILTIN_CVTPS2DQ,
26512 IX86_BUILTIN_CVTPS2PD,
26513 IX86_BUILTIN_CVTTPS2DQ,
26515 IX86_BUILTIN_MOVNTI,
26516 IX86_BUILTIN_MOVNTI64,
26517 IX86_BUILTIN_MOVNTPD,
26518 IX86_BUILTIN_MOVNTDQ,
26520 IX86_BUILTIN_MOVQ128,
26522 /* SSE2 MMX */
26523 IX86_BUILTIN_MASKMOVDQU,
26524 IX86_BUILTIN_MOVMSKPD,
26525 IX86_BUILTIN_PMOVMSKB128,
26527 IX86_BUILTIN_PACKSSWB128,
26528 IX86_BUILTIN_PACKSSDW128,
26529 IX86_BUILTIN_PACKUSWB128,
26531 IX86_BUILTIN_PADDB128,
26532 IX86_BUILTIN_PADDW128,
26533 IX86_BUILTIN_PADDD128,
26534 IX86_BUILTIN_PADDQ128,
26535 IX86_BUILTIN_PADDSB128,
26536 IX86_BUILTIN_PADDSW128,
26537 IX86_BUILTIN_PADDUSB128,
26538 IX86_BUILTIN_PADDUSW128,
26539 IX86_BUILTIN_PSUBB128,
26540 IX86_BUILTIN_PSUBW128,
26541 IX86_BUILTIN_PSUBD128,
26542 IX86_BUILTIN_PSUBQ128,
26543 IX86_BUILTIN_PSUBSB128,
26544 IX86_BUILTIN_PSUBSW128,
26545 IX86_BUILTIN_PSUBUSB128,
26546 IX86_BUILTIN_PSUBUSW128,
26548 IX86_BUILTIN_PAND128,
26549 IX86_BUILTIN_PANDN128,
26550 IX86_BUILTIN_POR128,
26551 IX86_BUILTIN_PXOR128,
26553 IX86_BUILTIN_PAVGB128,
26554 IX86_BUILTIN_PAVGW128,
26556 IX86_BUILTIN_PCMPEQB128,
26557 IX86_BUILTIN_PCMPEQW128,
26558 IX86_BUILTIN_PCMPEQD128,
26559 IX86_BUILTIN_PCMPGTB128,
26560 IX86_BUILTIN_PCMPGTW128,
26561 IX86_BUILTIN_PCMPGTD128,
26563 IX86_BUILTIN_PMADDWD128,
26565 IX86_BUILTIN_PMAXSW128,
26566 IX86_BUILTIN_PMAXUB128,
26567 IX86_BUILTIN_PMINSW128,
26568 IX86_BUILTIN_PMINUB128,
26570 IX86_BUILTIN_PMULUDQ,
26571 IX86_BUILTIN_PMULUDQ128,
26572 IX86_BUILTIN_PMULHUW128,
26573 IX86_BUILTIN_PMULHW128,
26574 IX86_BUILTIN_PMULLW128,
26576 IX86_BUILTIN_PSADBW128,
26577 IX86_BUILTIN_PSHUFHW,
26578 IX86_BUILTIN_PSHUFLW,
26579 IX86_BUILTIN_PSHUFD,
26581 IX86_BUILTIN_PSLLDQI128,
26582 IX86_BUILTIN_PSLLWI128,
26583 IX86_BUILTIN_PSLLDI128,
26584 IX86_BUILTIN_PSLLQI128,
26585 IX86_BUILTIN_PSRAWI128,
26586 IX86_BUILTIN_PSRADI128,
26587 IX86_BUILTIN_PSRLDQI128,
26588 IX86_BUILTIN_PSRLWI128,
26589 IX86_BUILTIN_PSRLDI128,
26590 IX86_BUILTIN_PSRLQI128,
26592 IX86_BUILTIN_PSLLDQ128,
26593 IX86_BUILTIN_PSLLW128,
26594 IX86_BUILTIN_PSLLD128,
26595 IX86_BUILTIN_PSLLQ128,
26596 IX86_BUILTIN_PSRAW128,
26597 IX86_BUILTIN_PSRAD128,
26598 IX86_BUILTIN_PSRLW128,
26599 IX86_BUILTIN_PSRLD128,
26600 IX86_BUILTIN_PSRLQ128,
26602 IX86_BUILTIN_PUNPCKHBW128,
26603 IX86_BUILTIN_PUNPCKHWD128,
26604 IX86_BUILTIN_PUNPCKHDQ128,
26605 IX86_BUILTIN_PUNPCKHQDQ128,
26606 IX86_BUILTIN_PUNPCKLBW128,
26607 IX86_BUILTIN_PUNPCKLWD128,
26608 IX86_BUILTIN_PUNPCKLDQ128,
26609 IX86_BUILTIN_PUNPCKLQDQ128,
26611 IX86_BUILTIN_CLFLUSH,
26612 IX86_BUILTIN_MFENCE,
26613 IX86_BUILTIN_LFENCE,
26614 IX86_BUILTIN_PAUSE,
26616 IX86_BUILTIN_BSRSI,
26617 IX86_BUILTIN_BSRDI,
26618 IX86_BUILTIN_RDPMC,
26619 IX86_BUILTIN_RDTSC,
26620 IX86_BUILTIN_RDTSCP,
26621 IX86_BUILTIN_ROLQI,
26622 IX86_BUILTIN_ROLHI,
26623 IX86_BUILTIN_RORQI,
26624 IX86_BUILTIN_RORHI,
26626 /* SSE3. */
26627 IX86_BUILTIN_ADDSUBPS,
26628 IX86_BUILTIN_HADDPS,
26629 IX86_BUILTIN_HSUBPS,
26630 IX86_BUILTIN_MOVSHDUP,
26631 IX86_BUILTIN_MOVSLDUP,
26632 IX86_BUILTIN_ADDSUBPD,
26633 IX86_BUILTIN_HADDPD,
26634 IX86_BUILTIN_HSUBPD,
26635 IX86_BUILTIN_LDDQU,
26637 IX86_BUILTIN_MONITOR,
26638 IX86_BUILTIN_MWAIT,
26640 /* SSSE3. */
26641 IX86_BUILTIN_PHADDW,
26642 IX86_BUILTIN_PHADDD,
26643 IX86_BUILTIN_PHADDSW,
26644 IX86_BUILTIN_PHSUBW,
26645 IX86_BUILTIN_PHSUBD,
26646 IX86_BUILTIN_PHSUBSW,
26647 IX86_BUILTIN_PMADDUBSW,
26648 IX86_BUILTIN_PMULHRSW,
26649 IX86_BUILTIN_PSHUFB,
26650 IX86_BUILTIN_PSIGNB,
26651 IX86_BUILTIN_PSIGNW,
26652 IX86_BUILTIN_PSIGND,
26653 IX86_BUILTIN_PALIGNR,
26654 IX86_BUILTIN_PABSB,
26655 IX86_BUILTIN_PABSW,
26656 IX86_BUILTIN_PABSD,
26658 IX86_BUILTIN_PHADDW128,
26659 IX86_BUILTIN_PHADDD128,
26660 IX86_BUILTIN_PHADDSW128,
26661 IX86_BUILTIN_PHSUBW128,
26662 IX86_BUILTIN_PHSUBD128,
26663 IX86_BUILTIN_PHSUBSW128,
26664 IX86_BUILTIN_PMADDUBSW128,
26665 IX86_BUILTIN_PMULHRSW128,
26666 IX86_BUILTIN_PSHUFB128,
26667 IX86_BUILTIN_PSIGNB128,
26668 IX86_BUILTIN_PSIGNW128,
26669 IX86_BUILTIN_PSIGND128,
26670 IX86_BUILTIN_PALIGNR128,
26671 IX86_BUILTIN_PABSB128,
26672 IX86_BUILTIN_PABSW128,
26673 IX86_BUILTIN_PABSD128,
26675 /* AMDFAM10 - SSE4A New Instructions. */
26676 IX86_BUILTIN_MOVNTSD,
26677 IX86_BUILTIN_MOVNTSS,
26678 IX86_BUILTIN_EXTRQI,
26679 IX86_BUILTIN_EXTRQ,
26680 IX86_BUILTIN_INSERTQI,
26681 IX86_BUILTIN_INSERTQ,
26683 /* SSE4.1. */
26684 IX86_BUILTIN_BLENDPD,
26685 IX86_BUILTIN_BLENDPS,
26686 IX86_BUILTIN_BLENDVPD,
26687 IX86_BUILTIN_BLENDVPS,
26688 IX86_BUILTIN_PBLENDVB128,
26689 IX86_BUILTIN_PBLENDW128,
26691 IX86_BUILTIN_DPPD,
26692 IX86_BUILTIN_DPPS,
26694 IX86_BUILTIN_INSERTPS128,
26696 IX86_BUILTIN_MOVNTDQA,
26697 IX86_BUILTIN_MPSADBW128,
26698 IX86_BUILTIN_PACKUSDW128,
26699 IX86_BUILTIN_PCMPEQQ,
26700 IX86_BUILTIN_PHMINPOSUW128,
26702 IX86_BUILTIN_PMAXSB128,
26703 IX86_BUILTIN_PMAXSD128,
26704 IX86_BUILTIN_PMAXUD128,
26705 IX86_BUILTIN_PMAXUW128,
26707 IX86_BUILTIN_PMINSB128,
26708 IX86_BUILTIN_PMINSD128,
26709 IX86_BUILTIN_PMINUD128,
26710 IX86_BUILTIN_PMINUW128,
26712 IX86_BUILTIN_PMOVSXBW128,
26713 IX86_BUILTIN_PMOVSXBD128,
26714 IX86_BUILTIN_PMOVSXBQ128,
26715 IX86_BUILTIN_PMOVSXWD128,
26716 IX86_BUILTIN_PMOVSXWQ128,
26717 IX86_BUILTIN_PMOVSXDQ128,
26719 IX86_BUILTIN_PMOVZXBW128,
26720 IX86_BUILTIN_PMOVZXBD128,
26721 IX86_BUILTIN_PMOVZXBQ128,
26722 IX86_BUILTIN_PMOVZXWD128,
26723 IX86_BUILTIN_PMOVZXWQ128,
26724 IX86_BUILTIN_PMOVZXDQ128,
26726 IX86_BUILTIN_PMULDQ128,
26727 IX86_BUILTIN_PMULLD128,
26729 IX86_BUILTIN_ROUNDSD,
26730 IX86_BUILTIN_ROUNDSS,
26732 IX86_BUILTIN_ROUNDPD,
26733 IX86_BUILTIN_ROUNDPS,
26735 IX86_BUILTIN_FLOORPD,
26736 IX86_BUILTIN_CEILPD,
26737 IX86_BUILTIN_TRUNCPD,
26738 IX86_BUILTIN_RINTPD,
26739 IX86_BUILTIN_ROUNDPD_AZ,
26741 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
26742 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
26743 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
26745 IX86_BUILTIN_FLOORPS,
26746 IX86_BUILTIN_CEILPS,
26747 IX86_BUILTIN_TRUNCPS,
26748 IX86_BUILTIN_RINTPS,
26749 IX86_BUILTIN_ROUNDPS_AZ,
26751 IX86_BUILTIN_FLOORPS_SFIX,
26752 IX86_BUILTIN_CEILPS_SFIX,
26753 IX86_BUILTIN_ROUNDPS_AZ_SFIX,
26755 IX86_BUILTIN_PTESTZ,
26756 IX86_BUILTIN_PTESTC,
26757 IX86_BUILTIN_PTESTNZC,
26759 IX86_BUILTIN_VEC_INIT_V2SI,
26760 IX86_BUILTIN_VEC_INIT_V4HI,
26761 IX86_BUILTIN_VEC_INIT_V8QI,
26762 IX86_BUILTIN_VEC_EXT_V2DF,
26763 IX86_BUILTIN_VEC_EXT_V2DI,
26764 IX86_BUILTIN_VEC_EXT_V4SF,
26765 IX86_BUILTIN_VEC_EXT_V4SI,
26766 IX86_BUILTIN_VEC_EXT_V8HI,
26767 IX86_BUILTIN_VEC_EXT_V2SI,
26768 IX86_BUILTIN_VEC_EXT_V4HI,
26769 IX86_BUILTIN_VEC_EXT_V16QI,
26770 IX86_BUILTIN_VEC_SET_V2DI,
26771 IX86_BUILTIN_VEC_SET_V4SF,
26772 IX86_BUILTIN_VEC_SET_V4SI,
26773 IX86_BUILTIN_VEC_SET_V8HI,
26774 IX86_BUILTIN_VEC_SET_V4HI,
26775 IX86_BUILTIN_VEC_SET_V16QI,
26777 IX86_BUILTIN_VEC_PACK_SFIX,
26778 IX86_BUILTIN_VEC_PACK_SFIX256,
26780 /* SSE4.2. */
26781 IX86_BUILTIN_CRC32QI,
26782 IX86_BUILTIN_CRC32HI,
26783 IX86_BUILTIN_CRC32SI,
26784 IX86_BUILTIN_CRC32DI,
26786 IX86_BUILTIN_PCMPESTRI128,
26787 IX86_BUILTIN_PCMPESTRM128,
26788 IX86_BUILTIN_PCMPESTRA128,
26789 IX86_BUILTIN_PCMPESTRC128,
26790 IX86_BUILTIN_PCMPESTRO128,
26791 IX86_BUILTIN_PCMPESTRS128,
26792 IX86_BUILTIN_PCMPESTRZ128,
26793 IX86_BUILTIN_PCMPISTRI128,
26794 IX86_BUILTIN_PCMPISTRM128,
26795 IX86_BUILTIN_PCMPISTRA128,
26796 IX86_BUILTIN_PCMPISTRC128,
26797 IX86_BUILTIN_PCMPISTRO128,
26798 IX86_BUILTIN_PCMPISTRS128,
26799 IX86_BUILTIN_PCMPISTRZ128,
26801 IX86_BUILTIN_PCMPGTQ,
26803 /* AES instructions */
26804 IX86_BUILTIN_AESENC128,
26805 IX86_BUILTIN_AESENCLAST128,
26806 IX86_BUILTIN_AESDEC128,
26807 IX86_BUILTIN_AESDECLAST128,
26808 IX86_BUILTIN_AESIMC128,
26809 IX86_BUILTIN_AESKEYGENASSIST128,
26811 /* PCLMUL instruction */
26812 IX86_BUILTIN_PCLMULQDQ128,
26814 /* AVX */
26815 IX86_BUILTIN_ADDPD256,
26816 IX86_BUILTIN_ADDPS256,
26817 IX86_BUILTIN_ADDSUBPD256,
26818 IX86_BUILTIN_ADDSUBPS256,
26819 IX86_BUILTIN_ANDPD256,
26820 IX86_BUILTIN_ANDPS256,
26821 IX86_BUILTIN_ANDNPD256,
26822 IX86_BUILTIN_ANDNPS256,
26823 IX86_BUILTIN_BLENDPD256,
26824 IX86_BUILTIN_BLENDPS256,
26825 IX86_BUILTIN_BLENDVPD256,
26826 IX86_BUILTIN_BLENDVPS256,
26827 IX86_BUILTIN_DIVPD256,
26828 IX86_BUILTIN_DIVPS256,
26829 IX86_BUILTIN_DPPS256,
26830 IX86_BUILTIN_HADDPD256,
26831 IX86_BUILTIN_HADDPS256,
26832 IX86_BUILTIN_HSUBPD256,
26833 IX86_BUILTIN_HSUBPS256,
26834 IX86_BUILTIN_MAXPD256,
26835 IX86_BUILTIN_MAXPS256,
26836 IX86_BUILTIN_MINPD256,
26837 IX86_BUILTIN_MINPS256,
26838 IX86_BUILTIN_MULPD256,
26839 IX86_BUILTIN_MULPS256,
26840 IX86_BUILTIN_ORPD256,
26841 IX86_BUILTIN_ORPS256,
26842 IX86_BUILTIN_SHUFPD256,
26843 IX86_BUILTIN_SHUFPS256,
26844 IX86_BUILTIN_SUBPD256,
26845 IX86_BUILTIN_SUBPS256,
26846 IX86_BUILTIN_XORPD256,
26847 IX86_BUILTIN_XORPS256,
26848 IX86_BUILTIN_CMPSD,
26849 IX86_BUILTIN_CMPSS,
26850 IX86_BUILTIN_CMPPD,
26851 IX86_BUILTIN_CMPPS,
26852 IX86_BUILTIN_CMPPD256,
26853 IX86_BUILTIN_CMPPS256,
26854 IX86_BUILTIN_CVTDQ2PD256,
26855 IX86_BUILTIN_CVTDQ2PS256,
26856 IX86_BUILTIN_CVTPD2PS256,
26857 IX86_BUILTIN_CVTPS2DQ256,
26858 IX86_BUILTIN_CVTPS2PD256,
26859 IX86_BUILTIN_CVTTPD2DQ256,
26860 IX86_BUILTIN_CVTPD2DQ256,
26861 IX86_BUILTIN_CVTTPS2DQ256,
26862 IX86_BUILTIN_EXTRACTF128PD256,
26863 IX86_BUILTIN_EXTRACTF128PS256,
26864 IX86_BUILTIN_EXTRACTF128SI256,
26865 IX86_BUILTIN_VZEROALL,
26866 IX86_BUILTIN_VZEROUPPER,
26867 IX86_BUILTIN_VPERMILVARPD,
26868 IX86_BUILTIN_VPERMILVARPS,
26869 IX86_BUILTIN_VPERMILVARPD256,
26870 IX86_BUILTIN_VPERMILVARPS256,
26871 IX86_BUILTIN_VPERMILPD,
26872 IX86_BUILTIN_VPERMILPS,
26873 IX86_BUILTIN_VPERMILPD256,
26874 IX86_BUILTIN_VPERMILPS256,
26875 IX86_BUILTIN_VPERMIL2PD,
26876 IX86_BUILTIN_VPERMIL2PS,
26877 IX86_BUILTIN_VPERMIL2PD256,
26878 IX86_BUILTIN_VPERMIL2PS256,
26879 IX86_BUILTIN_VPERM2F128PD256,
26880 IX86_BUILTIN_VPERM2F128PS256,
26881 IX86_BUILTIN_VPERM2F128SI256,
26882 IX86_BUILTIN_VBROADCASTSS,
26883 IX86_BUILTIN_VBROADCASTSD256,
26884 IX86_BUILTIN_VBROADCASTSS256,
26885 IX86_BUILTIN_VBROADCASTPD256,
26886 IX86_BUILTIN_VBROADCASTPS256,
26887 IX86_BUILTIN_VINSERTF128PD256,
26888 IX86_BUILTIN_VINSERTF128PS256,
26889 IX86_BUILTIN_VINSERTF128SI256,
26890 IX86_BUILTIN_LOADUPD256,
26891 IX86_BUILTIN_LOADUPS256,
26892 IX86_BUILTIN_STOREUPD256,
26893 IX86_BUILTIN_STOREUPS256,
26894 IX86_BUILTIN_LDDQU256,
26895 IX86_BUILTIN_MOVNTDQ256,
26896 IX86_BUILTIN_MOVNTPD256,
26897 IX86_BUILTIN_MOVNTPS256,
26898 IX86_BUILTIN_LOADDQU256,
26899 IX86_BUILTIN_STOREDQU256,
26900 IX86_BUILTIN_MASKLOADPD,
26901 IX86_BUILTIN_MASKLOADPS,
26902 IX86_BUILTIN_MASKSTOREPD,
26903 IX86_BUILTIN_MASKSTOREPS,
26904 IX86_BUILTIN_MASKLOADPD256,
26905 IX86_BUILTIN_MASKLOADPS256,
26906 IX86_BUILTIN_MASKSTOREPD256,
26907 IX86_BUILTIN_MASKSTOREPS256,
26908 IX86_BUILTIN_MOVSHDUP256,
26909 IX86_BUILTIN_MOVSLDUP256,
26910 IX86_BUILTIN_MOVDDUP256,
26912 IX86_BUILTIN_SQRTPD256,
26913 IX86_BUILTIN_SQRTPS256,
26914 IX86_BUILTIN_SQRTPS_NR256,
26915 IX86_BUILTIN_RSQRTPS256,
26916 IX86_BUILTIN_RSQRTPS_NR256,
26918 IX86_BUILTIN_RCPPS256,
26920 IX86_BUILTIN_ROUNDPD256,
26921 IX86_BUILTIN_ROUNDPS256,
26923 IX86_BUILTIN_FLOORPD256,
26924 IX86_BUILTIN_CEILPD256,
26925 IX86_BUILTIN_TRUNCPD256,
26926 IX86_BUILTIN_RINTPD256,
26927 IX86_BUILTIN_ROUNDPD_AZ256,
26929 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
26930 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
26931 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
26933 IX86_BUILTIN_FLOORPS256,
26934 IX86_BUILTIN_CEILPS256,
26935 IX86_BUILTIN_TRUNCPS256,
26936 IX86_BUILTIN_RINTPS256,
26937 IX86_BUILTIN_ROUNDPS_AZ256,
26939 IX86_BUILTIN_FLOORPS_SFIX256,
26940 IX86_BUILTIN_CEILPS_SFIX256,
26941 IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
26943 IX86_BUILTIN_UNPCKHPD256,
26944 IX86_BUILTIN_UNPCKLPD256,
26945 IX86_BUILTIN_UNPCKHPS256,
26946 IX86_BUILTIN_UNPCKLPS256,
26948 IX86_BUILTIN_SI256_SI,
26949 IX86_BUILTIN_PS256_PS,
26950 IX86_BUILTIN_PD256_PD,
26951 IX86_BUILTIN_SI_SI256,
26952 IX86_BUILTIN_PS_PS256,
26953 IX86_BUILTIN_PD_PD256,
26955 IX86_BUILTIN_VTESTZPD,
26956 IX86_BUILTIN_VTESTCPD,
26957 IX86_BUILTIN_VTESTNZCPD,
26958 IX86_BUILTIN_VTESTZPS,
26959 IX86_BUILTIN_VTESTCPS,
26960 IX86_BUILTIN_VTESTNZCPS,
26961 IX86_BUILTIN_VTESTZPD256,
26962 IX86_BUILTIN_VTESTCPD256,
26963 IX86_BUILTIN_VTESTNZCPD256,
26964 IX86_BUILTIN_VTESTZPS256,
26965 IX86_BUILTIN_VTESTCPS256,
26966 IX86_BUILTIN_VTESTNZCPS256,
26967 IX86_BUILTIN_PTESTZ256,
26968 IX86_BUILTIN_PTESTC256,
26969 IX86_BUILTIN_PTESTNZC256,
26971 IX86_BUILTIN_MOVMSKPD256,
26972 IX86_BUILTIN_MOVMSKPS256,
26974 /* AVX2 */
26975 IX86_BUILTIN_MPSADBW256,
26976 IX86_BUILTIN_PABSB256,
26977 IX86_BUILTIN_PABSW256,
26978 IX86_BUILTIN_PABSD256,
26979 IX86_BUILTIN_PACKSSDW256,
26980 IX86_BUILTIN_PACKSSWB256,
26981 IX86_BUILTIN_PACKUSDW256,
26982 IX86_BUILTIN_PACKUSWB256,
26983 IX86_BUILTIN_PADDB256,
26984 IX86_BUILTIN_PADDW256,
26985 IX86_BUILTIN_PADDD256,
26986 IX86_BUILTIN_PADDQ256,
26987 IX86_BUILTIN_PADDSB256,
26988 IX86_BUILTIN_PADDSW256,
26989 IX86_BUILTIN_PADDUSB256,
26990 IX86_BUILTIN_PADDUSW256,
26991 IX86_BUILTIN_PALIGNR256,
26992 IX86_BUILTIN_AND256I,
26993 IX86_BUILTIN_ANDNOT256I,
26994 IX86_BUILTIN_PAVGB256,
26995 IX86_BUILTIN_PAVGW256,
26996 IX86_BUILTIN_PBLENDVB256,
26997 IX86_BUILTIN_PBLENDVW256,
26998 IX86_BUILTIN_PCMPEQB256,
26999 IX86_BUILTIN_PCMPEQW256,
27000 IX86_BUILTIN_PCMPEQD256,
27001 IX86_BUILTIN_PCMPEQQ256,
27002 IX86_BUILTIN_PCMPGTB256,
27003 IX86_BUILTIN_PCMPGTW256,
27004 IX86_BUILTIN_PCMPGTD256,
27005 IX86_BUILTIN_PCMPGTQ256,
27006 IX86_BUILTIN_PHADDW256,
27007 IX86_BUILTIN_PHADDD256,
27008 IX86_BUILTIN_PHADDSW256,
27009 IX86_BUILTIN_PHSUBW256,
27010 IX86_BUILTIN_PHSUBD256,
27011 IX86_BUILTIN_PHSUBSW256,
27012 IX86_BUILTIN_PMADDUBSW256,
27013 IX86_BUILTIN_PMADDWD256,
27014 IX86_BUILTIN_PMAXSB256,
27015 IX86_BUILTIN_PMAXSW256,
27016 IX86_BUILTIN_PMAXSD256,
27017 IX86_BUILTIN_PMAXUB256,
27018 IX86_BUILTIN_PMAXUW256,
27019 IX86_BUILTIN_PMAXUD256,
27020 IX86_BUILTIN_PMINSB256,
27021 IX86_BUILTIN_PMINSW256,
27022 IX86_BUILTIN_PMINSD256,
27023 IX86_BUILTIN_PMINUB256,
27024 IX86_BUILTIN_PMINUW256,
27025 IX86_BUILTIN_PMINUD256,
27026 IX86_BUILTIN_PMOVMSKB256,
27027 IX86_BUILTIN_PMOVSXBW256,
27028 IX86_BUILTIN_PMOVSXBD256,
27029 IX86_BUILTIN_PMOVSXBQ256,
27030 IX86_BUILTIN_PMOVSXWD256,
27031 IX86_BUILTIN_PMOVSXWQ256,
27032 IX86_BUILTIN_PMOVSXDQ256,
27033 IX86_BUILTIN_PMOVZXBW256,
27034 IX86_BUILTIN_PMOVZXBD256,
27035 IX86_BUILTIN_PMOVZXBQ256,
27036 IX86_BUILTIN_PMOVZXWD256,
27037 IX86_BUILTIN_PMOVZXWQ256,
27038 IX86_BUILTIN_PMOVZXDQ256,
27039 IX86_BUILTIN_PMULDQ256,
27040 IX86_BUILTIN_PMULHRSW256,
27041 IX86_BUILTIN_PMULHUW256,
27042 IX86_BUILTIN_PMULHW256,
27043 IX86_BUILTIN_PMULLW256,
27044 IX86_BUILTIN_PMULLD256,
27045 IX86_BUILTIN_PMULUDQ256,
27046 IX86_BUILTIN_POR256,
27047 IX86_BUILTIN_PSADBW256,
27048 IX86_BUILTIN_PSHUFB256,
27049 IX86_BUILTIN_PSHUFD256,
27050 IX86_BUILTIN_PSHUFHW256,
27051 IX86_BUILTIN_PSHUFLW256,
27052 IX86_BUILTIN_PSIGNB256,
27053 IX86_BUILTIN_PSIGNW256,
27054 IX86_BUILTIN_PSIGND256,
27055 IX86_BUILTIN_PSLLDQI256,
27056 IX86_BUILTIN_PSLLWI256,
27057 IX86_BUILTIN_PSLLW256,
27058 IX86_BUILTIN_PSLLDI256,
27059 IX86_BUILTIN_PSLLD256,
27060 IX86_BUILTIN_PSLLQI256,
27061 IX86_BUILTIN_PSLLQ256,
27062 IX86_BUILTIN_PSRAWI256,
27063 IX86_BUILTIN_PSRAW256,
27064 IX86_BUILTIN_PSRADI256,
27065 IX86_BUILTIN_PSRAD256,
27066 IX86_BUILTIN_PSRLDQI256,
27067 IX86_BUILTIN_PSRLWI256,
27068 IX86_BUILTIN_PSRLW256,
27069 IX86_BUILTIN_PSRLDI256,
27070 IX86_BUILTIN_PSRLD256,
27071 IX86_BUILTIN_PSRLQI256,
27072 IX86_BUILTIN_PSRLQ256,
27073 IX86_BUILTIN_PSUBB256,
27074 IX86_BUILTIN_PSUBW256,
27075 IX86_BUILTIN_PSUBD256,
27076 IX86_BUILTIN_PSUBQ256,
27077 IX86_BUILTIN_PSUBSB256,
27078 IX86_BUILTIN_PSUBSW256,
27079 IX86_BUILTIN_PSUBUSB256,
27080 IX86_BUILTIN_PSUBUSW256,
27081 IX86_BUILTIN_PUNPCKHBW256,
27082 IX86_BUILTIN_PUNPCKHWD256,
27083 IX86_BUILTIN_PUNPCKHDQ256,
27084 IX86_BUILTIN_PUNPCKHQDQ256,
27085 IX86_BUILTIN_PUNPCKLBW256,
27086 IX86_BUILTIN_PUNPCKLWD256,
27087 IX86_BUILTIN_PUNPCKLDQ256,
27088 IX86_BUILTIN_PUNPCKLQDQ256,
27089 IX86_BUILTIN_PXOR256,
27090 IX86_BUILTIN_MOVNTDQA256,
27091 IX86_BUILTIN_VBROADCASTSS_PS,
27092 IX86_BUILTIN_VBROADCASTSS_PS256,
27093 IX86_BUILTIN_VBROADCASTSD_PD256,
27094 IX86_BUILTIN_VBROADCASTSI256,
27095 IX86_BUILTIN_PBLENDD256,
27096 IX86_BUILTIN_PBLENDD128,
27097 IX86_BUILTIN_PBROADCASTB256,
27098 IX86_BUILTIN_PBROADCASTW256,
27099 IX86_BUILTIN_PBROADCASTD256,
27100 IX86_BUILTIN_PBROADCASTQ256,
27101 IX86_BUILTIN_PBROADCASTB128,
27102 IX86_BUILTIN_PBROADCASTW128,
27103 IX86_BUILTIN_PBROADCASTD128,
27104 IX86_BUILTIN_PBROADCASTQ128,
27105 IX86_BUILTIN_VPERMVARSI256,
27106 IX86_BUILTIN_VPERMDF256,
27107 IX86_BUILTIN_VPERMVARSF256,
27108 IX86_BUILTIN_VPERMDI256,
27109 IX86_BUILTIN_VPERMTI256,
27110 IX86_BUILTIN_VEXTRACT128I256,
27111 IX86_BUILTIN_VINSERT128I256,
27112 IX86_BUILTIN_MASKLOADD,
27113 IX86_BUILTIN_MASKLOADQ,
27114 IX86_BUILTIN_MASKLOADD256,
27115 IX86_BUILTIN_MASKLOADQ256,
27116 IX86_BUILTIN_MASKSTORED,
27117 IX86_BUILTIN_MASKSTOREQ,
27118 IX86_BUILTIN_MASKSTORED256,
27119 IX86_BUILTIN_MASKSTOREQ256,
27120 IX86_BUILTIN_PSLLVV4DI,
27121 IX86_BUILTIN_PSLLVV2DI,
27122 IX86_BUILTIN_PSLLVV8SI,
27123 IX86_BUILTIN_PSLLVV4SI,
27124 IX86_BUILTIN_PSRAVV8SI,
27125 IX86_BUILTIN_PSRAVV4SI,
27126 IX86_BUILTIN_PSRLVV4DI,
27127 IX86_BUILTIN_PSRLVV2DI,
27128 IX86_BUILTIN_PSRLVV8SI,
27129 IX86_BUILTIN_PSRLVV4SI,
27131 IX86_BUILTIN_GATHERSIV2DF,
27132 IX86_BUILTIN_GATHERSIV4DF,
27133 IX86_BUILTIN_GATHERDIV2DF,
27134 IX86_BUILTIN_GATHERDIV4DF,
27135 IX86_BUILTIN_GATHERSIV4SF,
27136 IX86_BUILTIN_GATHERSIV8SF,
27137 IX86_BUILTIN_GATHERDIV4SF,
27138 IX86_BUILTIN_GATHERDIV8SF,
27139 IX86_BUILTIN_GATHERSIV2DI,
27140 IX86_BUILTIN_GATHERSIV4DI,
27141 IX86_BUILTIN_GATHERDIV2DI,
27142 IX86_BUILTIN_GATHERDIV4DI,
27143 IX86_BUILTIN_GATHERSIV4SI,
27144 IX86_BUILTIN_GATHERSIV8SI,
27145 IX86_BUILTIN_GATHERDIV4SI,
27146 IX86_BUILTIN_GATHERDIV8SI,
27148 /* Alternate 4 element gather for the vectorizer where
27149 all operands are 32-byte wide. */
27150 IX86_BUILTIN_GATHERALTSIV4DF,
27151 IX86_BUILTIN_GATHERALTDIV8SF,
27152 IX86_BUILTIN_GATHERALTSIV4DI,
27153 IX86_BUILTIN_GATHERALTDIV8SI,
27155 /* TFmode support builtins. */
27156 IX86_BUILTIN_INFQ,
27157 IX86_BUILTIN_HUGE_VALQ,
27158 IX86_BUILTIN_FABSQ,
27159 IX86_BUILTIN_COPYSIGNQ,
27161 /* Vectorizer support builtins. */
27162 IX86_BUILTIN_CPYSGNPS,
27163 IX86_BUILTIN_CPYSGNPD,
27164 IX86_BUILTIN_CPYSGNPS256,
27165 IX86_BUILTIN_CPYSGNPD256,
27167 /* FMA4 instructions. */
27168 IX86_BUILTIN_VFMADDSS,
27169 IX86_BUILTIN_VFMADDSD,
27170 IX86_BUILTIN_VFMADDPS,
27171 IX86_BUILTIN_VFMADDPD,
27172 IX86_BUILTIN_VFMADDPS256,
27173 IX86_BUILTIN_VFMADDPD256,
27174 IX86_BUILTIN_VFMADDSUBPS,
27175 IX86_BUILTIN_VFMADDSUBPD,
27176 IX86_BUILTIN_VFMADDSUBPS256,
27177 IX86_BUILTIN_VFMADDSUBPD256,
27179 /* FMA3 instructions. */
27180 IX86_BUILTIN_VFMADDSS3,
27181 IX86_BUILTIN_VFMADDSD3,
27183 /* XOP instructions. */
27184 IX86_BUILTIN_VPCMOV,
27185 IX86_BUILTIN_VPCMOV_V2DI,
27186 IX86_BUILTIN_VPCMOV_V4SI,
27187 IX86_BUILTIN_VPCMOV_V8HI,
27188 IX86_BUILTIN_VPCMOV_V16QI,
27189 IX86_BUILTIN_VPCMOV_V4SF,
27190 IX86_BUILTIN_VPCMOV_V2DF,
27191 IX86_BUILTIN_VPCMOV256,
27192 IX86_BUILTIN_VPCMOV_V4DI256,
27193 IX86_BUILTIN_VPCMOV_V8SI256,
27194 IX86_BUILTIN_VPCMOV_V16HI256,
27195 IX86_BUILTIN_VPCMOV_V32QI256,
27196 IX86_BUILTIN_VPCMOV_V8SF256,
27197 IX86_BUILTIN_VPCMOV_V4DF256,
27199 IX86_BUILTIN_VPPERM,
27201 IX86_BUILTIN_VPMACSSWW,
27202 IX86_BUILTIN_VPMACSWW,
27203 IX86_BUILTIN_VPMACSSWD,
27204 IX86_BUILTIN_VPMACSWD,
27205 IX86_BUILTIN_VPMACSSDD,
27206 IX86_BUILTIN_VPMACSDD,
27207 IX86_BUILTIN_VPMACSSDQL,
27208 IX86_BUILTIN_VPMACSSDQH,
27209 IX86_BUILTIN_VPMACSDQL,
27210 IX86_BUILTIN_VPMACSDQH,
27211 IX86_BUILTIN_VPMADCSSWD,
27212 IX86_BUILTIN_VPMADCSWD,
27214 IX86_BUILTIN_VPHADDBW,
27215 IX86_BUILTIN_VPHADDBD,
27216 IX86_BUILTIN_VPHADDBQ,
27217 IX86_BUILTIN_VPHADDWD,
27218 IX86_BUILTIN_VPHADDWQ,
27219 IX86_BUILTIN_VPHADDDQ,
27220 IX86_BUILTIN_VPHADDUBW,
27221 IX86_BUILTIN_VPHADDUBD,
27222 IX86_BUILTIN_VPHADDUBQ,
27223 IX86_BUILTIN_VPHADDUWD,
27224 IX86_BUILTIN_VPHADDUWQ,
27225 IX86_BUILTIN_VPHADDUDQ,
27226 IX86_BUILTIN_VPHSUBBW,
27227 IX86_BUILTIN_VPHSUBWD,
27228 IX86_BUILTIN_VPHSUBDQ,
27230 IX86_BUILTIN_VPROTB,
27231 IX86_BUILTIN_VPROTW,
27232 IX86_BUILTIN_VPROTD,
27233 IX86_BUILTIN_VPROTQ,
27234 IX86_BUILTIN_VPROTB_IMM,
27235 IX86_BUILTIN_VPROTW_IMM,
27236 IX86_BUILTIN_VPROTD_IMM,
27237 IX86_BUILTIN_VPROTQ_IMM,
27239 IX86_BUILTIN_VPSHLB,
27240 IX86_BUILTIN_VPSHLW,
27241 IX86_BUILTIN_VPSHLD,
27242 IX86_BUILTIN_VPSHLQ,
27243 IX86_BUILTIN_VPSHAB,
27244 IX86_BUILTIN_VPSHAW,
27245 IX86_BUILTIN_VPSHAD,
27246 IX86_BUILTIN_VPSHAQ,
27248 IX86_BUILTIN_VFRCZSS,
27249 IX86_BUILTIN_VFRCZSD,
27250 IX86_BUILTIN_VFRCZPS,
27251 IX86_BUILTIN_VFRCZPD,
27252 IX86_BUILTIN_VFRCZPS256,
27253 IX86_BUILTIN_VFRCZPD256,
27255 IX86_BUILTIN_VPCOMEQUB,
27256 IX86_BUILTIN_VPCOMNEUB,
27257 IX86_BUILTIN_VPCOMLTUB,
27258 IX86_BUILTIN_VPCOMLEUB,
27259 IX86_BUILTIN_VPCOMGTUB,
27260 IX86_BUILTIN_VPCOMGEUB,
27261 IX86_BUILTIN_VPCOMFALSEUB,
27262 IX86_BUILTIN_VPCOMTRUEUB,
27264 IX86_BUILTIN_VPCOMEQUW,
27265 IX86_BUILTIN_VPCOMNEUW,
27266 IX86_BUILTIN_VPCOMLTUW,
27267 IX86_BUILTIN_VPCOMLEUW,
27268 IX86_BUILTIN_VPCOMGTUW,
27269 IX86_BUILTIN_VPCOMGEUW,
27270 IX86_BUILTIN_VPCOMFALSEUW,
27271 IX86_BUILTIN_VPCOMTRUEUW,
27273 IX86_BUILTIN_VPCOMEQUD,
27274 IX86_BUILTIN_VPCOMNEUD,
27275 IX86_BUILTIN_VPCOMLTUD,
27276 IX86_BUILTIN_VPCOMLEUD,
27277 IX86_BUILTIN_VPCOMGTUD,
27278 IX86_BUILTIN_VPCOMGEUD,
27279 IX86_BUILTIN_VPCOMFALSEUD,
27280 IX86_BUILTIN_VPCOMTRUEUD,
27282 IX86_BUILTIN_VPCOMEQUQ,
27283 IX86_BUILTIN_VPCOMNEUQ,
27284 IX86_BUILTIN_VPCOMLTUQ,
27285 IX86_BUILTIN_VPCOMLEUQ,
27286 IX86_BUILTIN_VPCOMGTUQ,
27287 IX86_BUILTIN_VPCOMGEUQ,
27288 IX86_BUILTIN_VPCOMFALSEUQ,
27289 IX86_BUILTIN_VPCOMTRUEUQ,
27291 IX86_BUILTIN_VPCOMEQB,
27292 IX86_BUILTIN_VPCOMNEB,
27293 IX86_BUILTIN_VPCOMLTB,
27294 IX86_BUILTIN_VPCOMLEB,
27295 IX86_BUILTIN_VPCOMGTB,
27296 IX86_BUILTIN_VPCOMGEB,
27297 IX86_BUILTIN_VPCOMFALSEB,
27298 IX86_BUILTIN_VPCOMTRUEB,
27300 IX86_BUILTIN_VPCOMEQW,
27301 IX86_BUILTIN_VPCOMNEW,
27302 IX86_BUILTIN_VPCOMLTW,
27303 IX86_BUILTIN_VPCOMLEW,
27304 IX86_BUILTIN_VPCOMGTW,
27305 IX86_BUILTIN_VPCOMGEW,
27306 IX86_BUILTIN_VPCOMFALSEW,
27307 IX86_BUILTIN_VPCOMTRUEW,
27309 IX86_BUILTIN_VPCOMEQD,
27310 IX86_BUILTIN_VPCOMNED,
27311 IX86_BUILTIN_VPCOMLTD,
27312 IX86_BUILTIN_VPCOMLED,
27313 IX86_BUILTIN_VPCOMGTD,
27314 IX86_BUILTIN_VPCOMGED,
27315 IX86_BUILTIN_VPCOMFALSED,
27316 IX86_BUILTIN_VPCOMTRUED,
27318 IX86_BUILTIN_VPCOMEQQ,
27319 IX86_BUILTIN_VPCOMNEQ,
27320 IX86_BUILTIN_VPCOMLTQ,
27321 IX86_BUILTIN_VPCOMLEQ,
27322 IX86_BUILTIN_VPCOMGTQ,
27323 IX86_BUILTIN_VPCOMGEQ,
27324 IX86_BUILTIN_VPCOMFALSEQ,
27325 IX86_BUILTIN_VPCOMTRUEQ,
27327 /* LWP instructions. */
27328 IX86_BUILTIN_LLWPCB,
27329 IX86_BUILTIN_SLWPCB,
27330 IX86_BUILTIN_LWPVAL32,
27331 IX86_BUILTIN_LWPVAL64,
27332 IX86_BUILTIN_LWPINS32,
27333 IX86_BUILTIN_LWPINS64,
27335 IX86_BUILTIN_CLZS,
27337 /* RTM */
27338 IX86_BUILTIN_XBEGIN,
27339 IX86_BUILTIN_XEND,
27340 IX86_BUILTIN_XABORT,
27341 IX86_BUILTIN_XTEST,
27343 /* BMI instructions. */
27344 IX86_BUILTIN_BEXTR32,
27345 IX86_BUILTIN_BEXTR64,
27346 IX86_BUILTIN_CTZS,
27348 /* TBM instructions. */
27349 IX86_BUILTIN_BEXTRI32,
27350 IX86_BUILTIN_BEXTRI64,
27352 /* BMI2 instructions. */
27353 IX86_BUILTIN_BZHI32,
27354 IX86_BUILTIN_BZHI64,
27355 IX86_BUILTIN_PDEP32,
27356 IX86_BUILTIN_PDEP64,
27357 IX86_BUILTIN_PEXT32,
27358 IX86_BUILTIN_PEXT64,
27360 /* ADX instructions. */
27361 IX86_BUILTIN_ADDCARRYX32,
27362 IX86_BUILTIN_ADDCARRYX64,
27364 /* FSGSBASE instructions. */
27365 IX86_BUILTIN_RDFSBASE32,
27366 IX86_BUILTIN_RDFSBASE64,
27367 IX86_BUILTIN_RDGSBASE32,
27368 IX86_BUILTIN_RDGSBASE64,
27369 IX86_BUILTIN_WRFSBASE32,
27370 IX86_BUILTIN_WRFSBASE64,
27371 IX86_BUILTIN_WRGSBASE32,
27372 IX86_BUILTIN_WRGSBASE64,
27374 /* RDRND instructions. */
27375 IX86_BUILTIN_RDRAND16_STEP,
27376 IX86_BUILTIN_RDRAND32_STEP,
27377 IX86_BUILTIN_RDRAND64_STEP,
27379 /* RDSEED instructions. */
27380 IX86_BUILTIN_RDSEED16_STEP,
27381 IX86_BUILTIN_RDSEED32_STEP,
27382 IX86_BUILTIN_RDSEED64_STEP,
27384 /* F16C instructions. */
27385 IX86_BUILTIN_CVTPH2PS,
27386 IX86_BUILTIN_CVTPH2PS256,
27387 IX86_BUILTIN_CVTPS2PH,
27388 IX86_BUILTIN_CVTPS2PH256,
27390 /* CFString built-in for darwin */
27391 IX86_BUILTIN_CFSTRING,
27393 /* Builtins to get CPU type and supported features. */
27394 IX86_BUILTIN_CPU_INIT,
27395 IX86_BUILTIN_CPU_IS,
27396 IX86_BUILTIN_CPU_SUPPORTS,
27398 IX86_BUILTIN_MAX
27399 };
27401 /* Table for the ix86 builtin decls. */
27404 /* Table of all of the builtin functions that are possible with different ISA's
27405 but are waiting to be built until a function is declared to use that
27406 ISA. */
27413 };
27418 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
27419 of which isa_flags to use in the ix86_builtins_isa array. Stores the
27420 function decl in the ix86_builtins array. Returns the function decl or
27421 NULL_TREE, if the builtin was not added.
27423 If the front end has a special hook for builtin functions, delay adding
27424 builtin functions that aren't in the current ISA until the ISA is changed
27425 with function specific optimization. Doing so, can save about 300K for the
27426 default compiler. When the builtin is expanded, check at that time whether
27427 it is valid.
27429 If the front end doesn't have a special hook, record all builtins, even if
27430 it isn't an instruction set in the current ISA in case the user uses
27431 function specific options for a different ISA, so that we don't get scope
27432 errors if a builtin is added in the middle of a function scope. */
27438 {
27442 {
27451 {
27457 }
27458 else
27459 {
27465 }
27466 }
27469 }
27471 /* Like def_builtin, but also marks the function decl "const". */
27476 {
27480 else
27484 }
27486 /* Add any new builtin functions for a given ISA that may not have been
27487 declared. This saves a bit of space compared to adding all of the
27488 declarations to the tree, even if we didn't use them. */
27490 static void
27492 {
27496 {
27499 {
27502 /* Don't define the builtin again. */
27508 NULL_TREE);
27513 }
27514 }
27515 }
27517 /* Bits for builtin_description.flag. */
27519 /* Set when we don't support the comparison natively, and should
27520 swap_comparison in order to support it. */
27521 #define BUILTIN_DESC_SWAP_OPERANDS 1
27523 struct builtin_description
27524 {
27531 };
27534 {
27535 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
27536 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
27537 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
27538 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
27539 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
27540 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
27541 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
27542 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
27543 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
27544 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
27545 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
27546 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
27547 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
27548 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
27549 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
27550 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
27551 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
27552 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
27553 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
27554 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
27555 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
27556 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
27557 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
27558 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
27559 };
27562 {
27563 /* SSE4.2 */
27564 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
27565 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
27566 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
27567 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
27568 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
27569 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
27570 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
27571 };
27574 {
27575 /* SSE4.2 */
27576 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
27577 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
27578 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
27579 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
27580 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
27581 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
27582 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
27583 };
27585 /* Special builtins with variable number of arguments. */
27587 {
27588 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtsc", IX86_BUILTIN_RDTSC, UNKNOWN, (int) UINT64_FTYPE_VOID },
27589 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
27590 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
27592 /* MMX */
27593 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
27595 /* 3DNow! */
27596 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
27598 /* FXSR, XSAVE and XSAVEOPT */
27599 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxsave", IX86_BUILTIN_FXSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID },
27600 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxrstor", IX86_BUILTIN_FXRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID },
27601 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xsave", IX86_BUILTIN_XSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27602 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xrstor", IX86_BUILTIN_XRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27603 { OPTION_MASK_ISA_XSAVEOPT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt", IX86_BUILTIN_XSAVEOPT, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27605 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxsave64", IX86_BUILTIN_FXSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID },
27606 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxrstor64", IX86_BUILTIN_FXRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID },
27607 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsave64", IX86_BUILTIN_XSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27608 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xrstor64", IX86_BUILTIN_XRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27609 { OPTION_MASK_ISA_XSAVEOPT | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt64", IX86_BUILTIN_XSAVEOPT64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
27611 /* SSE */
27612 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storeups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
27613 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
27614 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
27616 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
27617 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
27618 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
27619 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
27621 /* SSE or 3DNow!A */
27622 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
27623 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntq, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PULONGLONG_ULONGLONG },
27625 /* SSE2 */
27626 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
27627 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
27628 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storeupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
27629 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storedqu, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
27630 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
27631 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
27632 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
27633 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
27634 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
27635 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loaddqu, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
27637 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
27638 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
27640 /* SSE3 */
27641 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
27643 /* SSE4.1 */
27644 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
27646 /* SSE4A */
27647 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
27648 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
27650 /* AVX */
27651 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
27652 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, "__builtin_ia32_vzeroupper", IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
27654 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4sf, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
27655 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4df, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
27656 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv8sf, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
27657 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v4df, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
27658 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v8sf, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
27660 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
27661 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
27662 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
27663 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
27664 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loaddqu256, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
27665 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storedqu256, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
27666 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
27668 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
27669 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
27670 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
27672 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI },
27673 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI },
27674 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI },
27675 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI },
27676 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF },
27677 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF },
27678 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF },
27679 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF },
27681 /* AVX2 */
27682 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_movntdqa, "__builtin_ia32_movntdqa256", IX86_BUILTIN_MOVNTDQA256, UNKNOWN, (int) V4DI_FTYPE_PV4DI },
27683 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd, "__builtin_ia32_maskloadd", IX86_BUILTIN_MASKLOADD, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI },
27684 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq, "__builtin_ia32_maskloadq", IX86_BUILTIN_MASKLOADQ, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI },
27685 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd256, "__builtin_ia32_maskloadd256", IX86_BUILTIN_MASKLOADD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI },
27686 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq256, "__builtin_ia32_maskloadq256", IX86_BUILTIN_MASKLOADQ256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI },
27687 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored, "__builtin_ia32_maskstored", IX86_BUILTIN_MASKSTORED, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_V4SI },
27688 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq, "__builtin_ia32_maskstoreq", IX86_BUILTIN_MASKSTOREQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_V2DI },
27689 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored256, "__builtin_ia32_maskstored256", IX86_BUILTIN_MASKSTORED256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_V8SI },
27690 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq256, "__builtin_ia32_maskstoreq256", IX86_BUILTIN_MASKSTOREQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_V4DI },
27692 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID },
27693 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID },
27694 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvalsi3, "__builtin_ia32_lwpval32", IX86_BUILTIN_LWPVAL32, UNKNOWN, (int) VOID_FTYPE_UINT_UINT_UINT },
27695 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvaldi3, "__builtin_ia32_lwpval64", IX86_BUILTIN_LWPVAL64, UNKNOWN, (int) VOID_FTYPE_UINT64_UINT_UINT },
27696 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT },
27697 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT },
27699 /* FSGSBASE */
27700 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
27701 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
27702 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
27703 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
27704 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
27705 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
27706 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
27707 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
27709 /* RTM */
27710 { OPTION_MASK_ISA_RTM, CODE_FOR_xbegin, "__builtin_ia32_xbegin", IX86_BUILTIN_XBEGIN, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
27711 { OPTION_MASK_ISA_RTM, CODE_FOR_xend, "__builtin_ia32_xend", IX86_BUILTIN_XEND, UNKNOWN, (int) VOID_FTYPE_VOID },
27712 { OPTION_MASK_ISA_RTM, CODE_FOR_xtest, "__builtin_ia32_xtest", IX86_BUILTIN_XTEST, UNKNOWN, (int) INT_FTYPE_VOID },
27713 };
27715 /* Builtins with variable number of arguments. */
27717 {
27718 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_bsr, "__builtin_ia32_bsrsi", IX86_BUILTIN_BSRSI, UNKNOWN, (int) INT_FTYPE_INT },
27719 { OPTION_MASK_ISA_64BIT, CODE_FOR_bsr_rex64, "__builtin_ia32_bsrdi", IX86_BUILTIN_BSRDI, UNKNOWN, (int) INT64_FTYPE_INT64 },
27720 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdpmc", IX86_BUILTIN_RDPMC, UNKNOWN, (int) UINT64_FTYPE_INT },
27721 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlqi3, "__builtin_ia32_rolqi", IX86_BUILTIN_ROLQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
27722 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlhi3, "__builtin_ia32_rolhi", IX86_BUILTIN_ROLHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
27723 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrqi3, "__builtin_ia32_rorqi", IX86_BUILTIN_RORQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
27724 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrhi3, "__builtin_ia32_rorhi", IX86_BUILTIN_RORHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
27726 /* MMX */
27727 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27728 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27729 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27730 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27731 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27732 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27734 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27735 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27736 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27737 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27738 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27739 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27740 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27741 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27743 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27744 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27746 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27747 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27748 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27749 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27751 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27752 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27753 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27754 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27755 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27756 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27758 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27759 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27760 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
27761 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27762 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
27763 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
27765 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
27766 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
27767 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
27769 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
27771 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
27772 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
27773 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
27774 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
27775 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
27776 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
27778 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
27779 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
27780 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
27781 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
27782 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
27783 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
27785 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
27786 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
27787 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
27788 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
27790 /* 3DNow! */
27791 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
27792 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
27793 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
27794 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
27796 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27797 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27798 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27799 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
27800 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
27801 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
27802 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27803 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27804 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27805 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27806 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27807 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27808 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27809 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27810 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27812 /* 3DNow!A */
27813 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
27814 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
27815 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
27816 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
27817 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27818 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
27820 /* SSE */
27821 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
27822 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
27823 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
27824 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
27825 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
27826 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
27827 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
27828 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
27829 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
27830 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
27831 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
27832 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
27834 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
27836 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27837 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27838 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27839 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27840 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27841 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27842 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27843 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27845 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
27846 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
27847 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
27848 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
27849 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
27850 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
27851 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
27852 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
27853 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
27854 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
27855 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
27856 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
27857 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
27858 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
27859 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
27860 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
27861 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
27862 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
27863 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
27864 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
27866 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27867 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27868 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27869 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27871 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27872 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27873 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27874 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27876 { OPTION_MASK_ISA_SSE, CODE_FOR_copysignv4sf3, "__builtin_ia32_copysignps", IX86_BUILTIN_CPYSGNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27878 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27879 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27880 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27881 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_highv4sf, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27882 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_lowv4sf, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
27884 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
27885 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
27886 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
27888 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
27890 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
27891 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
27892 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
27894 { OPTION_MASK_ISA_SSE, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
27895 { OPTION_MASK_ISA_SSE, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
27897 /* SSE MMX or 3Dnow!A */
27898 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27899 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27900 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27902 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27903 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27904 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
27905 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
27907 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
27908 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
27910 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
27912 /* SSE2 */
27913 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
27915 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
27916 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
27917 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
27918 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
27919 { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
27921 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
27922 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
27923 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
27924 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
27925 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
27927 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
27929 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
27930 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
27931 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
27932 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
27934 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2dq, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
27935 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
27936 { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
27938 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27939 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27940 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27941 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27942 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27943 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27944 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27945 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27947 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
27948 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
27949 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
27950 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
27951 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
27952 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
27953 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
27954 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
27955 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
27956 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
27957 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
27958 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
27959 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
27960 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
27961 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
27962 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
27963 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
27964 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
27965 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
27966 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
27968 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27969 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27970 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27971 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27973 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27974 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27975 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27976 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27978 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysignv2df3, "__builtin_ia32_copysignpd", IX86_BUILTIN_CPYSGNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27980 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27981 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2df, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27982 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2df, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
27984 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
27986 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
27987 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
27988 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
27989 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
27990 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
27991 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
27992 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
27993 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
27995 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
27996 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
27997 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
27998 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
27999 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28000 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28001 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28002 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28004 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28005 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
28007 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28008 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28009 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28010 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28012 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28013 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28015 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28016 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28017 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28018 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28019 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28020 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28022 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28023 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28024 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28025 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28027 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv16qi, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28028 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv8hi, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28029 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv4si, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28030 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2di, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28031 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv16qi, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28032 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv8hi, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28033 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv4si, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28034 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2di, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28036 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
28037 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
28038 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
28040 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28041 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
28043 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
28044 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_widen_umult_even_v4si, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
28046 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
28048 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
28049 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
28050 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
28051 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
28053 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlv1ti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
28054 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28055 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28056 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
28057 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28058 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28059 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
28061 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrv1ti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
28062 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28063 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28064 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
28065 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28066 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28067 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
28069 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28070 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28071 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28072 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28074 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
28075 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
28076 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
28078 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
28080 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28082 /* SSE2 MMX */
28083 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
28084 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
28086 /* SSE3 */
28087 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
28088 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28090 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28091 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28092 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28093 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28094 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28095 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28097 /* SSSE3 */
28098 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
28099 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
28100 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28101 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
28102 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
28103 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
28105 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28106 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28107 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28108 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28109 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28110 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28111 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28112 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28113 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28114 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28115 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28116 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28117 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
28118 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
28119 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28120 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28121 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28122 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28123 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28124 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28125 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28126 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28127 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28128 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28130 /* SSSE3. */
28131 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_CONVERT },
28132 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_INT_CONVERT },
28134 /* SSE4.1 */
28135 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28136 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28137 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
28138 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
28139 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28140 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28141 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28142 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
28143 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
28144 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
28146 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
28147 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
28148 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
28149 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
28150 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
28151 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
28152 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
28153 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
28154 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
28155 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
28156 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
28157 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
28158 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28160 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
28161 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28162 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28163 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28164 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28165 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28166 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28167 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28168 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28169 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28170 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
28171 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28173 /* SSE4.1 */
28174 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
28175 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
28176 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28177 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28179 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_floorpd", IX86_BUILTIN_FLOORPD, (enum rtx_code) ROUND_FLOOR, (int) V2DF_FTYPE_V2DF_ROUND },
28180 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_ceilpd", IX86_BUILTIN_CEILPD, (enum rtx_code) ROUND_CEIL, (int) V2DF_FTYPE_V2DF_ROUND },
28181 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
28182 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
28184 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
28185 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
28187 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
28188 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
28190 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
28191 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
28192 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
28193 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
28195 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
28196 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
28198 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28199 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28201 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28202 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28203 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28205 /* SSE4.2 */
28206 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28207 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
28208 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
28209 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28210 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28212 /* SSE4A */
28213 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
28214 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
28215 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
28216 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28218 /* AES */
28219 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
28220 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28222 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28223 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28224 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28225 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28227 /* PCLMUL */
28228 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
28230 /* AVX */
28231 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28232 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28233 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28234 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28235 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28236 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28237 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28238 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28239 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28240 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28241 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28242 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28243 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28244 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28245 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28246 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28247 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28248 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28249 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28250 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28251 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28252 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28253 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28254 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28255 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28256 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28258 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
28259 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
28260 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
28261 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
28263 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28264 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28265 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
28266 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
28267 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28268 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28269 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28270 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28271 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28272 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28273 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28274 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28275 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28276 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
28277 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
28278 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
28279 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
28280 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
28281 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
28282 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2dq256, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28283 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
28284 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
28285 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
28286 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28287 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28288 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28289 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
28290 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
28291 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
28292 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28293 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
28294 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
28295 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
28296 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
28298 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28299 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28300 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28302 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28303 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28304 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28305 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28306 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28308 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28310 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28311 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
28313 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_floorpd256", IX86_BUILTIN_FLOORPD256, (enum rtx_code) ROUND_FLOOR, (int) V4DF_FTYPE_V4DF_ROUND },
28314 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_ceilpd256", IX86_BUILTIN_CEILPD256, (enum rtx_code) ROUND_CEIL, (int) V4DF_FTYPE_V4DF_ROUND },
28315 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_truncpd256", IX86_BUILTIN_TRUNCPD256, (enum rtx_code) ROUND_TRUNC, (int) V4DF_FTYPE_V4DF_ROUND },
28316 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
28318 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28319 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
28321 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
28322 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
28324 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
28325 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
28326 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
28327 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
28329 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
28330 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
28332 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28333 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28335 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28336 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28337 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28338 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28340 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
28341 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
28342 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
28343 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
28344 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
28345 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
28347 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28348 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28349 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28350 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28351 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28352 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28353 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28354 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28355 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28356 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28357 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28358 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28359 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28360 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28361 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28363 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
28364 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
28366 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28367 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28369 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
28371 /* AVX2 */
28372 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
28373 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
28374 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv16hi2, "__builtin_ia32_pabsw256", IX86_BUILTIN_PABSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI },
28375 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv8si2, "__builtin_ia32_pabsd256", IX86_BUILTIN_PABSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI },
28376 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packssdw, "__builtin_ia32_packssdw256", IX86_BUILTIN_PACKSSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
28377 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packsswb, "__builtin_ia32_packsswb256", IX86_BUILTIN_PACKSSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
28378 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packusdw, "__builtin_ia32_packusdw256", IX86_BUILTIN_PACKUSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
28379 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packuswb, "__builtin_ia32_packuswb256", IX86_BUILTIN_PACKUSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
28380 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv32qi3, "__builtin_ia32_paddb256", IX86_BUILTIN_PADDB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28381 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv16hi3, "__builtin_ia32_paddw256", IX86_BUILTIN_PADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28382 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv8si3, "__builtin_ia32_paddd256", IX86_BUILTIN_PADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28383 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv4di3, "__builtin_ia32_paddq256", IX86_BUILTIN_PADDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28384 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv32qi3, "__builtin_ia32_paddsb256", IX86_BUILTIN_PADDSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28385 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28386 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28387 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28388 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
28389 { OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28390 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28391 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28392 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv16hi3, "__builtin_ia32_pavgw256", IX86_BUILTIN_PAVGW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28393 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendvb, "__builtin_ia32_pblendvb256", IX86_BUILTIN_PBLENDVB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI },
28394 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendw, "__builtin_ia32_pblendw256", IX86_BUILTIN_PBLENDVW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_INT },
28395 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv32qi3, "__builtin_ia32_pcmpeqb256", IX86_BUILTIN_PCMPEQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28396 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv16hi3, "__builtin_ia32_pcmpeqw256", IX86_BUILTIN_PCMPEQW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28397 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv8si3, "__builtin_ia32_pcmpeqd256", IX86_BUILTIN_PCMPEQD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28398 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv4di3, "__builtin_ia32_pcmpeqq256", IX86_BUILTIN_PCMPEQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28399 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv32qi3, "__builtin_ia32_pcmpgtb256", IX86_BUILTIN_PCMPGTB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28400 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv16hi3, "__builtin_ia32_pcmpgtw256", IX86_BUILTIN_PCMPGTW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28401 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv8si3, "__builtin_ia32_pcmpgtd256", IX86_BUILTIN_PCMPGTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28402 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv4di3, "__builtin_ia32_pcmpgtq256", IX86_BUILTIN_PCMPGTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28403 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddwv16hi3, "__builtin_ia32_phaddw256", IX86_BUILTIN_PHADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28404 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phadddv8si3, "__builtin_ia32_phaddd256", IX86_BUILTIN_PHADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28405 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddswv16hi3, "__builtin_ia32_phaddsw256", IX86_BUILTIN_PHADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28406 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubwv16hi3, "__builtin_ia32_phsubw256", IX86_BUILTIN_PHSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28407 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubdv8si3, "__builtin_ia32_phsubd256", IX86_BUILTIN_PHSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28408 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubswv16hi3, "__builtin_ia32_phsubsw256", IX86_BUILTIN_PHSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28409 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddubsw256, "__builtin_ia32_pmaddubsw256", IX86_BUILTIN_PMADDUBSW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
28410 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddwd, "__builtin_ia32_pmaddwd256", IX86_BUILTIN_PMADDWD256, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI },
28411 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv32qi3, "__builtin_ia32_pmaxsb256", IX86_BUILTIN_PMAXSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28412 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv16hi3, "__builtin_ia32_pmaxsw256", IX86_BUILTIN_PMAXSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28413 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv8si3 , "__builtin_ia32_pmaxsd256", IX86_BUILTIN_PMAXSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28414 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv32qi3, "__builtin_ia32_pmaxub256", IX86_BUILTIN_PMAXUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28415 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv16hi3, "__builtin_ia32_pmaxuw256", IX86_BUILTIN_PMAXUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28416 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv8si3 , "__builtin_ia32_pmaxud256", IX86_BUILTIN_PMAXUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28417 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv32qi3, "__builtin_ia32_pminsb256", IX86_BUILTIN_PMINSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28418 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv16hi3, "__builtin_ia32_pminsw256", IX86_BUILTIN_PMINSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28419 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv8si3 , "__builtin_ia32_pminsd256", IX86_BUILTIN_PMINSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28420 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv32qi3, "__builtin_ia32_pminub256", IX86_BUILTIN_PMINUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28421 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv16hi3, "__builtin_ia32_pminuw256", IX86_BUILTIN_PMINUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28422 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv8si3 , "__builtin_ia32_pminud256", IX86_BUILTIN_PMINUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28423 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmovmskb, "__builtin_ia32_pmovmskb256", IX86_BUILTIN_PMOVMSKB256, UNKNOWN, (int) INT_FTYPE_V32QI },
28424 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv16qiv16hi2, "__builtin_ia32_pmovsxbw256", IX86_BUILTIN_PMOVSXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
28425 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8qiv8si2 , "__builtin_ia32_pmovsxbd256", IX86_BUILTIN_PMOVSXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
28426 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4qiv4di2 , "__builtin_ia32_pmovsxbq256", IX86_BUILTIN_PMOVSXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
28427 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8hiv8si2 , "__builtin_ia32_pmovsxwd256", IX86_BUILTIN_PMOVSXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
28428 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4hiv4di2 , "__builtin_ia32_pmovsxwq256", IX86_BUILTIN_PMOVSXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
28429 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4siv4di2 , "__builtin_ia32_pmovsxdq256", IX86_BUILTIN_PMOVSXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
28430 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv16qiv16hi2, "__builtin_ia32_pmovzxbw256", IX86_BUILTIN_PMOVZXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
28431 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8qiv8si2 , "__builtin_ia32_pmovzxbd256", IX86_BUILTIN_PMOVZXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
28432 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4qiv4di2 , "__builtin_ia32_pmovzxbq256", IX86_BUILTIN_PMOVZXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
28433 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
28434 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
28435 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
28436 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_smult_even_v8si, "__builtin_ia32_pmuldq256", IX86_BUILTIN_PMULDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
28437 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28438 { OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28439 { OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28440 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28441 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28442 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_umult_even_v8si, "__builtin_ia32_pmuludq256", IX86_BUILTIN_PMULUDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
28443 { OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28444 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
28445 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28446 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufdv3, "__builtin_ia32_pshufd256", IX86_BUILTIN_PSHUFD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT },
28447 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufhwv3, "__builtin_ia32_pshufhw256", IX86_BUILTIN_PSHUFHW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
28448 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshuflwv3, "__builtin_ia32_pshuflw256", IX86_BUILTIN_PSHUFLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
28449 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28450 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28451 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28452 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
28453 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28454 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28455 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28456 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28457 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
28458 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
28459 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28460 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28461 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28462 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28463 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
28464 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28465 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28466 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28467 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrld256", IX86_BUILTIN_PSRLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28468 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlqi256", IX86_BUILTIN_PSRLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
28469 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlq256", IX86_BUILTIN_PSRLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
28470 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv32qi3, "__builtin_ia32_psubb256", IX86_BUILTIN_PSUBB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28471 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv16hi3, "__builtin_ia32_psubw256", IX86_BUILTIN_PSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28472 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv8si3, "__builtin_ia32_psubd256", IX86_BUILTIN_PSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28473 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv4di3, "__builtin_ia32_psubq256", IX86_BUILTIN_PSUBQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28474 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv32qi3, "__builtin_ia32_psubsb256", IX86_BUILTIN_PSUBSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28475 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv16hi3, "__builtin_ia32_psubsw256", IX86_BUILTIN_PSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28476 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv32qi3, "__builtin_ia32_psubusb256", IX86_BUILTIN_PSUBUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28477 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv16hi3, "__builtin_ia32_psubusw256", IX86_BUILTIN_PSUBUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28478 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv32qi, "__builtin_ia32_punpckhbw256", IX86_BUILTIN_PUNPCKHBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28479 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv16hi, "__builtin_ia32_punpckhwd256", IX86_BUILTIN_PUNPCKHWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28480 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv8si, "__builtin_ia32_punpckhdq256", IX86_BUILTIN_PUNPCKHDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28481 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv4di, "__builtin_ia32_punpckhqdq256", IX86_BUILTIN_PUNPCKHQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28482 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv32qi, "__builtin_ia32_punpcklbw256", IX86_BUILTIN_PUNPCKLBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28483 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv16hi, "__builtin_ia32_punpcklwd256", IX86_BUILTIN_PUNPCKLWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28484 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv8si, "__builtin_ia32_punpckldq256", IX86_BUILTIN_PUNPCKLDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28485 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv4di, "__builtin_ia32_punpcklqdq256", IX86_BUILTIN_PUNPCKLQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28486 { OPTION_MASK_ISA_AVX2, CODE_FOR_xorv4di3, "__builtin_ia32_pxor256", IX86_BUILTIN_PXOR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28487 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4sf, "__builtin_ia32_vbroadcastss_ps", IX86_BUILTIN_VBROADCASTSS_PS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28488 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv8sf, "__builtin_ia32_vbroadcastss_ps256", IX86_BUILTIN_VBROADCASTSS_PS256, UNKNOWN, (int) V8SF_FTYPE_V4SF },
28489 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4df, "__builtin_ia32_vbroadcastsd_pd256", IX86_BUILTIN_VBROADCASTSD_PD256, UNKNOWN, (int) V4DF_FTYPE_V2DF },
28490 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vbroadcasti128_v4di, "__builtin_ia32_vbroadcastsi256", IX86_BUILTIN_VBROADCASTSI256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
28491 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv4si, "__builtin_ia32_pblendd128", IX86_BUILTIN_PBLENDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
28492 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv8si, "__builtin_ia32_pblendd256", IX86_BUILTIN_PBLENDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
28493 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv32qi, "__builtin_ia32_pbroadcastb256", IX86_BUILTIN_PBROADCASTB256, UNKNOWN, (int) V32QI_FTYPE_V16QI },
28494 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16hi, "__builtin_ia32_pbroadcastw256", IX86_BUILTIN_PBROADCASTW256, UNKNOWN, (int) V16HI_FTYPE_V8HI },
28495 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8si, "__builtin_ia32_pbroadcastd256", IX86_BUILTIN_PBROADCASTD256, UNKNOWN, (int) V8SI_FTYPE_V4SI },
28496 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4di, "__builtin_ia32_pbroadcastq256", IX86_BUILTIN_PBROADCASTQ256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
28497 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16qi, "__builtin_ia32_pbroadcastb128", IX86_BUILTIN_PBROADCASTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
28498 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8hi, "__builtin_ia32_pbroadcastw128", IX86_BUILTIN_PBROADCASTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28499 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
28500 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28501 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28502 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
28503 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28504 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
28505 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
28506 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_extracti128, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
28507 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_inserti128, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
28508 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28509 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28510 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28511 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28512 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28513 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28514 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28515 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv2di, "__builtin_ia32_psrlv2di", IX86_BUILTIN_PSRLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28516 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv8si, "__builtin_ia32_psrlv8si", IX86_BUILTIN_PSRLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28517 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4si, "__builtin_ia32_psrlv4si", IX86_BUILTIN_PSRLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28519 { OPTION_MASK_ISA_LZCNT, CODE_FOR_clzhi2_lzcnt, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
28521 /* BMI */
28522 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28523 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28524 { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
28526 /* TBM */
28527 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28528 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28530 /* F16C */
28531 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI },
28532 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI },
28533 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT },
28534 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT },
28536 /* BMI2 */
28537 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_si3, "__builtin_ia32_bzhi_si", IX86_BUILTIN_BZHI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28538 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_di3, "__builtin_ia32_bzhi_di", IX86_BUILTIN_BZHI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28539 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_si3, "__builtin_ia32_pdep_si", IX86_BUILTIN_PDEP32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28540 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_di3, "__builtin_ia32_pdep_di", IX86_BUILTIN_PDEP64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28541 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_si3, "__builtin_ia32_pext_si", IX86_BUILTIN_PEXT32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28542 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_di3, "__builtin_ia32_pext_di", IX86_BUILTIN_PEXT64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28543 };
28545 /* FMA4 and XOP. */
28546 #define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT
28547 #define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT
28548 #define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT
28549 #define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT
28550 #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF
28551 #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF
28552 #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF
28553 #define MULTI_ARG_3_DF2 V4DF_FTYPE_V4DF_V4DF_V4DF
28554 #define MULTI_ARG_3_DI V2DI_FTYPE_V2DI_V2DI_V2DI
28555 #define MULTI_ARG_3_SI V4SI_FTYPE_V4SI_V4SI_V4SI
28556 #define MULTI_ARG_3_SI_DI V4SI_FTYPE_V4SI_V4SI_V2DI
28557 #define MULTI_ARG_3_HI V8HI_FTYPE_V8HI_V8HI_V8HI
28558 #define MULTI_ARG_3_HI_SI V8HI_FTYPE_V8HI_V8HI_V4SI
28559 #define MULTI_ARG_3_QI V16QI_FTYPE_V16QI_V16QI_V16QI
28560 #define MULTI_ARG_3_DI2 V4DI_FTYPE_V4DI_V4DI_V4DI
28561 #define MULTI_ARG_3_SI2 V8SI_FTYPE_V8SI_V8SI_V8SI
28562 #define MULTI_ARG_3_HI2 V16HI_FTYPE_V16HI_V16HI_V16HI
28563 #define MULTI_ARG_3_QI2 V32QI_FTYPE_V32QI_V32QI_V32QI
28564 #define MULTI_ARG_2_SF V4SF_FTYPE_V4SF_V4SF
28565 #define MULTI_ARG_2_DF V2DF_FTYPE_V2DF_V2DF
28566 #define MULTI_ARG_2_DI V2DI_FTYPE_V2DI_V2DI
28567 #define MULTI_ARG_2_SI V4SI_FTYPE_V4SI_V4SI
28568 #define MULTI_ARG_2_HI V8HI_FTYPE_V8HI_V8HI
28569 #define MULTI_ARG_2_QI V16QI_FTYPE_V16QI_V16QI
28570 #define MULTI_ARG_2_DI_IMM V2DI_FTYPE_V2DI_SI
28571 #define MULTI_ARG_2_SI_IMM V4SI_FTYPE_V4SI_SI
28572 #define MULTI_ARG_2_HI_IMM V8HI_FTYPE_V8HI_SI
28573 #define MULTI_ARG_2_QI_IMM V16QI_FTYPE_V16QI_SI
28574 #define MULTI_ARG_2_DI_CMP V2DI_FTYPE_V2DI_V2DI_CMP
28575 #define MULTI_ARG_2_SI_CMP V4SI_FTYPE_V4SI_V4SI_CMP
28576 #define MULTI_ARG_2_HI_CMP V8HI_FTYPE_V8HI_V8HI_CMP
28577 #define MULTI_ARG_2_QI_CMP V16QI_FTYPE_V16QI_V16QI_CMP
28578 #define MULTI_ARG_2_SF_TF V4SF_FTYPE_V4SF_V4SF_TF
28579 #define MULTI_ARG_2_DF_TF V2DF_FTYPE_V2DF_V2DF_TF
28580 #define MULTI_ARG_2_DI_TF V2DI_FTYPE_V2DI_V2DI_TF
28581 #define MULTI_ARG_2_SI_TF V4SI_FTYPE_V4SI_V4SI_TF
28582 #define MULTI_ARG_2_HI_TF V8HI_FTYPE_V8HI_V8HI_TF
28583 #define MULTI_ARG_2_QI_TF V16QI_FTYPE_V16QI_V16QI_TF
28584 #define MULTI_ARG_1_SF V4SF_FTYPE_V4SF
28585 #define MULTI_ARG_1_DF V2DF_FTYPE_V2DF
28586 #define MULTI_ARG_1_SF2 V8SF_FTYPE_V8SF
28587 #define MULTI_ARG_1_DF2 V4DF_FTYPE_V4DF
28588 #define MULTI_ARG_1_DI V2DI_FTYPE_V2DI
28589 #define MULTI_ARG_1_SI V4SI_FTYPE_V4SI
28590 #define MULTI_ARG_1_HI V8HI_FTYPE_V8HI
28591 #define MULTI_ARG_1_QI V16QI_FTYPE_V16QI
28592 #define MULTI_ARG_1_SI_DI V2DI_FTYPE_V4SI
28593 #define MULTI_ARG_1_HI_DI V2DI_FTYPE_V8HI
28594 #define MULTI_ARG_1_HI_SI V4SI_FTYPE_V8HI
28595 #define MULTI_ARG_1_QI_DI V2DI_FTYPE_V16QI
28596 #define MULTI_ARG_1_QI_SI V4SI_FTYPE_V16QI
28597 #define MULTI_ARG_1_QI_HI V8HI_FTYPE_V16QI
28600 {
28641 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
28642 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
28643 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4si, "__builtin_ia32_vpcmov_v4si", IX86_BUILTIN_VPCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
28644 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8hi, "__builtin_ia32_vpcmov_v8hi", IX86_BUILTIN_VPCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
28645 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16qi, "__builtin_ia32_vpcmov_v16qi",IX86_BUILTIN_VPCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
28646 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2df, "__builtin_ia32_vpcmov_v2df", IX86_BUILTIN_VPCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
28647 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4sf, "__builtin_ia32_vpcmov_v4sf", IX86_BUILTIN_VPCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
28649 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov256", IX86_BUILTIN_VPCMOV256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
28650 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov_v4di256", IX86_BUILTIN_VPCMOV_V4DI256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
28651 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8si256, "__builtin_ia32_vpcmov_v8si256", IX86_BUILTIN_VPCMOV_V8SI256, UNKNOWN, (int)MULTI_ARG_3_SI2 },
28652 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16hi256, "__builtin_ia32_vpcmov_v16hi256", IX86_BUILTIN_VPCMOV_V16HI256, UNKNOWN, (int)MULTI_ARG_3_HI2 },
28653 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v32qi256, "__builtin_ia32_vpcmov_v32qi256", IX86_BUILTIN_VPCMOV_V32QI256, UNKNOWN, (int)MULTI_ARG_3_QI2 },
28654 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4df256, "__builtin_ia32_vpcmov_v4df256", IX86_BUILTIN_VPCMOV_V4DF256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
28655 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8sf256, "__builtin_ia32_vpcmov_v8sf256", IX86_BUILTIN_VPCMOV_V8SF256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
28657 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pperm, "__builtin_ia32_vpperm", IX86_BUILTIN_VPPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
28659 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssww, "__builtin_ia32_vpmacssww", IX86_BUILTIN_VPMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
28660 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsww, "__builtin_ia32_vpmacsww", IX86_BUILTIN_VPMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
28661 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsswd, "__builtin_ia32_vpmacsswd", IX86_BUILTIN_VPMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
28662 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacswd, "__builtin_ia32_vpmacswd", IX86_BUILTIN_VPMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
28663 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdd, "__builtin_ia32_vpmacssdd", IX86_BUILTIN_VPMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
28664 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdd, "__builtin_ia32_vpmacsdd", IX86_BUILTIN_VPMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
28665 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdql, "__builtin_ia32_vpmacssdql", IX86_BUILTIN_VPMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
28666 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdqh, "__builtin_ia32_vpmacssdqh", IX86_BUILTIN_VPMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
28667 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdql, "__builtin_ia32_vpmacsdql", IX86_BUILTIN_VPMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
28668 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdqh, "__builtin_ia32_vpmacsdqh", IX86_BUILTIN_VPMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
28669 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcsswd, "__builtin_ia32_vpmadcsswd", IX86_BUILTIN_VPMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
28670 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcswd, "__builtin_ia32_vpmadcswd", IX86_BUILTIN_VPMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
28672 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv2di3, "__builtin_ia32_vprotq", IX86_BUILTIN_VPROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
28673 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv4si3, "__builtin_ia32_vprotd", IX86_BUILTIN_VPROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
28674 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv8hi3, "__builtin_ia32_vprotw", IX86_BUILTIN_VPROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
28675 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv16qi3, "__builtin_ia32_vprotb", IX86_BUILTIN_VPROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
28676 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv2di3, "__builtin_ia32_vprotqi", IX86_BUILTIN_VPROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
28677 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
28678 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
28679 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
28680 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
28681 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
28682 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
28683 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
28684 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
28685 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
28686 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
28687 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
28689 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_2_SF },
28690 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF },
28691 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
28692 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
28693 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
28694 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
28696 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
28697 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
28698 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbq, "__builtin_ia32_vphaddbq", IX86_BUILTIN_VPHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
28699 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwd, "__builtin_ia32_vphaddwd", IX86_BUILTIN_VPHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
28700 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwq, "__builtin_ia32_vphaddwq", IX86_BUILTIN_VPHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
28701 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadddq, "__builtin_ia32_vphadddq", IX86_BUILTIN_VPHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
28702 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubw, "__builtin_ia32_vphaddubw", IX86_BUILTIN_VPHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
28703 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubd, "__builtin_ia32_vphaddubd", IX86_BUILTIN_VPHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
28704 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubq, "__builtin_ia32_vphaddubq", IX86_BUILTIN_VPHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
28705 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwd, "__builtin_ia32_vphadduwd", IX86_BUILTIN_VPHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
28706 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwq, "__builtin_ia32_vphadduwq", IX86_BUILTIN_VPHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
28707 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddudq, "__builtin_ia32_vphaddudq", IX86_BUILTIN_VPHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
28708 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubbw, "__builtin_ia32_vphsubbw", IX86_BUILTIN_VPHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
28709 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubwd, "__builtin_ia32_vphsubwd", IX86_BUILTIN_VPHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
28710 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubdq, "__builtin_ia32_vphsubdq", IX86_BUILTIN_VPHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
28712 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomeqb", IX86_BUILTIN_VPCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
28713 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
28714 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneqb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
28715 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomltb", IX86_BUILTIN_VPCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
28716 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomleb", IX86_BUILTIN_VPCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
28717 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgtb", IX86_BUILTIN_VPCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
28718 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgeb", IX86_BUILTIN_VPCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
28720 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomeqw", IX86_BUILTIN_VPCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
28721 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomnew", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
28722 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomneqw", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
28723 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomltw", IX86_BUILTIN_VPCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
28724 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomlew", IX86_BUILTIN_VPCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
28725 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgtw", IX86_BUILTIN_VPCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
28726 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgew", IX86_BUILTIN_VPCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
28728 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomeqd", IX86_BUILTIN_VPCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
28729 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomned", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
28730 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomneqd", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
28731 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomltd", IX86_BUILTIN_VPCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
28732 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomled", IX86_BUILTIN_VPCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
28733 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomgtd", IX86_BUILTIN_VPCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
28734 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomged", IX86_BUILTIN_VPCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
28736 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomeqq", IX86_BUILTIN_VPCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
28737 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
28738 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneqq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
28739 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomltq", IX86_BUILTIN_VPCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
28740 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomleq", IX86_BUILTIN_VPCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
28741 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgtq", IX86_BUILTIN_VPCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
28742 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgeq", IX86_BUILTIN_VPCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
28744 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomequb", IX86_BUILTIN_VPCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
28745 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomneub", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
28746 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomnequb", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
28747 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomltub", IX86_BUILTIN_VPCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
28748 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomleub", IX86_BUILTIN_VPCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
28749 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgtub", IX86_BUILTIN_VPCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
28750 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgeub", IX86_BUILTIN_VPCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
28752 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomequw", IX86_BUILTIN_VPCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
28753 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomneuw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
28754 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomnequw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
28755 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomltuw", IX86_BUILTIN_VPCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
28756 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomleuw", IX86_BUILTIN_VPCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
28757 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgtuw", IX86_BUILTIN_VPCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
28758 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgeuw", IX86_BUILTIN_VPCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
28760 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomequd", IX86_BUILTIN_VPCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
28761 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomneud", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
28762 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomnequd", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
28763 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomltud", IX86_BUILTIN_VPCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
28764 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomleud", IX86_BUILTIN_VPCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
28765 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgtud", IX86_BUILTIN_VPCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
28766 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgeud", IX86_BUILTIN_VPCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
28768 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomequq", IX86_BUILTIN_VPCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
28769 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomneuq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
28770 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomnequq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
28771 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomltuq", IX86_BUILTIN_VPCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
28772 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomleuq", IX86_BUILTIN_VPCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
28773 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgtuq", IX86_BUILTIN_VPCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
28774 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgeuq", IX86_BUILTIN_VPCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
28776 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseb", IX86_BUILTIN_VPCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
28777 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalsew", IX86_BUILTIN_VPCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
28778 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalsed", IX86_BUILTIN_VPCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
28779 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseq", IX86_BUILTIN_VPCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
28780 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseub",IX86_BUILTIN_VPCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
28781 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalseuw",IX86_BUILTIN_VPCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
28782 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalseud",IX86_BUILTIN_VPCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
28783 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseuq",IX86_BUILTIN_VPCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
28785 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueb", IX86_BUILTIN_VPCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
28786 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtruew", IX86_BUILTIN_VPCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
28787 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrued", IX86_BUILTIN_VPCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
28788 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueq", IX86_BUILTIN_VPCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
28789 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueub", IX86_BUILTIN_VPCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
28790 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtrueuw", IX86_BUILTIN_VPCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
28791 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
28792 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
28794 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I },
28795 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I },
28796 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 },
28797 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
28799 };
28800 \f
28801 /* TM vector builtins. */
28803 /* Reuse the existing x86-specific `struct builtin_description' cause
28804 we're lazy. Add casts to make them fit. */
28806 {
28807 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
28808 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
28809 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
28810 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
28811 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
28812 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
28813 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
28815 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
28816 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
28817 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
28818 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
28819 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
28820 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
28821 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
28823 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
28824 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
28825 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
28826 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
28827 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
28828 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
28829 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
28831 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
28832 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
28833 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
28834 };
28836 /* TM callbacks. */
28838 /* Return the builtin decl needed to load a vector of TYPE. */
28840 static tree
28842 {
28844 {
28846 {
28853 }
28854 }
28856 }
28858 /* Return the builtin decl needed to store a vector of TYPE. */
28860 static tree
28862 {
28864 {
28866 {
28873 }
28874 }
28876 }
28877 \f
28878 /* Initialize the transactional memory vector load/store builtins. */
28880 static void
28882 {
28886 tree decl;
28893 /* If there are no builtins defined, we must be compiling in a
28894 language without trans-mem support. */
28898 /* Use whatever attributes a normal TM load has. */
28902 /* Use whatever attributes a normal TM store has. */
28906 /* Use whatever attributes a normal TM log has. */
28914 {
28918 {
28926 {
28929 }
28931 {
28934 }
28935 else
28936 {
28939 }
28941 /* The builtin without the prefix for
28942 calling it directly. */
28944 attrs);
28945 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
28946 set the TYPE_ATTRIBUTES. */
28950 }
28951 }
28952 }
28954 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
28955 in the current target ISA to allow the user to compile particular modules
28956 with different target specific options that differ from the command line
28957 options. */
28958 static void
28960 {
28965 /* Add all special builtins with variable number of operands. */
28969 {
28975 }
28977 /* Add all builtins with variable number of operands. */
28981 {
28987 }
28989 /* pcmpestr[im] insns. */
28993 {
28996 else
28999 }
29001 /* pcmpistr[im] insns. */
29005 {
29008 else
29011 }
29013 /* comi/ucomi insns. */
29015 {
29018 else
29021 }
29023 /* SSE */
29029 /* SSE or 3DNow!A */
29032 IX86_BUILTIN_MASKMOVQ);
29034 /* SSE2 */
29043 /* SSE3. */
29049 /* AES */
29063 /* PCLMUL */
29067 /* RDRND */
29074 IX86_BUILTIN_RDRAND64_STEP);
29076 /* AVX2 */
29078 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
29079 IX86_BUILTIN_GATHERSIV2DF);
29082 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
29083 IX86_BUILTIN_GATHERSIV4DF);
29086 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
29087 IX86_BUILTIN_GATHERDIV2DF);
29090 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
29091 IX86_BUILTIN_GATHERDIV4DF);
29094 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
29095 IX86_BUILTIN_GATHERSIV4SF);
29098 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
29099 IX86_BUILTIN_GATHERSIV8SF);
29102 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
29103 IX86_BUILTIN_GATHERDIV4SF);
29106 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
29107 IX86_BUILTIN_GATHERDIV8SF);
29110 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
29111 IX86_BUILTIN_GATHERSIV2DI);
29114 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
29115 IX86_BUILTIN_GATHERSIV4DI);
29118 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
29119 IX86_BUILTIN_GATHERDIV2DI);
29122 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
29123 IX86_BUILTIN_GATHERDIV4DI);
29126 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
29127 IX86_BUILTIN_GATHERSIV4SI);
29130 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
29131 IX86_BUILTIN_GATHERSIV8SI);
29134 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
29135 IX86_BUILTIN_GATHERDIV4SI);
29138 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
29139 IX86_BUILTIN_GATHERDIV8SI);
29142 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
29143 IX86_BUILTIN_GATHERALTSIV4DF);
29146 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
29147 IX86_BUILTIN_GATHERALTDIV8SF);
29150 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
29151 IX86_BUILTIN_GATHERALTSIV4DI);
29154 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
29155 IX86_BUILTIN_GATHERALTDIV8SI);
29157 /* RTM. */
29161 /* MMX access to the vec_init patterns. */
29166 V4HI_FTYPE_HI_HI_HI_HI,
29167 IX86_BUILTIN_VEC_INIT_V4HI);
29170 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
29171 IX86_BUILTIN_VEC_INIT_V8QI);
29173 /* Access to the vec_extract patterns. */
29195 /* Access to the vec_set patterns. */
29216 /* RDSEED */
29225 /* ADCX */
29230 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
29231 IX86_BUILTIN_ADDCARRYX64);
29233 /* Add FMA4 multi-arg argument instructions */
29235 {
29241 }
29242 }
29244 /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
29245 to return a pointer to VERSION_DECL if the outcome of the expression
29246 formed by PREDICATE_CHAIN is true. This function will be called during
29247 version dispatch to decide which function version to execute. It returns
29248 the basic block at the end, to which more conditions can be added. */
29250 static basic_block
29253 {
29254 gimple return_stmt;
29256 gimple convert_stmt;
29257 gimple call_cond_stmt;
29258 gimple if_else_stmt;
29264 gimple_seq gseq;
29281 {
29289 }
29292 {
29307 else
29308 {
29309 gimple assign_stmt;
29310 /* Use MIN_EXPR to check if any integer is zero?.
29311 and_expr_var = min_expr <cond_var, and_expr_var> */
29319 }
29320 }
29323 integer_zero_node,
29353 }
29355 /* This parses the attribute arguments to target in DECL and determines
29356 the right builtin to use to match the platform specification.
29357 It returns the priority value for this version decl. If PREDICATE_LIST
29358 is not NULL, it stores the list of cpu features that need to be checked
29359 before dispatching this function. */
29361 static unsigned int
29363 {
29364 tree attrs;
29366 tree target_node;
29373 /* Priority of i386 features, greater value is higher priority. This is
29374 used to decide the order in which function dispatch must happen. For
29375 instance, a version specialized for SSE4.2 should be checked for dispatch
29376 before a version for SSE3, as SSE4.2 implies SSE3. */
29377 enum feature_priority
29378 {
29380 P_MMX,
29381 P_SSE,
29382 P_SSE2,
29383 P_SSE3,
29384 P_SSSE3,
29385 P_PROC_SSSE3,
29386 P_SSE4_a,
29387 P_PROC_SSE4_a,
29388 P_SSE4_1,
29389 P_SSE4_2,
29390 P_PROC_SSE4_2,
29391 P_POPCNT,
29392 P_AVX,
29393 P_AVX2,
29394 P_FMA,
29395 P_PROC_FMA
29396 };
29400 /* These are the target attribute strings for which a dispatcher is
29401 available, from fold_builtin_cpu. */
29403 static struct _feature_list
29404 {
29407 }
29409 {
29420 };
29423 static unsigned int NUM_FEATURES
29439 /* Return priority zero for default function. */
29443 /* Handle arch= if specified. For priority, set it to be 1 more than
29444 the best instruction set the processor can handle. For instance, if
29445 there is a version for atom and a version for ssse3 (the highest ISA
29446 priority for atom), the atom version must be checked for dispatch
29447 before the ssse3 version. */
29449 {
29458 {
29460 {
29485 }
29486 }
29491 {
29495 }
29498 {
29500 /* For a C string literal the length includes the trailing NULL. */
29503 predicate_chain);
29504 }
29505 }
29507 /* Process feature name. */
29514 {
29515 /* Do not process "arch=" */
29517 {
29520 }
29522 {
29524 {
29526 {
29531 predicate_chain);
29532 }
29533 /* Find the maximum priority feature. */
29538 }
29539 }
29541 {
29545 }
29547 }
29551 {
29554 attrs_str);
29556 }
29558 {
29561 }
29564 }
29566 /* This compares the priority of target features in function DECL1
29567 and DECL2. It returns positive value if DECL1 is higher priority,
29568 negative value if DECL2 is higher priority and 0 if they are the
29569 same. */
29571 static int
29573 {
29578 }
29580 /* V1 and V2 point to function versions with different priorities
29581 based on the target ISA. This function compares their priorities. */
29583 static int
29585 {
29587 {
29588 tree version_decl;
29589 tree predicate_chain;
29596 }
29598 /* This function generates the dispatch function for
29599 multi-versioned functions. DISPATCH_DECL is the function which will
29600 contain the dispatch logic. FNDECLS are the function choices for
29601 dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
29602 in DISPATCH_DECL in which the dispatch code is generated. */
29604 static int
29608 {
29609 tree default_decl;
29610 gimple ifunc_cpu_init_stmt;
29611 gimple_seq gseq;
29613 tree ele;
29619 struct _function_version_info
29620 {
29621 tree version_decl;
29622 tree predicate_chain;
29630 /*fndecls_p is actually a vector. */
29633 /* At least one more version other than the default. */
29640 /* The first version in the vector is the default decl. */
29646 /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
29647 constructors, so explicity call __builtin_cpu_init here. */
29658 {
29662 /* Get attribute string, parse it and find the right predicate decl.
29663 The predicate function could be a lengthy combination of many
29664 features, like arch-type and various isa-variants. */
29675 actual_versions++;
29676 }
29678 /* Sort the versions according to descending order of dispatch priority. The
29679 priority is based on the ISA. This is not a perfect solution. There
29680 could still be ambiguity. If more than one function version is suitable
29681 to execute, which one should be dispatched? In future, allow the user
29682 to specify a dispatch priority next to the version. */
29692 /* dispatch default version at the end. */
29698 }
29700 /* Comparator function to be used in qsort routine to sort attribute
29701 specification strings to "target". */
29703 static int
29705 {
29709 }
29711 /* ARGLIST is the argument to target attribute. This function tokenizes
29712 the comma separated arguments, sorts them and returns a string which
29713 is a unique identifier for the comma separated arguments. It also
29714 replaces non-identifier characters "=,-" with "_". */
29718 {
29719 tree arg;
29728 {
29733 argnum++;
29736 argnum++;
29737 }
29742 {
29748 }
29750 /* Replace "=,-" with "_". */
29763 {
29765 i++;
29767 }
29774 {
29779 }
29784 }
29786 /* This function changes the assembler name for functions that are
29787 versions. If DECL is a function version and has a "target"
29788 attribute, it appends the attribute string to its assembler name. */
29790 static tree
29792 {
29793 tree version_attr;
29801 "Function versions cannot be marked as gnu_inline,"
29810 /* target attribute string cannot be NULL. */
29814 version_string
29825 /* Allow assembler name to be modified if already set. */
29833 }
29835 /* This function returns true if FN1 and FN2 are versions of the same function,
29836 that is, the target strings of the function decls are different. This assumes
29837 that FN1 and FN2 have the same signature. */
29839 static bool
29841 {
29853 /* At least one function decl should have the target attribute specified. */
29857 /* Diagnose missing target attribute if one of the decls is already
29858 multi-versioned. */
29860 {
29862 {
29864 {
29869 }
29872 fn2);
29875 /* Prevent diagnosing of the same error multiple times. */
29880 }
29882 }
29887 /* The sorted target strings must be different for fn1 and fn2
29888 to be versions. */
29891 else
29898 }
29900 static tree
29902 {
29903 /* For function version, add the target suffix to the assembler name. */
29907 #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
29909 #endif
29912 }
29914 /* Return a new name by appending SUFFIX to the DECL name. If make_unique
29915 is true, append the full path name of the source file. */
29919 {
29927 /* Get a unique name that can be used globally without any chances
29928 of collision at link time. */
29938 /* Use '.' to concatenate names as it is demangler friendly. */
29941 suffix);
29942 else
29946 }
29948 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
29950 /* Make a dispatcher declaration for the multi-versioned function DECL.
29951 Calls to DECL function will be replaced with calls to the dispatcher
29952 by the front-end. Return the decl created. */
29954 static tree
29956 {
29957 tree func_decl;
29977 /* Mark this func as external, the resolver will flip it again if
29978 it gets generated. */
29980 /* This will be of type IFUNCs have to be externally visible. */
29984 }
29986 #endif
29988 /* Returns true if decl is multi-versioned and DECL is the default function,
29989 that is it is not tagged with target specific optimization. */
29991 static bool
29993 {
30002 }
30004 /* Make a dispatcher declaration for the multi-versioned function DECL.
30005 Calls to DECL function will be replaced with calls to the dispatcher
30006 by the front-end. Returns the decl of the dispatcher function. */
30008 static tree
30010 {
30032 /* Find the default version and make it the first node. */
30034 /* Go to the beginning of the chain. */
30039 {
30044 }
30046 /* If there is no default node, just return NULL. */
30050 /* Make default info the first node. */
30052 {
30059 }
30063 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
30065 {
30070 /* Right now, the dispatching is done via ifunc. */
30076 dispatcher_version_info
30081 /* Set the dispatcher for all the versions. */
30084 {
30087 }
30088 }
30089 else
30090 #endif
30091 {
30093 "multiversioning needs ifunc which is not supported "
30095 }
30098 }
30100 /* Makes a function attribute of the form NAME(ARG_NAME) and chains
30101 it to CHAIN. */
30103 static tree
30105 {
30106 tree attr_name;
30107 tree attr_arg_name;
30108 tree attr_args;
30109 tree attr;
30116 }
30118 /* Make the resolver function decl to dispatch the versions of
30119 a multi-versioned function, DEFAULT_DECL. Create an
30120 empty basic block in the resolver and store the pointer in
30121 EMPTY_BB. Return the decl of the resolver function. */
30123 static tree
30127 {
30132 /* IFUNC's have to be globally visible. So, if the default_decl is
30133 not, then the name of the IFUNC should be made unique. */
30137 /* Append the filename to the resolver function if the versions are
30138 not externally visible. This is because the resolver function has
30139 to be externally visible for the loader to find it. So, appending
30140 the filename will prevent conflicts with a resolver function from
30141 another module which is based on the same version name. */
30144 /* The resolver function should return a (void *). */
30155 /* IFUNC resolvers have to be externally visible. */
30159 /* Resolver is not external, body is generated. */
30169 {
30170 /* In this case, each translation unit with a call to this
30171 versioned function will put out a resolver. Ensure it
30172 is comdat to keep just one copy. */
30175 }
30176 /* Build result decl and add to function_decl. */
30192 /* Mark dispatch_decl as "ifunc" with resolver as resolver_name. */
30196 /* Create the alias for dispatch to resolver here. */
30197 /*cgraph_create_function_alias (dispatch_decl, decl);*/
30201 }
30203 /* Generate the dispatching code body to dispatch multi-versioned function
30204 DECL. The target hook is called to process the "target" attributes and
30205 provide the code to dispatch the right function at run-time. NODE points
30206 to the dispatcher decl whose body will be created. */
30208 static tree
30210 {
30211 tree resolver_decl;
30212 basic_block empty_bb;
30214 tree default_ver_decl;
30230 /* The first version in the chain corresponds to the default version. */
30233 /* node is going to be an alias, so remove the finalized bit. */
30247 {
30249 /* Check for virtual functions here again, as by this time it should
30250 have been determined if this function needs a vtable index or
30251 not. This happens for methods in derived classes that override
30252 virtual methods in base classes but are not explicitly marked as
30253 virtual. */
30258 }
30265 }
30266 /* This builds the processor_model struct type defined in
30267 libgcc/config/i386/cpuinfo.c */
30269 static tree
30271 {
30278 /* The first 3 fields are unsigned int. */
30280 {
30286 }
30288 /* The last field is an array of unsigned integers of size one. */
30299 }
30301 /* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
30303 static tree
30305 {
30306 tree new_decl;
30309 VAR_DECL,
30311 type);
30324 }
30326 /* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
30327 into an integer defined in libgcc/config/i386/cpuinfo.c */
30329 static tree
30331 {
30337 /* This is the order of bit-fields in __processor_features in cpuinfo.c */
30338 enum processor_features
30339 {
30341 F_MMX,
30342 F_POPCNT,
30343 F_SSE,
30344 F_SSE2,
30345 F_SSE3,
30346 F_SSSE3,
30347 F_SSE4_1,
30348 F_SSE4_2,
30349 F_AVX,
30350 F_AVX2,
30351 F_MAX
30352 };
30354 /* These are the values for vendor types and cpu types and subtypes
30355 in cpuinfo.c. Cpu types and subtypes should be subtracted by
30356 the corresponding start value. */
30357 enum processor_model
30358 {
30360 M_AMD,
30361 M_CPU_TYPE_START,
30362 M_INTEL_ATOM,
30363 M_INTEL_CORE2,
30364 M_INTEL_COREI7,
30365 M_AMDFAM10H,
30366 M_AMDFAM15H,
30367 M_INTEL_SLM,
30368 M_CPU_SUBTYPE_START,
30369 M_INTEL_COREI7_NEHALEM,
30370 M_INTEL_COREI7_WESTMERE,
30371 M_INTEL_COREI7_SANDYBRIDGE,
30372 M_AMDFAM10H_BARCELONA,
30373 M_AMDFAM10H_SHANGHAI,
30374 M_AMDFAM10H_ISTANBUL,
30375 M_AMDFAM15H_BDVER1,
30376 M_AMDFAM15H_BDVER2,
30377 M_AMDFAM15H_BDVER3
30378 };
30380 static struct _arch_names_table
30381 {
30384 }
30386 {
30404 };
30406 static struct _isa_names_table
30407 {
30410 }
30412 {
30424 };
30438 {
30439 /* *args must be a expr that can contain other EXPRS leading to a
30440 STRING_CST. */
30442 {
30445 }
30447 }
30452 {
30453 tree ref;
30454 tree field;
30455 tree final;
30458 unsigned int NUM_ARCH_NAMES
30467 {
30471 }
30476 /* CPU types are stored in the next field. */
30479 {
30482 }
30484 /* CPU subtypes are stored in the next field. */
30486 {
30489 }
30491 /* Get the appropriate field in __cpu_model. */
30495 /* Check the value. */
30499 }
30501 {
30502 tree ref;
30503 tree array_elt;
30504 tree field;
30505 tree final;
30508 unsigned int NUM_ISA_NAMES
30517 {
30521 }
30524 /* Get the last field, which is __cpu_features. */
30528 /* Get the appropriate field: __cpu_model.__cpu_features */
30532 /* Access the 0th element of __cpu_features array. */
30537 /* Return __cpu_model.__cpu_features[0] & field_val */
30541 }
30543 }
30545 static tree
30548 {
30550 {
30555 {
30558 }
30559 }
30561 #ifdef SUBTARGET_FOLD_BUILTIN
30563 #endif
30566 }
30568 /* Make builtins to detect cpu type and features supported. NAME is
30569 the builtin name, CODE is the builtin code, and FTYPE is the function
30570 type of the builtin. */
30572 static void
30575 {
30576 tree decl;
30577 tree type;
30585 }
30587 /* Make builtins to get CPU type and features supported. The created
30588 builtins are :
30590 __builtin_cpu_init (), to detect cpu type and features,
30591 __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
30592 __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
30593 */
30595 static void
30597 {
30604 }
30606 /* Internal method for ix86_init_builtins. */
30608 static void
30610 {
30622 sysv_va_ref =
30625 fnvoid_va_end_ms =
30627 fnvoid_va_start_ms =
30629 fnvoid_va_end_sysv =
30631 fnvoid_va_start_sysv =
30633 NULL_TREE);
30634 fnvoid_va_copy_ms =
30636 NULL_TREE);
30637 fnvoid_va_copy_sysv =
30653 }
30655 static void
30657 {
30660 /* The __float80 type. */
30663 {
30664 /* The __float80 type. */
30669 }
30672 /* The __float128 type. */
30678 /* This macro is built by i386-builtin-types.awk. */
30679 DEFINE_BUILTIN_PRIMITIVE_TYPES;
30680 }
30682 static void
30684 {
30685 tree t;
30689 /* Builtins to get CPU type and features. */
30692 /* TFmode support builtins. */
30698 /* We will expand them to normal call if SSE isn't available since
30699 they are used by libgcc. */
30718 #ifdef SUBTARGET_INIT_BUILTINS
30719 SUBTARGET_INIT_BUILTINS;
30720 #endif
30721 }
30723 /* Return the ix86 builtin for CODE. */
30725 static tree
30727 {
30732 }
30734 /* Errors in the source file can cause expand_expr to return const0_rtx
30735 where we expect a vector. To avoid crashing, use one of the vector
30736 clear instructions. */
30737 static rtx
30739 {
30743 }
30745 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
30747 static rtx
30749 {
30750 rtx pat;
30770 {
30774 }
30788 }
30790 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
30792 static rtx
30796 {
30797 rtx pat;
30805 rtx op;
30812 {
30892 }
30902 {
30909 {
30911 {
30914 {
30932 xop_rotl:
30934 {
30937 gcc_checking_assert
30939 }
30940 else
30941 {
30942 gcc_checking_assert
30953 }
30957 }
30958 }
30959 }
30960 else
30961 {
30962 non_constant:
30966 /* If we aren't optimizing, only allow one memory operand to be
30967 generated. */
30969 num_memory++;
30977 }
30981 }
30984 {
30995 else
30996 {
31002 }
31015 }
31022 }
31024 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
31025 insns with vec_merge. */
31027 static rtx
31029 rtx target)
31030 {
31031 rtx pat;
31058 }
31060 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
31062 static rtx
31065 {
31066 rtx pat;
31071 rtx op2;
31082 /* Swap operands if we have a comparison that isn't available in
31083 hardware. */
31085 {
31090 }
31110 }
31112 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
31114 static rtx
31116 rtx target)
31117 {
31118 rtx pat;
31132 /* Swap operands if we have a comparison that isn't available in
31133 hardware. */
31135 {
31139 }
31160 const0_rtx)));
31163 }
31165 /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
31167 static rtx
31169 rtx target)
31170 {
31171 rtx pat;
31196 }
31198 static rtx
31201 {
31202 rtx pat;
31207 rtx op2;
31234 }
31236 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
31238 static rtx
31240 rtx target)
31241 {
31242 rtx pat;
31275 const0_rtx)));
31278 }
31280 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
31282 static rtx
31285 {
31286 rtx pat;
31324 {
31327 }
31330 {
31339 }
31341 {
31350 }
31351 else
31352 {
31359 }
31367 {
31372 emit_insn
31376 FLAGS_REG),
31377 const0_rtx)));
31379 }
31380 else
31382 }
31385 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
31387 static rtx
31390 {
31391 rtx pat;
31419 {
31422 }
31425 {
31434 }
31436 {
31445 }
31446 else
31447 {
31454 }
31462 {
31467 emit_insn
31471 FLAGS_REG),
31472 const0_rtx)));
31474 }
31475 else
31477 }
31479 /* Subroutine of ix86_expand_builtin to take care of insns with
31480 variable number of operands. */
31482 static rtx
31485 {
31490 struct
31491 {
31492 rtx op;
31504 {
31783 }
31788 {
31791 }
31794 {
31801 }
31802 else
31803 {
31806 }
31809 {
31816 {
31817 /* SIMD shift insns take either an 8-bit immediate or
31818 register as count. But builtin functions take int as
31819 count. If count doesn't match, we put it in register. */
31821 {
31825 }
31826 }
31828 {
31831 {
31885 {
31888 {
31891 }
31897 }
31899 }
31900 }
31901 else
31902 {
31906 /* If we aren't optimizing, only allow one memory operand to
31907 be generated. */
31909 num_memory++;
31912 {
31915 }
31916 else
31917 {
31920 }
31921 }
31925 }
31928 {
31945 }
31952 }
31954 /* Subroutine of ix86_expand_builtin to take care of special insns
31955 with variable number of operands. */
31957 static rtx
31960 {
31961 tree arg;
31964 struct
31965 {
31966 rtx op;
31976 {
32024 /* Reserve memory operand for target. */
32055 /* Reserve memory operand for target. */
32069 }
32074 {
32079 {
32082 }
32083 else
32086 }
32087 else
32088 {
32095 }
32098 {
32107 {
32109 {
32115 else
32118 }
32119 }
32120 else
32121 {
32123 {
32124 /* This must be the memory operand. */
32129 }
32130 else
32131 {
32132 /* This must be register. */
32139 }
32140 }
32144 }
32147 {
32162 }
32168 }
32170 /* Return the integer constant in ARG. Constrain it to be in the range
32171 of the subparts of VEC_TYPE; issue an error if not. */
32173 static int
32175 {
32180 {
32183 }
32186 }
32188 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32189 ix86_expand_vector_init. We DO have language-level syntax for this, in
32190 the form of (type){ init-list }. Except that since we can't place emms
32191 instructions from inside the compiler, we can't allow the use of MMX
32192 registers unless the user explicitly asks for it. So we do *not* define
32193 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
32194 we have builtins invoked by mmintrin.h that gives us license to emit
32195 these sorts of instructions. */
32197 static rtx
32199 {
32209 {
32212 }
32219 }
32221 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32222 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
32223 had a language-level syntax for referencing vector elements. */
32225 static rtx
32227 {
32231 rtx op0;
32251 }
32253 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32254 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
32255 a language-level syntax for referencing vector elements. */
32257 static rtx
32259 {
32283 /* OP0 is the source of these builtin functions and shouldn't be
32284 modified. Create a copy, use it and return it as target. */
32290 }
32292 /* Expand an expression EXP that calls a built-in function,
32293 with result going to TARGET if that's convenient
32294 (and in mode MODE if that's convenient).
32295 SUBTARGET may be used as the target for computing one of EXP's operands.
32296 IGNORE is nonzero if the value is to be ignored. */
32298 static rtx
32302 {
32312 /* For CPU builtins that can be folded, fold first and expand the fold. */
32314 {
32316 {
32317 /* Make it call __cpu_indicator_init in libgcc. */
32323 }
32326 {
32331 }
32332 }
32334 /* Determine whether the builtin function is available under the current ISA.
32335 Originally the builtin was not created if it wasn't applicable to the
32336 current ISA based on the command line switches. With function specific
32337 options, we need to check in the context of the function making the call
32338 whether it is supported. */
32341 {
32347 else
32348 {
32352 }
32354 }
32357 {
32361 ? CODE_FOR_mmx_maskmovq
32363 /* Note the arg order is different from the operand order. */
32464 {
32465 REAL_VALUE_TYPE inf;
32466 rtx tmp;
32478 }
32488 {
32494 insn = (TARGET_64BIT
32498 }
32500 {
32501 insn = (TARGET_64BIT
32505 }
32506 else
32507 {
32510 insn = (TARGET_64BIT
32518 {
32521 }
32523 }
32526 {
32527 /* mode is VOIDmode if __builtin_rd* has been called
32528 without lhs. */
32532 }
32535 {
32540 }
32550 {
32565 }
32571 {
32574 }
32594 {
32597 }
32603 {
32607 {
32628 }
32633 }
32634 else
32635 {
32637 {
32649 }
32651 }
32681 ? CODE_FOR_tbm_bextri_si
32684 {
32687 }
32688 else
32689 {
32698 }
32714 rdrand_step:
32721 {
32724 }
32730 /* Emit SImode conditional move. */
32732 {
32735 }
32738 else
32745 const0_rtx);
32764 rdseed_step:
32771 {
32774 }
32780 const0_rtx);
32798 addcarryx:
32806 /* Generate CF from input operand. */
32811 /* Gen ADCX instruction to compute X+Y+CF. */
32826 /* Store the result. */
32829 {
32832 }
32835 /* Return current CF value. */
32904 gather_gen:
32915 /* Note the arg order is different from the operand order. */
32924 else
32929 {
32935 }
32938 {
32943 else
32953 }
32955 /* Force memory operand only with base register here. But we
32956 don't want to do it on memory operand for other builtin
32957 functions. */
32969 {
32972 }
32974 /* Optimize. If mask is known to have all high bits set,
32975 replace op0 with pc_rtx to signal that the instruction
32976 overwrites the whole destination and doesn't use its
32977 previous contents. */
32979 {
32981 {
32984 {
32988 negative++;
32991 negative++;
32992 }
32995 }
32997 {
32998 /* Recognize also when mask is like:
32999 __v2df src = _mm_setzero_pd ();
33000 __v2df mask = _mm_cmpeq_pd (src, src);
33001 or
33002 __v8sf src = _mm256_setzero_ps ();
33003 __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
33004 as that is a cheaper way to load all ones into
33005 a register than having to load a constant from
33006 memory. */
33009 {
33014 {
33021 /* FALLTHRU */
33031 }
33032 }
33033 }
33034 }
33043 {
33050 else
33052 }
33053 else
33064 {
33067 }
33073 }
33086 {
33090 /* Emit a normal call if SSE isn't available. */
33094 }
33119 }
33121 /* Returns a function decl for a vectorized version of the builtin function
33122 with builtin function code FN and the result vector type TYPE, or NULL_TREE
33123 if it is not available. */
33125 static tree
33127 tree type_in)
33128 {
33144 {
33147 {
33152 }
33157 {
33162 }
33168 /* The round insn does not trap on denormals. */
33173 {
33178 }
33184 /* The round insn does not trap on denormals. */
33189 {
33194 }
33200 /* The round insn does not trap on denormals. */
33205 {
33210 }
33216 /* The round insn does not trap on denormals. */
33221 {
33226 }
33233 {
33238 }
33245 {
33250 }
33256 /* The round insn does not trap on denormals. */
33261 {
33266 }
33272 /* The round insn does not trap on denormals. */
33277 {
33282 }
33287 {
33292 }
33297 {
33302 }
33306 /* The round insn does not trap on denormals. */
33311 {
33316 }
33320 /* The round insn does not trap on denormals. */
33325 {
33330 }
33334 /* The round insn does not trap on denormals. */
33339 {
33344 }
33348 /* The round insn does not trap on denormals. */
33353 {
33358 }
33362 /* The round insn does not trap on denormals. */
33367 {
33372 }
33376 /* The round insn does not trap on denormals. */
33381 {
33386 }
33390 /* The round insn does not trap on denormals. */
33395 {
33400 }
33404 /* The round insn does not trap on denormals. */
33409 {
33414 }
33418 /* The round insn does not trap on denormals. */
33423 {
33428 }
33432 /* The round insn does not trap on denormals. */
33437 {
33442 }
33447 {
33452 }
33457 {
33462 }
33467 }
33469 /* Dispatch to a handler for a vectorization library. */
33472 type_in);
33475 }
33477 /* Handler for an SVML-style interface to
33478 a library with vectorized intrinsics. */
33480 static tree
33482 {
33490 /* The SVML is suitable for unsafe math only. */
33503 {
33550 }
33559 {
33562 }
33563 else
33566 /* Convert to uppercase. */
33571 args;
33573 arity++;
33577 else
33580 /* Build a function declaration for the vectorized function. */
33589 }
33591 /* Handler for an ACML-style interface to
33592 a library with vectorized intrinsics. */
33594 static tree
33596 {
33604 /* The ACML is 64bits only and suitable for unsafe math only as
33605 it does not correctly support parts of IEEE with the required
33606 precision such as denormals. */
33620 {
33650 }
33657 args;
33659 arity++;
33663 else
33666 /* Build a function declaration for the vectorized function. */
33675 }
33677 /* Returns a decl of a function that implements gather load with
33678 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
33679 Return NULL_TREE if it is not available. */
33681 static tree
33684 {
33700 /* v*gather* insn sign extends index to pointer mode. */
33712 {
33739 }
33742 }
33744 /* Returns a code for a target-specific builtin that implements
33745 reciprocal of the function, or NULL_TREE if not available. */
33747 static tree
33750 {
33757 /* Machine dependent builtins. */
33759 {
33760 /* Vectorized version of sqrt to rsqrt conversion. */
33769 }
33770 else
33771 /* Normal builtins. */
33773 {
33774 /* Sqrt to rsqrt conversion. */
33780 }
33781 }
33782 \f
33783 /* Helper for avx_vpermilps256_operand et al. This is also used by
33784 the expansion functions to turn the parallel back into a mask.
33785 The return value is 0 for no match and the imm8+1 for a match. */
33787 int
33789 {
33797 /* Validate that all of the elements are constants, and not totally
33798 out of range. Copy the data into an integral array to make the
33799 subsequent checks easier. */
33801 {
33811 }
33814 {
33816 /* In the 256-bit DFmode case, we can only move elements within
33817 a 128-bit lane. */
33819 {
33823 }
33825 {
33829 }
33833 /* In the 256-bit SFmode case, we have full freedom of movement
33834 within the low 128-bit lane, but the high 128-bit lane must
33835 mirror the exact same pattern. */
33840 /* FALLTHRU */
33844 /* In the 128-bit case, we've full freedom in the placement of
33845 the elements from the source operand. */
33852 }
33854 /* Make sure success has a non-zero value by adding one. */
33856 }
33858 /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
33859 the expansion functions to turn the parallel back into a mask.
33860 The return value is 0 for no match and the imm8+1 for a match. */
33862 int
33864 {
33872 /* Validate that all of the elements are constants, and not totally
33873 out of range. Copy the data into an integral array to make the
33874 subsequent checks easier. */
33876 {
33886 }
33888 /* Validate that the halves of the permute are halves. */
33896 /* Reconstruct the mask. */
33898 {
33904 }
33906 /* Make sure success has a non-zero value by adding one. */
33908 }
33909 \f
33910 /* Store OPERAND to the memory after reload is completed. This means
33911 that we can't easily use assign_stack_local. */
33912 rtx
33914 {
33915 rtx result;
33919 {
33922 stack_pointer_rtx,
33925 }
33927 {
33929 {
33933 /* FALLTHRU */
33939 stack_pointer_rtx)),
33940 operand));
33944 }
33946 }
33947 else
33948 {
33950 {
33952 {
33959 stack_pointer_rtx)),
33965 stack_pointer_rtx)),
33967 }
33970 /* Store HImodes as SImodes. */
33972 /* FALLTHRU */
33978 stack_pointer_rtx)),
33979 operand));
33983 }
33985 }
33987 }
33989 /* Free operand from the memory. */
33990 void
33992 {
33994 {
33999 else
34001 /* Use LEA to deallocate stack space. In peephole2 it will be converted
34002 to pop or add instruction if registers are available. */
34006 }
34007 }
34009 /* Return a register priority for hard reg REGNO. */
34010 static int
34012 {
34013 /* ebp and r13 as the base always wants a displacement, r12 as the
34014 base always wants an index. So discourage their usage in an
34015 address. */
34020 /* New x86-64 int registers result in bigger code size. Discourage
34021 them. */
34024 /* New x86-64 SSE registers result in bigger code size. Discourage
34025 them. */
34028 /* Usage of AX register results in smaller code. Prefer it. */
34032 }
34034 /* Implement TARGET_PREFERRED_RELOAD_CLASS.
34036 Put float CONST_DOUBLE in the constant pool instead of fp regs.
34037 QImode must go into class Q_REGS.
34038 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
34039 movdf to do mem-to-mem moves through integer regs. */
34041 static reg_class_t
34043 {
34046 /* We're only allowed to return a subclass of CLASS. Many of the
34047 following checks fail for NO_REGS, so eliminate that early. */
34051 /* All classes can load zeros. */
34055 /* Force constants into memory if we are loading a (nonzero) constant into
34056 an MMX or SSE register. This is because there are no MMX/SSE instructions
34057 to load from a constant. */
34062 /* Prefer SSE regs only, if we can use them for math. */
34066 /* Floating-point constants need more complex checks. */
34068 {
34069 /* General regs can load everything. */
34073 /* Floats can load 0 and 1 plus some others. Note that we eliminated
34074 zero above. We only want to wind up preferring 80387 registers if
34075 we plan on doing computation with them. */
34078 {
34079 /* Limit class to non-sse. */
34088 }
34091 }
34093 /* Generally when we see PLUS here, it's the function invariant
34094 (plus soft-fp const_int). Which can only be computed into general
34095 regs. */
34099 /* QImode constants are easy to load, but non-constant QImode data
34100 must go into Q_REGS. */
34102 {
34108 }
34111 }
34113 /* Discourage putting floating-point values in SSE registers unless
34114 SSE math is being used, and likewise for the 387 registers. */
34115 static reg_class_t
34117 {
34120 /* Restrict the output reload class to the register bank that we are doing
34121 math on. If we would like not to return a subset of CLASS, reject this
34122 alternative: if reload cannot do this, it will still use its choice. */
34128 {
34133 else
34135 }
34138 }
34140 static reg_class_t
34143 {
34144 /* Double-word spills from general registers to non-offsettable memory
34145 references (zero-extended addresses) require special handling. */
34151 {
34153 ? CODE_FOR_reload_noff_load
34155 /* Add the cost of moving address to a temporary. */
34159 }
34161 /* QImode spills from non-QI registers require
34162 intermediate register on 32bit targets. */
34167 {
34172 else
34178 /* Return Q_REGS if the operand is in memory. */
34181 }
34183 /* This condition handles corner case where an expression involving
34184 pointers gets vectorized. We're trying to use the address of a
34185 stack slot as a vector initializer.
34187 (set (reg:V2DI 74 [ vect_cst_.2 ])
34188 (vec_duplicate:V2DI (reg/f:DI 20 frame)))
34190 Eventually frame gets turned into sp+offset like this:
34192 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34193 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
34194 (const_int 392 [0x188]))))
34196 That later gets turned into:
34198 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34199 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
34200 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
34202 We'll have the following reload recorded:
34204 Reload 0: reload_in (DI) =
34205 (plus:DI (reg/f:DI 7 sp)
34206 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
34207 reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34208 SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
34209 reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
34210 reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34211 reload_reg_rtx: (reg:V2DI 22 xmm1)
34213 Which isn't going to work since SSE instructions can't handle scalar
34214 additions. Returning GENERAL_REGS forces the addition into integer
34215 register and reload can handle subsequent reloads without problems. */
34223 }
34225 /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
34227 static bool
34229 {
34231 {
34246 }
34249 }
34251 /* If we are copying between general and FP registers, we need a memory
34252 location. The same is true for SSE and MMX registers.
34254 To optimize register_move_cost performance, allow inline variant.
34256 The macro can't work reliably when one of the CLASSES is class containing
34257 registers from multiple units (SSE, MMX, integer). We avoid this by never
34258 combining those units in single alternative in the machine description.
34259 Ensure that this constraint holds to avoid unexpected surprises.
34261 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
34262 enforce these sanity checks. */
34267 {
34276 {
34279 }
34284 /* ??? This is a lie. We do have moves between mmx/general, and for
34285 mmx/sse2. But by saying we need secondary memory we discourage the
34286 register allocator from using the mmx registers unless needed. */
34291 {
34292 /* SSE1 doesn't have any direct moves from other classes. */
34296 /* If the target says that inter-unit moves are more expensive
34297 than moving through memory, then don't generate them. */
34302 /* Between SSE and general, we have moves no larger than word size. */
34305 }
34308 }
34310 bool
34313 {
34315 }
34317 /* Implement the TARGET_CLASS_MAX_NREGS hook.
34319 On the 80386, this is the size of MODE in words,
34320 except in the FP regs, where a single reg is always enough. */
34322 static unsigned char
34324 {
34326 {
34331 else
34333 }
34334 else
34335 {
34338 else
34340 }
34341 }
34343 /* Return true if the registers in CLASS cannot represent the change from
34344 modes FROM to TO. */
34346 bool
34349 {
34353 /* x87 registers can't do subreg at all, as all values are reformatted
34354 to extended precision. */
34359 {
34360 /* Vector registers do not support QI or HImode loads. If we don't
34361 disallow a change to these modes, reload will assume it's ok to
34362 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
34363 the vec_dupv4hi pattern. */
34367 /* Vector registers do not support subreg with nonzero offsets, which
34368 are otherwise valid for integer registers. Since we can't see
34369 whether we have a nonzero offset from here, prohibit all
34370 nonparadoxical subregs changing size. */
34373 }
34376 }
34378 /* Return the cost of moving data of mode M between a
34379 register and memory. A value of 2 is the default; this cost is
34380 relative to those in `REGISTER_MOVE_COST'.
34382 This function is used extensively by register_move_cost that is used to
34383 build tables at startup. Make it inline in this case.
34384 When IN is 2, return maximum of in and out move cost.
34386 If moving between registers and memory is more expensive than
34387 between two registers, you should define this macro to express the
34388 relative cost.
34390 Model also increased moving costs of QImode registers in non
34391 Q_REGS classes.
34392 */
34396 {
34399 {
34402 {
34414 }
34418 }
34420 {
34423 {
34435 }
34439 }
34441 {
34444 {
34453 }
34457 }
34459 {
34462 {
34468 else
34473 }
34474 else
34475 {
34480 else
34482 }
34489 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
34496 else
34500 }
34501 }
34503 static int
34506 {
34508 }
34511 /* Return the cost of moving data from a register in class CLASS1 to
34512 one in class CLASS2.
34514 It is not required that the cost always equal 2 when FROM is the same as TO;
34515 on some machines it is expensive to move between registers if they are not
34516 general registers. */
34518 static int
34520 reg_class_t class2_i)
34521 {
34525 /* In case we require secondary memory, compute cost of the store followed
34526 by load. In order to avoid bad register allocation choices, we need
34527 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
34530 {
34536 /* In case of copying from general_purpose_register we may emit multiple
34537 stores followed by single load causing memory size mismatch stall.
34538 Count this as arbitrarily high cost of 20. */
34543 /* In the case of FP/MMX moves, the registers actually overlap, and we
34544 have to switch modes in order to treat them differently. */
34550 }
34552 /* Moves between SSE/MMX and integer unit are expensive. */
34556 /* ??? By keeping returned value relatively high, we limit the number
34557 of moves between integer and MMX/SSE registers for all targets.
34558 Additionally, high value prevents problem with x86_modes_tieable_p(),
34559 where integer modes in MMX/SSE registers are not tieable
34560 because of missing QImode and HImode moves to, from or between
34561 MMX/SSE registers. */
34571 }
34573 /* Return TRUE if hard register REGNO can hold a value of machine-mode
34574 MODE. */
34576 bool
34578 {
34579 /* Flags and only flags can only hold CCmode values. */
34589 {
34590 /* We implement the move patterns for all vector modes into and
34591 out of SSE registers, even when no operation instructions
34592 are available. OImode move is available only when AVX is
34593 enabled. */
34600 }
34602 {
34603 /* We implement the move patterns for 3DNOW modes even in MMX mode,
34604 so if the register is available at all, then we can move data of
34605 the given mode into or out of it. */
34608 }
34611 {
34612 /* Take care for QImode values - they can be in non-QI regs,
34613 but then they do cause partial register stalls. */
34618 /* LRA checks if the hard register is OK for the given mode.
34619 QImode values can live in non-QI regs, so we allow all
34620 registers here. */
34624 }
34625 /* We handle both integer and floats in the general purpose registers. */
34632 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
34633 on to use that value in smaller contexts, this can easily force a
34634 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
34635 supporting DImode, allow it. */
34640 }
34642 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
34643 tieable integer mode. */
34645 static bool
34647 {
34649 {
34662 }
34663 }
34665 /* Return true if MODE1 is accessible in a register that can hold MODE2
34666 without copying. That is, all register classes that can hold MODE2
34667 can also hold MODE1. */
34669 bool
34671 {
34679 /* MODE2 being XFmode implies fp stack or general regs, which means we
34680 can tie any smaller floating point modes to it. Note that we do not
34681 tie this with TFmode. */
34685 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
34686 that we can tie it with SFmode. */
34690 /* If MODE2 is only appropriate for an SSE register, then tie with
34691 any other mode acceptable to SSE registers. */
34701 /* If MODE2 is appropriate for an MMX register, then tie
34702 with any other mode acceptable to MMX registers. */
34709 }
34711 /* Return the cost of moving between two registers of mode MODE. */
34713 static int
34715 {
34719 {
34750 }
34752 /* Return the cost of moving between two registers of mode MODE,
34753 assuming that the move will be in pieces of at most UNITS bytes. */
34755 }
34757 /* Compute a (partial) cost for rtx X. Return true if the complete
34758 cost has been computed, and false if subexpressions should be
34759 scanned. In either case, *TOTAL contains the cost result. */
34761 static bool
34764 {
34771 {
34775 {
34778 }
34790 && (!TARGET_64BIT
34795 else
34801 {
34804 }
34806 {
34816 }
34818 {
34821 {
34830 }
34831 }
34832 /* Fall back to (MEM (SYMBOL_REF)), since that's where
34833 it'll probably end up. Add a penalty for size. */
34840 /* The zero extensions is often completely free on x86_64, so make
34841 it as cheap as possible. */
34847 else
34859 {
34862 {
34865 }
34868 {
34871 }
34872 }
34873 /* FALLTHRU */
34880 {
34881 /* ??? Should be SSE vector operation cost. */
34882 /* At least for published AMD latencies, this really is the same
34883 as the latency for a simple fpu operation like fabs. */
34884 /* V*QImode is emulated with 1-11 insns. */
34886 {
34889 {
34890 /* For XOP we use vpshab, which requires a broadcast of the
34891 value to the variable shift insn. For constants this
34892 means a V16Q const in mem; even when we can perform the
34893 shift with one insn set the cost to prefer paddb. */
34895 {
34900 }
34902 }
34906 }
34907 else
34909 }
34911 {
34913 {
34916 else
34918 }
34919 else
34920 {
34923 else
34925 }
34926 }
34927 else
34928 {
34933 {
34934 /* Return the cost after shift-and truncation. */
34937 }
34938 else
34940 }
34944 {
34945 rtx sub;
34950 /* ??? SSE scalar/vector cost should be used here. */
34951 /* ??? Bald assumption that fma has the same cost as fmul. */
34955 /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
34966 }
34970 {
34971 /* ??? SSE scalar cost should be used here. */
34974 }
34976 {
34979 }
34981 {
34982 /* ??? SSE vector cost should be used here. */
34985 }
34987 {
34988 /* V*QImode is emulated with 7-13 insns. */
34990 {
34997 }
34998 /* V*DImode is emulated with 5-8 insns. */
35000 {
35003 else
35005 }
35006 /* Without sse4.1, we don't have PMULLD; it's emulated with 7
35007 insns, including two PMULUDQ. */
35010 else
35013 }
35014 else
35015 {
35020 {
35023 nbits++;
35024 }
35025 else
35026 /* This is arbitrary. */
35029 /* Compute costs correctly for widening multiplication. */
35033 {
35040 {
35044 else
35046 }
35050 }
35058 }
35065 /* ??? SSE cost should be used here. */
35070 /* ??? SSE vector cost should be used here. */
35072 else
35079 {
35084 {
35087 {
35095 }
35096 }
35099 {
35102 {
35108 }
35109 }
35111 {
35119 }
35120 }
35121 /* FALLTHRU */
35125 {
35126 /* ??? SSE cost should be used here. */
35129 }
35131 {
35134 }
35136 {
35137 /* ??? SSE vector cost should be used here. */
35140 }
35141 /* FALLTHRU */
35148 {
35155 }
35156 /* FALLTHRU */
35160 {
35161 /* ??? SSE cost should be used here. */
35164 }
35166 {
35169 }
35171 {
35172 /* ??? SSE vector cost should be used here. */
35175 }
35176 /* FALLTHRU */
35180 {
35181 /* ??? Should be SSE vector operation cost. */
35182 /* At least for published AMD latencies, this really is the same
35183 as the latency for a simple fpu operation like fabs. */
35185 }
35188 else
35197 {
35198 /* This kind of construct is implemented using test[bwl].
35199 Treat it as if we had an AND. */
35204 }
35214 /* ??? SSE cost should be used here. */
35219 /* ??? SSE vector cost should be used here. */
35225 /* ??? SSE cost should be used here. */
35230 /* ??? SSE vector cost should be used here. */
35243 /* ??? Assume all of these vector manipulation patterns are
35244 recognizable. In which case they all pretty much have the
35245 same cost. */
35251 }
35252 }
35254 #if TARGET_MACHO
35258 /* Given a symbol name and its associated stub, write out the
35259 definition of the stub. */
35261 void
35263 {
35268 /* For 64-bit we shouldn't get here. */
35271 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
35288 else
35295 {
35297 }
35299 {
35300 /* PIC stub. */
35301 /* 25-byte PIC stub using "CALL get_pc_thunk". */
35307 }
35308 else
35311 /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
35312 it needs no stub-binding-helper. */
35319 {
35322 }
35323 else
35328 /* N.B. Keep the correspondence of these
35329 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
35330 old-pic/new-pic/non-pic stubs; altering this will break
35331 compatibility with existing dylibs. */
35333 {
35334 /* 25-byte PIC stub using "CALL get_pc_thunk". */
35336 }
35337 else
35338 /* 16-byte -mdynamic-no-pic stub. */
35344 }
35347 /* Order the registers for register allocator. */
35349 void
35351 {
35355 /* First allocate the local general purpose registers. */
35360 /* Global general purpose registers. */
35365 /* x87 registers come first in case we are doing FP math
35366 using them. */
35371 /* SSE registers. */
35377 /* x87 registers. */
35385 /* Initialize the rest of array as we do not allocate some registers
35386 at all. */
35389 }
35391 /* Handle a "callee_pop_aggregate_return" attribute; arguments as
35392 in struct attribute_spec handler. */
35393 static tree
35395 tree args,
35398 {
35403 {
35405 name);
35408 }
35410 {
35412 name);
35415 }
35417 {
35418 tree cst;
35422 {
35425 name);
35427 }
35430 {
35433 name);
35435 }
35438 }
35441 }
35443 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
35444 struct attribute_spec.handler. */
35445 static tree
35447 tree args ATTRIBUTE_UNUSED,
35449 {
35454 {
35456 name);
35459 }
35461 /* Can combine regparm with all attributes but fastcall. */
35463 {
35465 {
35467 }
35470 }
35472 {
35474 {
35476 }
35479 }
35482 }
35484 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
35485 struct attribute_spec.handler. */
35486 static tree
35488 tree args ATTRIBUTE_UNUSED,
35490 {
35493 {
35496 }
35497 else
35501 {
35503 name);
35505 }
35511 {
35513 name);
35515 }
35518 }
35520 static tree
35522 tree args ATTRIBUTE_UNUSED,
35524 {
35526 {
35528 name);
35530 }
35532 }
35534 static bool
35536 {
35540 }
35542 /* Returns an expression indicating where the this parameter is
35543 located on entry to the FUNCTION. */
35545 static rtx
35547 {
35553 {
35558 else
35561 }
35566 {
35573 {
35578 }
35579 else
35580 {
35583 {
35589 }
35590 }
35592 }
35596 }
35598 /* Determine whether x86_output_mi_thunk can succeed. */
35600 static bool
35602 HOST_WIDE_INT delta ATTRIBUTE_UNUSED,
35604 {
35605 /* 64-bit can handle anything. */
35609 /* For 32-bit, everything's fine if we have one free register. */
35613 /* Need a free register for vcall_offset. */
35617 /* Need a free register for GOT references. */
35621 /* Otherwise ok. */
35623 }
35625 /* Output the assembler code for a thunk function. THUNK_DECL is the
35626 declaration for the thunk function itself, FUNCTION is the decl for
35627 the target function. DELTA is an immediate constant offset to be
35628 added to THIS. If VCALL_OFFSET is nonzero, the word at
35629 *(*this + vcall_offset) should be added to THIS. */
35631 static void
35635 {
35642 else
35643 {
35649 else
35651 }
35655 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
35656 pull it in now and let DELTA benefit. */
35660 {
35661 /* Put the this parameter into %eax. */
35664 }
35665 else
35668 /* Adjust the this parameter by a fixed constant. */
35670 {
35675 {
35677 {
35681 }
35682 }
35685 }
35687 /* Adjust the this parameter by a value stored in the vtable. */
35689 {
35699 /* Adjust the this parameter. */
35703 {
35707 }
35714 vcall_mem));
35715 else
35717 }
35719 /* If necessary, drop THIS back to its stack slot. */
35725 {
35728 ;
35729 else
35730 {
35734 }
35735 }
35736 else
35737 {
35739 ;
35740 #if TARGET_MACHO
35742 {
35745 }
35747 else
35748 {
35755 }
35756 }
35758 /* Our sibling call patterns do not allow memories, because we have no
35759 predicate that can distinguish between frame and non-frame memory.
35760 For our purposes here, we can get away with (ab)using a jump pattern,
35761 because we're going to do no optimization. */
35764 else
35765 {
35771 {
35777 }
35783 }
35786 /* Emit just enough of rest_of_compilation to get the insns emitted.
35787 Note that use_thunk calls assemble_start_function et al. */
35793 }
35795 static void
35797 {
35799 #if TARGET_MACHO
35801 #endif
35808 }
35810 int
35812 {
35824 }
35826 /* Output assembler code to FILE to increment profiler label # LABELNO
35827 for profiling a function entry. */
35828 void
35830 {
35835 {
35836 #ifndef NO_PROFILE_COUNTERS
35838 #endif
35842 else
35844 }
35846 {
35847 #ifndef NO_PROFILE_COUNTERS
35850 #endif
35852 }
35853 else
35854 {
35855 #ifndef NO_PROFILE_COUNTERS
35858 #endif
35860 }
35861 }
35863 /* We don't have exact information about the insn sizes, but we may assume
35864 quite safely that we are informed about all 1 byte insns and memory
35865 address sizes. This is enough to eliminate unnecessary padding in
35866 99% of cases. */
35868 static int
35870 {
35876 /* Discard alignments we've emit and jump instructions. */
35881 /* Important case - calls are always 5 bytes.
35882 It is common to have many calls in the row. */
35891 /* For normal instructions we rely on get_attr_length being exact,
35892 with a few exceptions. */
35894 {
35898 {
35908 /* Otherwise trust get_attr_length. */
35910 }
35915 }
35918 else
35920 }
35922 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
35924 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
35925 window. */
35927 static void
35929 {
35934 /* Look for all minimal intervals of instructions containing 4 jumps.
35935 The intervals are bounded by START and INSN. NBYTES is the total
35936 size of instructions in the interval including INSN and not including
35937 START. When the NBYTES is smaller than 16 bytes, it is possible
35938 that the end of START and INSN ends up in the same 16byte page.
35940 The smallest offset in the page INSN can start is the case where START
35941 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
35942 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
35943 */
35945 {
35949 {
35955 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
35956 already in the current 16 byte page, because otherwise
35957 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
35958 bytes to reach 16 byte boundary. */
35966 {
35968 {
35972 else
35975 }
35976 }
35978 }
35986 njumps++;
35987 else
35991 {
35995 else
35998 }
36005 {
36012 }
36013 }
36014 }
36015 #endif
36017 /* AMD Athlon works faster
36018 when RET is not destination of conditional jump or directly preceded
36019 by other jump instruction. We avoid the penalty by inserting NOP just
36020 before the RET instructions in such cases. */
36021 static void
36023 {
36024 edge e;
36025 edge_iterator ei;
36028 {
36031 rtx prev;
36041 {
36042 edge e;
36043 edge_iterator ei;
36048 {
36051 }
36052 }
36054 {
36060 /* Empty functions get branch mispredict even when
36061 the jump destination is not visible to us. */
36064 }
36066 {
36069 }
36070 }
36071 }
36073 /* Count the minimum number of instructions in BB. Return 4 if the
36074 number of instructions >= 4. */
36076 static int
36078 {
36079 rtx insn;
36082 /* Count number of instructions in this block. Return 4 if the number
36083 of instructions >= 4. */
36085 {
36086 /* Only happen in exit blocks. */
36094 {
36095 insn_count++;
36098 }
36099 }
36102 }
36105 /* Count the minimum number of instructions in code path in BB.
36106 Return 4 if the number of instructions >= 4. */
36108 static int
36110 {
36111 edge e;
36112 edge_iterator ei;
36115 /* Only bother counting instructions along paths with no
36116 more than 2 basic blocks between entry and exit. Given
36117 that BB has an edge to exit, determine if a predecessor
36118 of BB has an edge from entry. If so, compute the number
36119 of instructions in the predecessor block. If there
36120 happen to be multiple such blocks, compute the minimum. */
36123 {
36124 edge prev_e;
36125 edge_iterator prev_ei;
36128 {
36131 }
36133 {
36135 {
36140 }
36141 }
36142 }
36148 }
36150 /* Pad short function to 4 instructions. */
36152 static void
36154 {
36155 edge e;
36156 edge_iterator ei;
36159 {
36162 {
36165 /* Pad short function. */
36167 {
36170 /* Find epilogue. */
36179 /* Two NOPs count as one instruction. */
36182 }
36183 }
36184 }
36185 }
36187 /* Fix up a Windows system unwinder issue. If an EH region falls through into
36188 the epilogue, the Windows system unwinder will apply epilogue logic and
36189 produce incorrect offsets. This can be avoided by adding a nop between
36190 the last insn that can throw and the first insn of the epilogue. */
36192 static void
36194 {
36195 edge e;
36196 edge_iterator ei;
36199 {
36202 /* Find the beginning of the epilogue. */
36209 /* We only care about preceding insns that can throw. */
36214 /* Do not separate calls from their debug information. */
36220 else
36224 }
36225 }
36227 /* Implement machine specific optimizations. We implement padding of returns
36228 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
36229 static void
36231 {
36232 /* We are freeing block_for_insn in the toplev to keep compatibility
36233 with old MDEP_REORGS that are not CFG based. Recompute it now. */
36240 {
36245 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
36248 #endif
36249 }
36250 }
36252 /* Return nonzero when QImode register that must be represented via REX prefix
36253 is used. */
36254 bool
36256 {
36264 }
36266 /* Return nonzero when P points to register encoded via REX prefix.
36267 Called via for_each_rtx. */
36268 static int
36270 {
36276 }
36278 /* Return true when INSN mentions register that must be encoded using REX
36279 prefix. */
36280 bool
36282 {
36285 }
36287 /* If profitable, negate (without causing overflow) integer constant
36288 of mode MODE at location LOC. Return true in this case. */
36289 bool
36291 {
36292 HOST_WIDE_INT val;
36298 {
36300 /* DImode x86_64 constants must fit in 32 bits. */
36313 }
36315 /* Avoid overflows. */
36321 /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
36322 Exceptions: -128 encodes smaller than 128, so swap sign and op. */
36325 {
36328 }
36331 }
36333 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
36334 optabs would emit if we didn't have TFmode patterns. */
36336 void
36338 {
36372 }
36373 \f
36374 /* AVX2 does support 32-byte integer vector operations,
36375 thus the longest vector we are faced with is V32QImode. */
36376 #define MAX_VECT_LEN 32
36378 struct expand_vec_perm_d
36379 {
36386 };
36392 /* Get a vector mode of the same size as the original but with elements
36393 twice as wide. This is only guaranteed to apply to integral vectors. */
36397 {
36398 /* ??? Rely on the ordering that genmodes.c gives to vectors. */
36403 }
36405 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
36406 with all elements equal to VAR. Return true if successful. */
36408 static bool
36411 {
36415 {
36420 /* FALLTHRU */
36430 {
36433 /* First attempt to recognize VAL as-is. */
36437 {
36438 rtx seq;
36439 /* If that fails, force VAL into a register. */
36450 }
36451 }
36458 {
36459 rtx x;
36466 }
36476 {
36480 permute:
36488 /* Extend to SImode using a paradoxical SUBREG. */
36492 /* Insert the SImode value as low element of a V4SImode vector. */
36500 }
36508 widen:
36509 /* Replicate the value once into the next wider mode and recurse. */
36510 {
36512 rtx x;
36528 }
36532 {
36541 }
36546 }
36547 }
36549 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
36550 whose ONE_VAR element is VAR, and other elements are zero. Return true
36551 if successful. */
36553 static bool
36556 {
36558 rtx new_target;
36563 {
36565 /* For SSE4.1, we normally use vector set. But if the second
36566 element is zero and inter-unit moves are OK, we use movq
36567 instead. */
36569 && !(TARGET_INTER_UNIT_MOVES_TO_VEC
36591 /* Use ix86_expand_vector_set in 64bit mode only. */
36596 }
36599 {
36604 }
36607 {
36612 /* FALLTHRU */
36627 else
36634 {
36635 /* We need to shuffle the value to the correct position, so
36636 create a new pseudo to store the intermediate result. */
36638 /* With SSE2, we can use the integer shuffle insns. */
36640 {
36642 const1_rtx,
36649 }
36651 /* Otherwise convert the intermediate result to V4SFmode and
36652 use the SSE1 shuffle instructions. */
36654 {
36657 }
36658 else
36662 const1_rtx,
36671 }
36686 widen:
36690 /* Zero extend the variable element to SImode and recurse. */
36703 }
36704 }
36706 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
36707 consisting of the values in VALS. It is known that all elements
36708 except ONE_VAR are constants. Return true if successful. */
36710 static bool
36713 {
36723 {
36728 /* For the two element vectors, it's just as easy to use
36729 the general case. */
36733 /* Use ix86_expand_vector_set in 64bit mode only. */
36755 widen:
36756 /* There's no way to set one QImode entry easily. Combine
36757 the variable value with its adjacent constant value, and
36758 promote to an HImode set. */
36761 {
36766 }
36767 else
36768 {
36771 }
36785 }
36790 }
36792 /* A subroutine of ix86_expand_vector_init_general. Use vector
36793 concatenate to handle the most general case: all values variable,
36794 and none identical. */
36796 static void
36799 {
36802 rtvec v;
36806 {
36809 {
36842 }
36855 {
36870 }
36875 {
36886 }
36889 half:
36890 /* FIXME: We process inputs backward to help RA. PR 36222. */
36894 {
36899 }
36903 {
36906 {
36910 }
36913 }
36914 else
36920 }
36921 }
36923 /* A subroutine of ix86_expand_vector_init_general. Use vector
36924 interleave to handle the most general case: all values variable,
36925 and none identical. */
36927 static void
36930 {
36939 {
36960 }
36963 {
36964 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
36968 /* Insert the SImode value as low element of V4SImode vector. */
36972 op0),
36974 const1_rtx);
36977 /* Cast the V4SImode vector back to a vector in orignal mode. */
36981 /* Load even elements into the second position. */
36985 const1_rtx));
36987 /* Cast vector to FIRST_IMODE vector. */
36990 }
36992 /* Interleave low FIRST_IMODE vectors. */
36994 {
36998 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
37001 }
37003 /* Interleave low SECOND_IMODE vectors. */
37005 {
37008 {
37013 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
37014 vector. */
37017 }
37020 /* FALLTHRU */
37027 /* Cast the SECOND_IMODE vector back to a vector on original
37028 mode. */
37035 }
37036 }
37038 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
37039 all values variable, and none identical. */
37041 static void
37044 {
37050 {
37055 /* FALLTHRU */
37079 half:
37096 /* FALLTHRU */
37102 /* Don't use ix86_expand_vector_init_interleave if we can't
37103 move from GPR to SSE register directly. */
37119 }
37121 {
37133 {
37137 {
37143 else
37144 {
37149 }
37150 }
37153 }
37158 {
37164 }
37166 {
37172 }
37173 else
37175 }
37176 }
37178 /* Initialize vector TARGET via VALS. Suppress the use of MMX
37179 instructions unless MMX_OK is true. */
37181 void
37183 {
37190 rtx x;
37193 {
37203 }
37205 /* Constants are best loaded from the constant pool. */
37207 {
37210 }
37212 /* If all values are identical, broadcast the value. */
37218 /* Values where only one field is non-constant are best loaded from
37219 the pool and overwritten via move later. */
37221 {
37225 one_var))
37230 }
37233 }
37235 void
37237 {
37242 rtx tmp;
37244 = {
37251 };
37253 = {
37260 };
37264 {
37268 {
37273 else
37277 }
37289 else
37295 {
37298 /* For the two element vectors, we implement a VEC_CONCAT with
37299 the extraction of the other element. */
37306 else
37311 }
37320 {
37326 /* tmp = target = A B C D */
37328 /* target = A A B B */
37330 /* target = X A B B */
37332 /* target = A X C D */
37339 /* tmp = target = A B C D */
37341 /* tmp = X B C D */
37343 /* target = A B X D */
37350 /* tmp = target = A B C D */
37352 /* tmp = X B C D */
37354 /* target = A B X D */
37362 }
37370 /* Element 0 handled by vec_merge below. */
37372 {
37375 }
37378 {
37379 /* With SSE2, use integer shuffles to swap element 0 and ELT,
37380 store into element 0, then shuffle them back. */
37397 }
37398 else
37399 {
37400 /* For SSE1, we have to reuse the V4SF code. */
37403 }
37456 half:
37457 /* Compute offset. */
37463 /* Extract the half. */
37467 /* Put val in tmp at elt. */
37470 /* Put it back. */
37476 }
37479 {
37483 }
37484 else
37485 {
37494 }
37495 }
37497 void
37499 {
37503 rtx tmp;
37506 {
37511 /* FALLTHRU */
37524 {
37544 }
37556 {
37558 {
37578 }
37582 }
37583 else
37584 {
37585 /* For SSE1, we have to reuse the V4SF code. */
37589 }
37605 {
37609 else
37613 }
37618 {
37622 else
37626 }
37631 {
37635 else
37639 }
37644 {
37648 else
37652 }
37657 {
37661 else
37665 }
37670 {
37674 else
37678 }
37682 /* ??? Could extract the appropriate HImode element and shift. */
37685 }
37688 {
37692 /* Let the rtl optimizers know about the zero extension performed. */
37694 {
37697 }
37700 }
37701 else
37702 {
37709 }
37710 }
37712 /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
37713 to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
37714 The upper bits of DEST are undefined, though they shouldn't cause
37715 exceptions (some bits from src or all zeros are ok). */
37717 static void
37719 {
37720 rtx tem;
37722 {
37726 else
37744 else
37751 else
37762 const1_rtx);
37763 else
37770 }
37772 }
37774 /* Expand a vector reduction. FN is the binary pattern to reduce;
37775 DEST is the destination; IN is the input vector. */
37777 void
37779 {
37784 /* SSE4 has a special instruction for V8HImode UMIN reduction. */
37788 {
37791 }
37796 {
37801 else
37805 }
37806 }
37807 \f
37808 /* Target hook for scalar_mode_supported_p. */
37809 static bool
37811 {
37816 else
37818 }
37820 /* Implements target hook vector_mode_supported_p. */
37821 static bool
37823 {
37835 }
37837 /* Target hook for c_mode_for_suffix. */
37838 static enum machine_mode
37840 {
37847 }
37849 /* Worker function for TARGET_MD_ASM_CLOBBERS.
37851 We do this in the new i386 backend to maintain source compatibility
37852 with the old cc0-based compiler. */
37854 static tree
37856 tree inputs ATTRIBUTE_UNUSED,
37857 tree clobbers)
37858 {
37860 clobbers);
37862 clobbers);
37864 }
37866 /* Implements target vector targetm.asm.encode_section_info. */
37868 static void ATTRIBUTE_UNUSED
37870 {
37877 }
37879 /* Worker function for REVERSE_CONDITION. */
37881 enum rtx_code
37883 {
37887 }
37889 /* Output code to perform an x87 FP register move, from OPERANDS[1]
37890 to OPERANDS[0]. */
37894 {
37896 {
37899 {
37903 }
37907 }
37909 {
37913 else
37914 {
37915 /* There is no non-popping store to memory for XFmode.
37916 So if we need one, follow the store with a load. */
37919 else
37921 }
37922 }
37923 else
37925 }
37927 /* Output code to perform a conditional jump to LABEL, if C2 flag in
37928 FP status register is set. */
37930 void
37932 {
37934 rtx temp;
37939 {
37944 }
37945 else
37946 {
37951 }
37955 pc_rtx);
37960 }
37962 /* Output code to perform a log1p XFmode calculation. */
37965 {
37971 rtx test;
37977 XFmode));
37991 }
37993 /* Emit code for round calculation. */
37995 {
38002 rtx insn;
38007 {
38019 }
38022 {
38043 }
38051 /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
38053 /* scratch = fxam(op1) */
38056 UNSPEC_FXAM)));
38057 /* e1 = fabs(op1) */
38060 /* e2 = e1 + 0.5 */
38065 /* res = floor(e2) */
38067 {
38072 }
38073 else
38077 {
38080 {
38087 UNSPEC_TRUNC_NOOP)));
38088 }
38104 }
38106 /* flags = signbit(a) */
38109 /* if (flags) then res = -res */
38113 pc_rtx);
38124 }
38126 /* Output code to perform a Newton-Rhapson approximation of a single precision
38127 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
38130 {
38138 /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
38142 /* x0 = rcp(b) estimate */
38145 UNSPEC_RCP)));
38146 /* e0 = x0 * b */
38150 /* e0 = x0 * e0 */
38154 /* e1 = x0 + x0 */
38158 /* x1 = e1 - e0 */
38162 /* res = a * x1 */
38165 }
38167 /* Output code to perform a Newton-Rhapson approximation of a
38168 single precision floating point [reciprocal] square root. */
38172 {
38174 REAL_VALUE_TYPE r;
38189 {
38192 }
38194 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
38195 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
38199 /* x0 = rsqrt(a) estimate */
38202 UNSPEC_RSQRT)));
38204 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
38206 {
38218 }
38220 /* e0 = x0 * a */
38223 /* e1 = e0 * x0 */
38227 /* e2 = e1 - 3. */
38234 /* e3 = -.5 * x0 */
38237 else
38238 /* e3 = -.5 * e0 */
38241 /* ret = e2 * e3 */
38244 }
38246 #ifdef TARGET_SOLARIS
38247 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
38249 static void
38251 tree decl)
38252 {
38253 /* With Binutils 2.15, the "@unwind" marker must be specified on
38254 every occurrence of the ".eh_frame" section, not just the first
38255 one. */
38258 {
38262 }
38264 #ifndef USE_GAS
38266 {
38269 }
38270 #endif
38273 }
38276 /* Return the mangling of TYPE if it is an extended fundamental type. */
38280 {
38288 {
38290 /* __float128 is "g". */
38293 /* "long double" or __float80 is "e". */
38297 }
38298 }
38300 /* For 32-bit code we can save PIC register setup by using
38301 __stack_chk_fail_local hidden function instead of calling
38302 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
38303 register, so it is better to call __stack_chk_fail directly. */
38305 static tree ATTRIBUTE_UNUSED
38307 {
38308 return TARGET_64BIT
38311 }
38313 /* Select a format to encode pointers in exception handling data. CODE
38314 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
38315 true if the symbol may be affected by dynamic relocations.
38317 ??? All x86 object file formats are capable of representing this.
38318 After all, the relocation needed is the same as for the call insn.
38319 Whether or not a particular assembler allows us to enter such, I
38320 guess we'll have to see. */
38321 int
38323 {
38325 {
38332 }
38337 }
38338 \f
38339 /* Expand copysign from SIGN to the positive value ABS_VALUE
38340 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
38341 the sign-bit. */
38342 static void
38344 {
38348 {
38355 else
38360 {
38361 /* We need to generate a scalar mode mask in this case. */
38366 }
38367 }
38368 else
38374 }
38376 /* Expand fabs (OP0) and return a new rtx that holds the result. The
38377 mask for masking out the sign-bit is stored in *SMASK, if that is
38378 non-null. */
38379 static rtx
38381 {
38390 else
38394 {
38395 /* We need to generate a scalar mode mask in this case. */
38400 }
38408 }
38410 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
38411 swapping the operands if SWAP_OPERANDS is true. The expanded
38412 code is a forward jump to a newly created label in case the
38413 comparison is true. The generated label rtx is returned. */
38414 static rtx
38417 {
38421 {
38425 }
38438 }
38440 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
38441 using comparison code CODE. Operands are swapped for the comparison if
38442 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
38443 static rtx
38446 {
38452 {
38456 }
38463 }
38465 /* Generate and return a rtx of mode MODE for 2**n where n is the number
38466 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
38467 static rtx
38469 {
38470 REAL_VALUE_TYPE TWO52r;
38471 rtx TWO52;
38478 }
38480 /* Expand SSE sequence for computing lround from OP1 storing
38481 into OP0. */
38482 void
38484 {
38485 /* C code for the stuff we're doing below:
38486 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
38487 return (long)tmp;
38488 */
38492 rtx adj;
38494 /* load nextafter (0.5, 0.0) */
38499 /* adj = copysign (0.5, op1) */
38503 /* adj = op1 + adj */
38506 /* op0 = (imode)adj */
38508 }
38510 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
38511 into OPERAND0. */
38512 void
38514 {
38515 /* C code for the stuff we're doing below (for do_floor):
38516 xi = (long)op1;
38517 xi -= (double)xi > op1 ? 1 : 0;
38518 return xi;
38519 */
38524 /* reg = (long)op1 */
38528 /* freg = (double)reg */
38532 /* ireg = (freg > op1) ? ireg - 1 : ireg */
38543 }
38545 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
38546 result in OPERAND0. */
38547 void
38549 {
38550 /* C code for the stuff we're doing below:
38551 xa = fabs (operand1);
38552 if (!isless (xa, 2**52))
38553 return operand1;
38554 xa = xa + 2**52 - 2**52;
38555 return copysign (xa, operand1);
38556 */
38563 /* xa = abs (operand1) */
38566 /* if (!isless (xa, TWO52)) goto label; */
38579 }
38581 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
38582 into OPERAND0. */
38583 void
38585 {
38586 /* C code for the stuff we expand below.
38587 double xa = fabs (x), x2;
38588 if (!isless (xa, TWO52))
38589 return x;
38590 xa = xa + TWO52 - TWO52;
38591 x2 = copysign (xa, x);
38592 Compensate. Floor:
38593 if (x2 > x)
38594 x2 -= 1;
38595 Compensate. Ceil:
38596 if (x2 < x)
38597 x2 -= -1;
38598 return x2;
38599 */
38605 /* Temporary for holding the result, initialized to the input
38606 operand to ease control flow. */
38610 /* xa = abs (operand1) */
38613 /* if (!isless (xa, TWO52)) goto label; */
38616 /* xa = xa + TWO52 - TWO52; */
38620 /* xa = copysign (xa, operand1) */
38623 /* generate 1.0 or -1.0 */
38628 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
38632 /* We always need to subtract here to preserve signed zero. */
38641 }
38643 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
38644 into OPERAND0. */
38645 void
38647 {
38648 /* C code for the stuff we expand below.
38649 double xa = fabs (x), x2;
38650 if (!isless (xa, TWO52))
38651 return x;
38652 x2 = (double)(long)x;
38653 Compensate. Floor:
38654 if (x2 > x)
38655 x2 -= 1;
38656 Compensate. Ceil:
38657 if (x2 < x)
38658 x2 += 1;
38659 if (HONOR_SIGNED_ZEROS (mode))
38660 return copysign (x2, x);
38661 return x2;
38662 */
38668 /* Temporary for holding the result, initialized to the input
38669 operand to ease control flow. */
38673 /* xa = abs (operand1) */
38676 /* if (!isless (xa, TWO52)) goto label; */
38679 /* xa = (double)(long)x */
38684 /* generate 1.0 */
38687 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
38702 }
38704 /* Expand SSE sequence for computing round from OPERAND1 storing
38705 into OPERAND0. Sequence that works without relying on DImode truncation
38706 via cvttsd2siq that is only available on 64bit targets. */
38707 void
38709 {
38710 /* C code for the stuff we expand below.
38711 double xa = fabs (x), xa2, x2;
38712 if (!isless (xa, TWO52))
38713 return x;
38714 Using the absolute value and copying back sign makes
38715 -0.0 -> -0.0 correct.
38716 xa2 = xa + TWO52 - TWO52;
38717 Compensate.
38718 dxa = xa2 - xa;
38719 if (dxa <= -0.5)
38720 xa2 += 1;
38721 else if (dxa > 0.5)
38722 xa2 -= 1;
38723 x2 = copysign (xa2, x);
38724 return x2;
38725 */
38731 /* Temporary for holding the result, initialized to the input
38732 operand to ease control flow. */
38736 /* xa = abs (operand1) */
38739 /* if (!isless (xa, TWO52)) goto label; */
38742 /* xa2 = xa + TWO52 - TWO52; */
38746 /* dxa = xa2 - xa; */
38749 /* generate 0.5, 1.0 and -0.5 */
38755 /* Compensate. */
38757 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
38762 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
38768 /* res = copysign (xa2, operand1) */
38775 }
38777 /* Expand SSE sequence for computing trunc from OPERAND1 storing
38778 into OPERAND0. */
38779 void
38781 {
38782 /* C code for SSE variant we expand below.
38783 double xa = fabs (x), x2;
38784 if (!isless (xa, TWO52))
38785 return x;
38786 x2 = (double)(long)x;
38787 if (HONOR_SIGNED_ZEROS (mode))
38788 return copysign (x2, x);
38789 return x2;
38790 */
38796 /* Temporary for holding the result, initialized to the input
38797 operand to ease control flow. */
38801 /* xa = abs (operand1) */
38804 /* if (!isless (xa, TWO52)) goto label; */
38807 /* x = (double)(long)x */
38819 }
38821 /* Expand SSE sequence for computing trunc from OPERAND1 storing
38822 into OPERAND0. */
38823 void
38825 {
38829 /* C code for SSE variant we expand below.
38830 double xa = fabs (x), x2;
38831 if (!isless (xa, TWO52))
38832 return x;
38833 xa2 = xa + TWO52 - TWO52;
38834 Compensate:
38835 if (xa2 > xa)
38836 xa2 -= 1.0;
38837 x2 = copysign (xa2, x);
38838 return x2;
38839 */
38843 /* Temporary for holding the result, initialized to the input
38844 operand to ease control flow. */
38848 /* xa = abs (operand1) */
38851 /* if (!isless (xa, TWO52)) goto label; */
38854 /* res = xa + TWO52 - TWO52; */
38859 /* generate 1.0 */
38862 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
38870 /* res = copysign (res, operand1) */
38877 }
38879 /* Expand SSE sequence for computing round from OPERAND1 storing
38880 into OPERAND0. */
38881 void
38883 {
38884 /* C code for the stuff we're doing below:
38885 double xa = fabs (x);
38886 if (!isless (xa, TWO52))
38887 return x;
38888 xa = (double)(long)(xa + nextafter (0.5, 0.0));
38889 return copysign (xa, x);
38890 */
38896 /* Temporary for holding the result, initialized to the input
38897 operand to ease control flow. */
38905 /* load nextafter (0.5, 0.0) */
38910 /* xa = xa + 0.5 */
38914 /* xa = (double)(int64_t)xa */
38919 /* res = copysign (xa, operand1) */
38926 }
38928 /* Expand SSE sequence for computing round
38929 from OP1 storing into OP0 using sse4 round insn. */
38930 void
38932 {
38941 {
38952 }
38954 /* round (a) = trunc (a + copysign (0.5, a)) */
38956 /* load nextafter (0.5, 0.0) */
38962 /* e1 = copysign (0.5, op1) */
38966 /* e2 = op1 + e1 */
38969 /* res = trunc (e2) */
38974 }
38975 \f
38977 /* Table of valid machine attributes. */
38979 {
38980 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
38981 affects_type_identity } */
38982 /* Stdcall attribute says callee is responsible for popping arguments
38983 if they are not variable. */
38986 /* Fastcall attribute says callee is responsible for popping arguments
38987 if they are not variable. */
38990 /* Thiscall attribute says callee is responsible for popping arguments
38991 if they are not variable. */
38994 /* Cdecl attribute says the callee is a normal C declaration */
38997 /* Regparm attribute specifies how many integer arguments are to be
38998 passed in registers. */
39001 /* Sseregparm attribute says we are using x86_64 calling conventions
39002 for FP arguments. */
39005 /* The transactional memory builtins are implicitly regparm or fastcall
39006 depending on the ABI. Override the generic do-nothing attribute that
39007 these builtins were declared with. */
39010 /* force_align_arg_pointer says this function realigns the stack at entry. */
39013 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
39018 #endif
39023 #ifdef SUBTARGET_ATTRIBUTE_TABLE
39024 SUBTARGET_ATTRIBUTE_TABLE,
39025 #endif
39026 /* ms_abi and sysv_abi calling convention function attributes. */
39033 /* End element. */
39035 };
39037 /* Implement targetm.vectorize.builtin_vectorization_cost. */
39038 static int
39040 tree vectype,
39042 {
39046 {
39091 }
39092 }
39094 /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
39095 insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
39096 insn every time. */
39100 /* Initialize vselect_insn. */
39102 static void
39104 {
39106 rtx x;
39117 }
39119 /* Construct (set target (vec_select op0 (parallel perm))) and
39120 return true if that's a valid instruction in the active ISA. */
39122 static bool
39125 {
39151 }
39153 /* Similar, but generate a vec_concat from op0 and op1 as well. */
39155 static bool
39159 {
39161 rtx x;
39176 }
39178 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39179 in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
39181 static bool
39183 {
39192 ;
39194 ;
39196 ;
39197 else
39200 /* This is a blend, not a permute. Elements must stay in their
39201 respective lanes. */
39203 {
39207 }
39212 /* ??? Without SSE4.1, we could implement this with and/andn/or. This
39213 decision should be extracted elsewhere, so that we only try that
39214 sequence once all budget==3 options have been tried. */
39221 {
39245 /* See if bytes move in pairs so we can use pblendw with
39246 an immediate argument, rather than pblendvb with a vector
39247 argument. */
39250 {
39251 use_pblendvb:
39255 finish_pblendvb:
39261 else
39264 }
39269 /* FALLTHRU */
39271 do_subreg:
39278 /* See if bytes move in pairs. If not, vpblendvb must be used. */
39282 /* See if bytes move in quadruplets. If yes, vpblendd
39283 with immediate can be used. */
39288 {
39289 /* See if bytes move the same in both lanes. If yes,
39290 vpblendw with immediate can be used. */
39295 /* Use vpblendw. */
39300 }
39302 /* Use vpblendd. */
39309 /* See if words move in pairs. If yes, vpblendd can be used. */
39314 {
39315 /* See if words move the same in both lanes. If not,
39316 vpblendvb must be used. */
39319 {
39320 /* Use vpblendvb. */
39330 }
39332 /* Use vpblendw. */
39336 }
39338 /* Use vpblendd. */
39345 /* Use vpblendd. */
39353 }
39355 /* This matches five different patterns with the different modes. */
39361 }
39363 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39364 in terms of the variable form of vpermilps.
39366 Note that we will have already failed the immediate input vpermilps,
39367 which requires that the high and low part shuffle be identical; the
39368 variable form doesn't require that. */
39370 static bool
39372 {
39379 /* We can only permute within the 128-bit lane. */
39381 {
39385 }
39391 {
39394 /* Within each 128-bit lane, the elements of op0 are numbered
39395 from 0 and the elements of op1 are numbered from 4. */
39402 }
39409 }
39411 /* Return true if permutation D can be performed as VMODE permutation
39412 instead. */
39414 static bool
39416 {
39431 else
39437 }
39439 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39440 in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
39442 static bool
39444 {
39453 {
39455 {
39458 {
39462 /* Use vperm2i128 insn. The pattern uses
39463 V4DImode instead of V2TImode. */
39472 }
39474 }
39475 }
39476 else
39477 {
39479 {
39482 }
39484 {
39488 /* V4DImode should be already handled through
39489 expand_vselect by vpermq instruction. */
39496 {
39497 /* First see if vpermq can be used for
39498 V8SImode/V16HImode/V32QImode. */
39500 {
39508 }
39510 /* Next see if vpermd can be used. */
39513 }
39514 /* Or if vpermps can be used. */
39519 {
39520 /* vpshufb only works intra lanes, it is not
39521 possible to shuffle bytes in between the lanes. */
39525 }
39526 }
39527 else
39529 }
39537 else
39538 {
39544 else
39548 {
39552 }
39553 }
39562 {
39569 else
39571 }
39572 else
39573 {
39576 }
39579 }
39581 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to instantiate D
39582 in a single instruction. */
39584 static bool
39586 {
39590 /* Check plain VEC_SELECT first, because AVX has instructions that could
39591 match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
39592 input where SEL+CONCAT may not. */
39594 {
39600 {
39606 }
39609 {
39613 }
39615 {
39616 /* Use vpbroadcast{b,w,d}. */
39619 {
39638 /* For other modes prefer other shuffles this function creates. */
39640 }
39642 {
39646 }
39647 }
39652 /* There are plenty of patterns in sse.md that are written for
39653 SEL+CONCAT and are not replicated for a single op. Perhaps
39654 that should be changed, to avoid the nastiness here. */
39656 /* Recognize interleave style patterns, which means incrementing
39657 every other permutation operand. */
39659 {
39662 }
39667 /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
39669 {
39671 {
39676 }
39681 }
39682 }
39684 /* Finally, try the fully general two operand permute. */
39689 /* Recognize interleave style patterns with reversed operands. */
39691 {
39693 {
39697 else
39700 }
39705 }
39707 /* Try the SSE4.1 blend variable merge instructions. */
39711 /* Try one of the AVX vpermil variable permutations. */
39715 /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
39716 vpshufb, vpermd, vpermps or vpermq variable permutation. */
39721 }
39723 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39724 in terms of a pair of pshuflw + pshufhw instructions. */
39726 static bool
39728 {
39736 /* The two permutations only operate in 64-bit lanes. */
39747 /* Emit the pshuflw. */
39754 /* Emit the pshufhw. */
39762 }
39764 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
39765 the permutation using the SSSE3 palignr instruction. This succeeds
39766 when all of the elements in PERM fit within one vector and we merely
39767 need to shift them down so that a single vector permutation has a
39768 chance to succeed. */
39770 static bool
39772 {
39776 rtx shift;
39778 /* Even with AVX, palignr only operates on 128-bit vectors. */
39784 {
39790 }
39794 /* Given that we have SSSE3, we know we'll be able to implement the
39795 single operand permutation after the palignr with pshufb. */
39809 {
39814 }
39816 /* Test for the degenerate case where the alignment by itself
39817 produces the desired permutation. */
39825 }
39829 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
39830 a two vector permutation into a single vector permutation by using
39831 an interleave operation to merge the vectors. */
39833 static bool
39835 {
39840 rtx seq;
39844 {
39847 }
39849 {
39852 /* For 32-byte modes allow even d->one_operand_p.
39853 The lack of cross-lane shuffling in some instructions
39854 might prevent a single insn shuffle. */
39857 /* If expand_vec_perm_interleave3 can expand this into
39858 a 3 insn sequence, give up and let it be expanded as
39859 3 insn sequence. While that is one insn longer,
39860 it doesn't need a memory operand and in the common
39861 case that both interleave low and high permutations
39862 with the same operands are adjacent needs 4 insns
39863 for both after CSE. */
39866 }
39867 else
39870 /* Examine from whence the elements come. */
39879 {
39882 /* Split the two input vectors into 4 halves. */
39888 /* If the elements from the low halves use interleave low, and similarly
39889 for interleave high. If the elements are from mis-matched halves, we
39890 can use shufps for V4SF/V4SI or do a DImode shuffle. */
39892 {
39893 /* punpckl* */
39895 {
39900 }
39903 }
39905 {
39906 /* punpckh* */
39908 {
39913 }
39916 }
39918 {
39919 /* shufps */
39921 {
39926 }
39928 {
39929 /* shufpd */
39934 }
39935 }
39937 {
39938 /* shufps */
39940 {
39945 }
39947 {
39948 /* shufpd */
39953 }
39954 }
39955 else
39957 }
39958 else
39959 {
39964 /* Split the two input vectors into 8 quarters. */
39970 {
39973 }
39976 {
39980 }
39982 {
39985 }
39988 {
39990 {
39991 /* Attempt to increase the likelihood that dfinal
39992 shuffle will be intra-lane. */
39996 }
39998 /* vperm2f128 or vperm2i128. */
40000 {
40005 }
40010 {
40014 {
40017 }
40018 }
40019 }
40024 {
40025 /* vpunpckl* */
40027 {
40036 }
40037 }
40040 {
40041 /* vpunpckh* */
40043 {
40052 }
40053 }
40054 else
40056 }
40058 /* Use the remapping array set up above to move the elements from their
40059 swizzled locations into their final destinations. */
40062 {
40065 /* If same_halves is true, both halves of the remapped vector are the
40066 same. Avoid cross-lane accesses if possible. */
40068 {
40071 }
40072 else
40074 }
40080 /* Test if the final remap can be done with a single insn. For V4SFmode or
40081 V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
40094 {
40098 }
40105 }
40107 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40108 a single vector cross-lane permutation into vpermq followed
40109 by any of the single insn permutations. */
40111 static bool
40113 {
40127 {
40130 }
40133 {
40138 }
40151 {
40158 }
40165 {
40170 ;
40173 else
40175 }
40184 }
40186 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to expand
40187 a vector permutation using two instructions, vperm2f128 resp.
40188 vperm2i128 followed by any single in-lane permutation. */
40190 static bool
40192 {
40206 /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
40207 immediate. For perm < 16 the second permutation uses
40208 d->op0 as first operand, for perm >= 16 it uses d->op1
40209 as first operand. The second operand is the result of
40210 vperm2[fi]128. */
40212 {
40213 /* Ignore permutations which do not move anything cross-lane. */
40215 {
40216 /* The second shuffle for e.g. V4DFmode has
40217 0123 and ABCD operands.
40218 Ignore AB23, as 23 is already in the second lane
40219 of the first operand. */
40221 /* And 01CD, as 01 is in the first lane of the first
40222 operand. */
40224 /* And 4567, as then the vperm2[fi]128 doesn't change
40225 anything on the original 4567 second operand. */
40227 }
40228 else
40229 {
40230 /* The second shuffle for e.g. V4DFmode has
40231 4567 and ABCD operands.
40232 Ignore AB67, as 67 is already in the second lane
40233 of the first operand. */
40235 /* And 45CD, as 45 is in the first lane of the first
40236 operand. */
40238 /* And 0123, as then the vperm2[fi]128 doesn't change
40239 anything on the original 0123 first operand. */
40241 }
40244 {
40250 else
40252 }
40255 {
40259 }
40260 else
40264 {
40268 /* Found a usable second shuffle. dfirst will be
40269 vperm2f128 on d->op0 and d->op1. */
40280 /* And dsecond is some single insn shuffle, taking
40281 d->op0 and result of vperm2f128 (if perm < 16) or
40282 d->op1 and result of vperm2f128 (otherwise). */
40291 }
40293 /* For one operand, the only useful vperm2f128 permutation is 0x10. */
40296 }
40299 }
40301 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40302 a two vector permutation using 2 intra-lane interleave insns
40303 and cross-lane shuffle for 32-byte vectors. */
40305 static bool
40307 {
40314 ;
40316 ;
40317 else
40332 {
40336 else
40342 else
40348 else
40354 else
40360 else
40366 else
40371 }
40375 }
40377 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement
40378 a single vector permutation using a single intra-lane vector
40379 permutation, vperm2f128 swapping the lanes and vblend* insn blending
40380 the non-swapped and swapped vectors together. */
40382 static bool
40384 {
40387 rtx seq;
40392 || TARGET_AVX2
40401 {
40408 }
40443 }
40445 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement a V4DF
40446 permutation using two vperm2f128, followed by a vshufpd insn blending
40447 the two vectors together. */
40449 static bool
40451 {
40494 }
40496 /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
40497 permutation with two pshufb insns and an ior. We should have already
40498 failed all two instruction sequences. */
40500 static bool
40502 {
40513 /* Generate two permutation masks. If the required element is within
40514 the given vector it is shuffled into the proper lane. If the required
40515 element is in the other vector, force a zero into the lane by setting
40516 bit 7 in the permutation mask. */
40519 {
40526 {
40529 }
40530 }
40550 }
40552 /* Implement arbitrary permutation of one V32QImode and V16QImode operand
40553 with two vpshufb insns, vpermq and vpor. We should have already failed
40554 all two or three instruction sequences. */
40556 static bool
40558 {
40573 /* Generate two permutation masks. If the required element is within
40574 the same lane, it is shuffled in. If the required element from the
40575 other lane, force a zero by setting bit 7 in the permutation mask.
40576 In the other mask the mask has non-negative elements if element
40577 is requested from the other lane, but also moved to the other lane,
40578 so that the result of vpshufb can have the two V2TImode halves
40579 swapped. */
40582 {
40587 {
40590 }
40591 }
40600 /* Swap the 128-byte lanes of h into hp. */
40604 const1_rtx));
40617 }
40619 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
40620 and extract-odd permutations of two V32QImode and V16QImode operand
40621 with two vpshufb insns, vpor and vpermq. We should have already
40622 failed all two or three instruction sequences. */
40624 static bool
40626 {
40645 /* Generate two permutation masks. In the first permutation mask
40646 the first quarter will contain indexes for the first half
40647 of the op0, the second quarter will contain bit 7 set, third quarter
40648 will contain indexes for the second half of the op0 and the
40649 last quarter bit 7 set. In the second permutation mask
40650 the first quarter will contain bit 7 set, the second quarter
40651 indexes for the first half of the op1, the third quarter bit 7 set
40652 and last quarter indexes for the second half of the op1.
40653 I.e. the first mask e.g. for V32QImode extract even will be:
40654 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
40655 (all values masked with 0xf except for -128) and second mask
40656 for extract even will be
40657 -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
40660 {
40666 {
40669 }
40670 }
40689 /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
40696 }
40698 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
40699 and extract-odd permutations. */
40701 static bool
40703 {
40707 {
40712 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
40716 /* Now an unpck[lh]pd will produce the result required. */
40719 else
40725 {
40732 /* Shuffle within the 128-bit lanes to produce:
40733 { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
40737 /* Shuffle the lanes around to produce:
40738 { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
40742 /* Shuffle within the 128-bit lanes to produce:
40743 { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
40746 /* Shuffle within the 128-bit lanes to produce:
40747 { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
40750 /* Shuffle the lanes around to produce:
40751 { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
40754 }
40761 /* These are always directly implementable by expand_vec_perm_1. */
40767 else
40768 {
40769 /* We need 2*log2(N)-1 operations to achieve odd/even
40770 with interleave. */
40779 else
40782 }
40788 else
40789 {
40801 else
40804 }
40813 {
40820 }
40825 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
40829 /* Now an vpunpck[lh]qdq will produce the result required. */
40832 else
40839 {
40846 }
40851 /* Shuffle the lanes around into
40852 { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
40862 /* Swap the 2nd and 3rd position in each lane into
40863 { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
40869 /* Now an vpunpck[lh]qdq will produce
40870 { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
40875 else
40884 }
40887 }
40889 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
40890 extract-even and extract-odd permutations. */
40892 static bool
40894 {
40906 }
40908 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement broadcast
40909 permutations. We assume that expand_vec_perm_1 has already failed. */
40911 static bool
40913 {
40921 {
40924 /* These are special-cased in sse.md so that we can optionally
40925 use the vbroadcast instruction. They expand to two insns
40926 if the input happens to be in a register. */
40933 /* These are always implementable using standard shuffle patterns. */
40938 /* These can be implemented via interleave. We save one insn by
40939 stopping once we have promoted to V4SImode and then use pshufd. */
40940 do
40941 {
40942 rtx dest;
40948 {
40952 }
40959 }
40972 /* For AVX2 broadcasts of the first element vpbroadcast* or
40973 vpermq should be used by expand_vec_perm_1. */
40979 }
40980 }
40982 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
40983 broadcast permutations. */
40985 static bool
40987 {
40999 }
41001 /* Implement arbitrary permutation of two V32QImode and V16QImode operands
41002 with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
41003 all the shorter instruction sequences. */
41005 static bool
41007 {
41023 /* Generate 4 permutation masks. If the required element is within
41024 the same lane, it is shuffled in. If the required element from the
41025 other lane, force a zero by setting bit 7 in the permutation mask.
41026 In the other mask the mask has non-negative elements if element
41027 is requested from the other lane, but also moved to the other lane,
41028 so that the result of vpshufb can have the two V2TImode halves
41029 swapped. */
41032 {
41037 }
41043 {
41051 }
41054 {
41056 {
41059 }
41066 }
41068 /* Swap the 128-byte lanes of h[X]. */
41070 {
41076 const1_rtx));
41078 }
41081 {
41083 {
41086 }
41092 }
41095 {
41097 {
41101 }
41104 }
41110 }
41112 /* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
41113 With all of the interface bits taken care of, perform the expansion
41114 in D and return true on success. */
41116 static bool
41118 {
41119 /* Try a single instruction expansion. */
41123 /* Try sequences of two instructions. */
41143 /* Try sequences of three instructions. */
41157 /* Try sequences of four instructions. */
41165 /* ??? Look for narrow permutations whose element orderings would
41166 allow the promotion to a wider mode. */
41168 /* ??? Look for sequences of interleave or a wider permute that place
41169 the data into the correct lanes for a half-vector shuffle like
41170 pshuf[lh]w or vpermilps. */
41172 /* ??? Look for sequences of interleave that produce the desired results.
41173 The combinatorics of punpck[lh] get pretty ugly... */
41178 /* Even longer sequences. */
41183 }
41185 /* If a permutation only uses one operand, make it clear. Returns true
41186 if the permutation references both operands. */
41188 static bool
41190 {
41198 {
41204 {
41207 }
41208 /* The elements of PERM do not suggest that only the first operand
41209 is used, but both operands are identical. Allow easier matching
41210 of the permutation by folding the permutation into the single
41211 input vector. */
41212 /* FALLTHRU */
41223 }
41226 }
41228 bool
41230 {
41235 rtx sel;
41252 {
41257 }
41264 /* If the selector says both arguments are needed, but the operands are the
41265 same, the above tried to expand with one_operand_p and flattened selector.
41266 If that didn't work, retry without one_operand_p; we succeeded with that
41267 during testing. */
41269 {
41273 }
41276 }
41278 /* Implement targetm.vectorize.vec_perm_const_ok. */
41280 static bool
41283 {
41292 /* Given sufficient ISA support we can just return true here
41293 for selected vector modes. */
41295 {
41296 /* All implementable with a single vpperm insn. */
41299 /* All implementable with 2 pshufb + 1 ior. */
41302 /* All implementable with shufpd or unpck[lh]pd. */
41305 }
41307 /* Extract the values from the vector CST into the permutation
41308 array in D. */
41311 {
41315 }
41317 /* For all elements from second vector, fold the elements to first. */
41322 /* Check whether the mask can be applied to the vector type. */
41325 /* Implementable with shufps or pshufd. */
41329 /* Otherwise we have to go through the motions and see if we can
41330 figure out how to generate the requested permutation. */
41341 }
41343 void
41345 {
41360 /* We'll either be able to implement the permutation directly... */
41364 /* ... or we use the special-case patterns. */
41366 }
41368 static void
41370 {
41385 {
41388 }
41390 /* Note that for AVX this isn't one instruction. */
41393 }
41396 /* Expand a vector operation CODE for a V*QImode in terms of the
41397 same operation on V*HImode. */
41399 void
41401 {
41413 {
41426 }
41430 {
41432 /* Unpack data such that we've got a source byte in each low byte of
41433 each word. We don't care what goes into the high byte of each word.
41434 Rather than trying to get zero in there, most convenient is to let
41435 it be a copy of the low byte. */
41440 /* FALLTHRU */
41452 /* FALLTHRU */
41462 }
41464 /* Perform the operation. */
41471 /* Merge the data back into the right place. */
41481 {
41482 /* For SSE2, we used an full interleave, so the desired
41483 results are in the even elements. */
41486 }
41487 else
41488 {
41489 /* For AVX, the interleave used above was not cross-lane. So the
41490 extraction is evens but with the second and third quarter swapped.
41491 Happily, that is even one insn shorter than even extraction. */
41494 }
41501 }
41503 /* Helper function of ix86_expand_mul_widen_evenodd. Return true
41504 if op is CONST_VECTOR with all odd elements equal to their
41505 preceding element. */
41507 static bool
41509 {
41519 }
41521 void
41524 {
41527 rtx x;
41535 /* We only play even/odd games with vectors of SImode. */
41538 /* If we're looking for the odd results, shift those members down to
41539 the even slots. For some cpus this is faster than a PSHUFD. */
41541 {
41542 /* For XOP use vpmacsdqh, but only for smult, as it is only
41543 signed. */
41545 {
41549 }
41560 }
41563 {
41566 else
41568 }
41573 else
41574 {
41577 /* The easiest way to implement this without PMULDQ is to go through
41578 the motions as if we are performing a full 64-bit multiply. With
41579 the exception that we need to do less shuffling of the elements. */
41581 /* Compute the sign-extension, aka highparts, of the two operands. */
41587 /* Multiply LO(A) * HI(B), and vice-versa. */
41593 /* Multiply LO(A) * LO(B). */
41597 /* Combine and shift the highparts into place. */
41602 /* Combine high and low parts. */
41605 }
41607 }
41609 void
41612 {
41618 {
41623 {
41624 /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
41625 shuffle the elements once so that all elements are in the right
41626 place for immediate use: { A C B D }. */
41631 }
41632 else
41633 {
41634 /* Put the elements into place for the multiply. */
41638 }
41643 /* Shuffle the elements between the lanes. After this we
41644 have { A B E F | C D G H } for each operand. */
41654 /* Shuffle the elements within the lanes. After this we
41655 have { A A B B | C C D D } or { E E F F | G G H H }. */
41691 }
41692 }
41694 void
41696 {
41706 /* Move the results in element 2 down to element 1; we don't care
41707 what goes in elements 2 and 3. Then we can merge the parts
41708 back together with an interleave.
41710 Note that two other sequences were tried:
41711 (1) Use interleaves at the start instead of psrldq, which allows
41712 us to use a single shufps to merge things back at the end.
41713 (2) Use shufps here to combine the two vectors, then pshufd to
41714 put the elements in the correct order.
41715 In both cases the cost of the reformatting stall was too high
41716 and the overall sequence slower. */
41725 }
41727 void
41729 {
41734 {
41735 /* op1: A,B,C,D, op2: E,F,G,H */
41744 /* t1: B,A,D,C */
41751 /* t2: (B*E),(A*F),(D*G),(C*H) */
41754 /* t3: (B*E)+(A*F), (D*G)+(C*H) */
41757 /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
41760 /* op0: (((B*E)+(A*F))<<32)+(B*F), (((D*G)+(C*H))<<32)+(D*H) */
41762 }
41763 else
41764 {
41769 {
41772 }
41774 {
41777 }
41778 else
41782 /* Multiply low parts. */
41786 /* Shift input vectors right 32 bits so we can multiply high parts. */
41791 /* Multiply high parts by low parts. */
41797 /* Combine and shift the highparts back. */
41801 /* Combine high and low parts. */
41803 }
41807 }
41809 /* Expand an insert into a vector register through pinsr insn.
41810 Return true if successful. */
41812 bool
41814 {
41822 {
41825 }
41831 {
41836 {
41843 {
41875 }
41884 }
41888 }
41889 }
41890 \f
41891 /* This function returns the calling abi specific va_list type node.
41892 It returns the FNDECL specific va_list type. */
41894 static tree
41896 {
41903 else
41905 }
41907 /* Returns the canonical va_list type specified by TYPE. If there
41908 is no valid TYPE provided, it return NULL_TREE. */
41910 static tree
41912 {
41915 /* Resolve references and pointers to va_list type. */
41924 {
41929 {
41930 /* If va_list is an array type, the argument may have decayed
41931 to a pointer type, e.g. by being passed to another function.
41932 In that case, unwrap both types so that we can compare the
41933 underlying records. */
41936 {
41939 }
41940 }
41947 {
41948 /* If va_list is an array type, the argument may have decayed
41949 to a pointer type, e.g. by being passed to another function.
41950 In that case, unwrap both types so that we can compare the
41951 underlying records. */
41954 {
41957 }
41958 }
41965 {
41966 /* If va_list is an array type, the argument may have decayed
41967 to a pointer type, e.g. by being passed to another function.
41968 In that case, unwrap both types so that we can compare the
41969 underlying records. */
41972 {
41975 }
41976 }
41980 }
41982 }
41984 /* Iterate through the target-specific builtin types for va_list.
41985 IDX denotes the iterator, *PTREE is set to the result type of
41986 the va_list builtin, and *PNAME to its internal type.
41987 Returns zero if there is no element for this index, otherwise
41988 IDX should be increased upon the next call.
41989 Note, do not iterate a base builtin's name like __builtin_va_list.
41990 Used from c_common_nodes_and_builtins. */
41992 static int
41994 {
41996 {
41998 {
42011 }
42012 }
42015 }
42017 #undef TARGET_SCHED_DISPATCH
42018 #define TARGET_SCHED_DISPATCH has_dispatch
42019 #undef TARGET_SCHED_DISPATCH_DO
42020 #define TARGET_SCHED_DISPATCH_DO do_dispatch
42021 #undef TARGET_SCHED_REASSOCIATION_WIDTH
42022 #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
42023 #undef TARGET_SCHED_REORDER
42024 #define TARGET_SCHED_REORDER ix86_sched_reorder
42025 #undef TARGET_SCHED_ADJUST_PRIORITY
42026 #define TARGET_SCHED_ADJUST_PRIORITY ix86_adjust_priority
42027 #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
42028 #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK \
42029 ix86_dependencies_evaluation_hook
42031 /* The size of the dispatch window is the total number of bytes of
42032 object code allowed in a window. */
42033 #define DISPATCH_WINDOW_SIZE 16
42035 /* Number of dispatch windows considered for scheduling. */
42036 #define MAX_DISPATCH_WINDOWS 3
42038 /* Maximum number of instructions in a window. */
42039 #define MAX_INSN 4
42041 /* Maximum number of immediate operands in a window. */
42042 #define MAX_IMM 4
42044 /* Maximum number of immediate bits allowed in a window. */
42045 #define MAX_IMM_SIZE 128
42047 /* Maximum number of 32 bit immediates allowed in a window. */
42048 #define MAX_IMM_32 4
42050 /* Maximum number of 64 bit immediates allowed in a window. */
42051 #define MAX_IMM_64 2
42053 /* Maximum total of loads or prefetches allowed in a window. */
42054 #define MAX_LOAD 2
42056 /* Maximum total of stores allowed in a window. */
42057 #define MAX_STORE 1
42059 #undef BIG
42060 #define BIG 100
42063 /* Dispatch groups. Istructions that affect the mix in a dispatch window. */
42066 disp_load,
42067 disp_store,
42068 disp_load_store,
42069 disp_prefetch,
42070 disp_imm,
42071 disp_imm_32,
42072 disp_imm_64,
42073 disp_branch,
42074 disp_cmp,
42075 disp_jcc,
42076 disp_last
42077 };
42079 /* Number of allowable groups in a dispatch window. It is an array
42080 indexed by dispatch_group enum. 100 is used as a big number,
42081 because the number of these kind of operations does not have any
42082 effect in dispatch window, but we need them for other reasons in
42083 the table. */
42086 };
42092 };
42094 /* Instruction path. */
42100 last_path
42101 };
42103 /* sched_insn_info defines a window to the instructions scheduled in
42104 the basic block. It contains a pointer to the insn_info table and
42105 the instruction scheduled.
42107 Windows are allocated for each basic block and are linked
42108 together. */
42110 rtx insn;
42117 /* Linked list of dispatch windows. This is a two way list of
42118 dispatch windows of a basic block. It contains information about
42119 the number of uops in the window and the total number of
42120 instructions and of bytes in the object code for this dispatch
42121 window. */
42139 /* Immediate valuse used in an insn. */
42141 {
42150 /* Get dispatch group of insn. */
42152 static enum dispatch_group
42154 {
42170 }
42172 /* Return true if insn is a compare instruction. */
42174 static bool
42176 {
42184 }
42186 /* Return true if a dispatch violation encountered. */
42188 static bool
42190 {
42194 }
42196 /* Return true if insn is a branch instruction. */
42198 static bool
42200 {
42202 }
42204 /* Return true if insn is a prefetch instruction. */
42206 static bool
42208 {
42210 }
42212 /* This function initializes a dispatch window and the list container holding a
42213 pointer to the window. */
42215 static void
42217 {
42223 else
42241 {
42247 }
42249 }
42251 /* This function allocates and initializes a dispatch window and the
42252 list container holding a pointer to the window. */
42256 {
42261 }
42263 /* This routine initializes the dispatch scheduling information. It
42264 initiates building dispatch scheduler tables and constructs the
42265 first dispatch window. */
42267 static void
42269 {
42270 /* Allocate a dispatch list and a window. */
42275 }
42277 /* This function returns true if a branch is detected. End of a basic block
42278 does not have to be a branch, but here we assume only branches end a
42279 window. */
42281 static bool
42283 {
42285 }
42287 /* This function is called when the end of a window processing is reached. */
42289 static void
42291 {
42294 {
42299 }
42301 }
42303 /* Allocates a new dispatch window and adds it to WINDOW_LIST.
42304 WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
42305 for 48 bytes of instructions. Note that these windows are not dispatch
42306 windows that their sizes are DISPATCH_WINDOW_SIZE. */
42310 {
42312 {
42317 }
42323 }
42325 /* Increment the number of immediate operands of an instruction. */
42327 static int
42329 {
42334 {
42341 else
42352 {
42355 }
42360 }
42363 }
42365 /* Compute number of immediate operands of an instruction. */
42367 static void
42369 {
42372 }
42374 /* Return total size of immediate operands of an instruction along with number
42375 of corresponding immediate-operands. It initializes its parameters to zero
42376 befor calling FIND_CONSTANT.
42377 INSN is the input instruction. IMM is the total of immediates.
42378 IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
42379 bit immediates. */
42381 static int
42383 {
42391 }
42393 /* This function indicates if an operand of an instruction is an
42394 immediate. */
42396 static bool
42398 {
42407 }
42409 /* Return single or double path for instructions. */
42411 static enum insn_path
42413 {
42423 }
42425 /* Return insn dispatch group. */
42427 static enum dispatch_group
42429 {
42447 }
42449 /* Count number of GROUP restricted instructions in a dispatch
42450 window WINDOW_LIST. */
42452 static int
42454 {
42465 {
42484 }
42497 }
42499 /* This function returns true if insn satisfies dispatch rules on the
42500 last window scheduled. */
42502 static bool
42504 {
42512 /* Make disp_cmp and disp_jcc get scheduled at the latest. These
42513 instructions should be given the lowest priority in the
42514 scheduling process in Haifa scheduler to make sure they will be
42515 scheduled in the same dispatch window as the reference to them. */
42519 /* Check nonrestricted. */
42523 /* Get last dispatch window. */
42528 {
42533 /* Window 1 is full. Go for next window. */
42535 }
42542 /* See if it fits in the first window. */
42544 {
42545 /* The first widow should have only single and double path
42546 uops. */
42552 }
42554 }
42556 /* Add an instruction INSN with NUM_UOPS micro-operations to the
42557 dispatch window WINDOW_LIST. */
42559 static void
42561 {
42579 /* Initialize window with new instruction. */
42600 {
42603 }
42604 }
42606 /* Adds a scheduled instruction, INSN, to the current dispatch window.
42607 If the total bytes of instructions or the number of instructions in
42608 the window exceed allowable, it allocates a new window. */
42610 static void
42612 {
42635 /* Get the last dispatch window. */
42643 else
42646 /* If current window is full, get a new window.
42647 Window number zero is full, if MAX_INSN uops are scheduled in it.
42648 Window number one is full, if window zero's bytes plus window
42649 one's bytes is 32, or if the bytes of the new instruction added
42650 to the total makes it greater than 48, or it has already MAX_INSN
42651 instructions in it. */
42660 {
42663 }
42666 {
42670 {
42673 }
42674 }
42676 {
42681 {
42684 }
42687 }
42688 else
42692 {
42693 /* End of basic block is reached do end-basic-block process. */
42696 }
42697 }
42699 /* Print the dispatch window, WINDOW_NUM, to FILE. */
42701 DEBUG_FUNCTION static void
42703 {
42709 else
42723 {
42726 fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
42732 }
42733 }
42735 /* Print to stdout a dispatch window. */
42737 DEBUG_FUNCTION void
42739 {
42741 }
42743 /* Print INSN dispatch information to FILE. */
42745 DEBUG_FUNCTION static void
42747 {
42770 }
42772 /* Print to STDERR the status of the ready list with respect to
42773 dispatch windows. */
42775 DEBUG_FUNCTION void
42777 {
42785 }
42787 /* This routine is the driver of the dispatch scheduler. */
42789 static void
42791 {
42796 }
42798 /* Return TRUE if Dispatch Scheduling is supported. */
42800 static bool
42802 {
42806 {
42822 }
42825 }
42827 /* Implementation of reassociation_width target hook used by
42828 reassoc phase to identify parallelism level in reassociated
42829 tree. Statements tree_code is passed in OPC. Arguments type
42830 is passed in MODE.
42832 Currently parallel reassociation is enabled for Atom
42833 processors only and we set reassociation width to be 2
42834 because Atom may issue up to 2 instructions per cycle.
42836 Return value should be fixed if parallel reassociation is
42837 enabled for other processors. */
42839 static int
42842 {
42851 }
42853 /* ??? No autovectorization into MMX or 3DNOW until we can reliably
42854 place emms and femms instructions. */
42856 static enum machine_mode
42858 {
42863 {
42876 else
42886 /* FALLTHRU */
42890 }
42891 }
42893 /* If AVX is enabled then try vectorizing with both 256bit and 128bit
42894 vectors. */
42896 static unsigned int
42898 {
42900 }
42902 \f
42904 /* Return class of registers which could be used for pseudo of MODE
42905 and of class RCLASS for spilling instead of memory. Return NO_REGS
42906 if it is not possible or non-profitable. */
42907 static reg_class_t
42909 {
42915 }
42917 /* Implement targetm.vectorize.init_cost. */
42921 {
42925 }
42927 /* Implement targetm.vectorize.add_stmt_cost. */
42929 static unsigned
42933 {
42938 {
42942 /* Statements in an inner loop relative to the loop being
42943 vectorized are weighted more heavily. The value here is
42944 arbitrary and could potentially be improved with analysis. */
42950 }
42953 }
42955 /* Implement targetm.vectorize.finish_cost. */
42957 static void
42960 {
42965 }
42967 /* Implement targetm.vectorize.destroy_cost_data. */
42969 static void
42971 {
42973 }
42975 /* Validate target specific memory model bits in VAL. */
42977 static unsigned HOST_WIDE_INT
42979 {
42986 {
42990 }
42993 {
42997 }
42999 {
43003 }
43005 }
43007 /* Initialize the GCC target structure. */
43008 #undef TARGET_RETURN_IN_MEMORY
43009 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
43011 #undef TARGET_LEGITIMIZE_ADDRESS
43012 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
43014 #undef TARGET_ATTRIBUTE_TABLE
43015 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
43016 #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
43017 #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P hook_bool_const_tree_true
43018 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
43019 # undef TARGET_MERGE_DECL_ATTRIBUTES
43020 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
43021 #endif
43023 #undef TARGET_COMP_TYPE_ATTRIBUTES
43024 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
43026 #undef TARGET_INIT_BUILTINS
43027 #define TARGET_INIT_BUILTINS ix86_init_builtins
43028 #undef TARGET_BUILTIN_DECL
43029 #define TARGET_BUILTIN_DECL ix86_builtin_decl
43030 #undef TARGET_EXPAND_BUILTIN
43031 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
43033 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
43034 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
43035 ix86_builtin_vectorized_function
43037 #undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
43038 #define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
43040 #undef TARGET_VECTORIZE_BUILTIN_TM_STORE
43041 #define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
43043 #undef TARGET_VECTORIZE_BUILTIN_GATHER
43044 #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
43046 #undef TARGET_BUILTIN_RECIPROCAL
43047 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
43049 #undef TARGET_ASM_FUNCTION_EPILOGUE
43050 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
43052 #undef TARGET_ENCODE_SECTION_INFO
43053 #ifndef SUBTARGET_ENCODE_SECTION_INFO
43054 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
43055 #else
43056 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
43057 #endif
43059 #undef TARGET_ASM_OPEN_PAREN
43061 #undef TARGET_ASM_CLOSE_PAREN
43064 #undef TARGET_ASM_BYTE_OP
43065 #define TARGET_ASM_BYTE_OP ASM_BYTE
43067 #undef TARGET_ASM_ALIGNED_HI_OP
43068 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
43069 #undef TARGET_ASM_ALIGNED_SI_OP
43070 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
43071 #ifdef ASM_QUAD
43072 #undef TARGET_ASM_ALIGNED_DI_OP
43073 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
43074 #endif
43076 #undef TARGET_PROFILE_BEFORE_PROLOGUE
43077 #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
43079 #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
43080 #define TARGET_MANGLE_DECL_ASSEMBLER_NAME ix86_mangle_decl_assembler_name
43082 #undef TARGET_ASM_UNALIGNED_HI_OP
43083 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
43084 #undef TARGET_ASM_UNALIGNED_SI_OP
43085 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
43086 #undef TARGET_ASM_UNALIGNED_DI_OP
43087 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
43089 #undef TARGET_PRINT_OPERAND
43090 #define TARGET_PRINT_OPERAND ix86_print_operand
43091 #undef TARGET_PRINT_OPERAND_ADDRESS
43092 #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
43093 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
43094 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
43095 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
43096 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
43098 #undef TARGET_SCHED_INIT_GLOBAL
43099 #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
43100 #undef TARGET_SCHED_ADJUST_COST
43101 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
43102 #undef TARGET_SCHED_ISSUE_RATE
43103 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
43104 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
43105 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
43106 ia32_multipass_dfa_lookahead
43108 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
43109 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
43111 #undef TARGET_MEMMODEL_CHECK
43112 #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
43114 #ifdef HAVE_AS_TLS
43115 #undef TARGET_HAVE_TLS
43116 #define TARGET_HAVE_TLS true
43117 #endif
43118 #undef TARGET_CANNOT_FORCE_CONST_MEM
43119 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
43120 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
43121 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
43123 #undef TARGET_DELEGITIMIZE_ADDRESS
43124 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
43126 #undef TARGET_MS_BITFIELD_LAYOUT_P
43127 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
43129 #if TARGET_MACHO
43130 #undef TARGET_BINDS_LOCAL_P
43131 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
43132 #endif
43133 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
43134 #undef TARGET_BINDS_LOCAL_P
43135 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
43136 #endif
43138 #undef TARGET_ASM_OUTPUT_MI_THUNK
43139 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
43140 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
43141 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
43143 #undef TARGET_ASM_FILE_START
43144 #define TARGET_ASM_FILE_START x86_file_start
43146 #undef TARGET_OPTION_OVERRIDE
43147 #define TARGET_OPTION_OVERRIDE ix86_option_override
43149 #undef TARGET_REGISTER_MOVE_COST
43150 #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
43151 #undef TARGET_MEMORY_MOVE_COST
43152 #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
43153 #undef TARGET_RTX_COSTS
43154 #define TARGET_RTX_COSTS ix86_rtx_costs
43155 #undef TARGET_ADDRESS_COST
43156 #define TARGET_ADDRESS_COST ix86_address_cost
43158 #undef TARGET_FIXED_CONDITION_CODE_REGS
43159 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
43160 #undef TARGET_CC_MODES_COMPATIBLE
43161 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
43163 #undef TARGET_MACHINE_DEPENDENT_REORG
43164 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
43166 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
43167 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
43169 #undef TARGET_BUILD_BUILTIN_VA_LIST
43170 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
43172 #undef TARGET_FOLD_BUILTIN
43173 #define TARGET_FOLD_BUILTIN ix86_fold_builtin
43175 #undef TARGET_COMPARE_VERSION_PRIORITY
43176 #define TARGET_COMPARE_VERSION_PRIORITY ix86_compare_version_priority
43178 #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
43179 #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
43180 ix86_generate_version_dispatcher_body
43182 #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
43183 #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
43184 ix86_get_function_versions_dispatcher
43186 #undef TARGET_ENUM_VA_LIST_P
43187 #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
43189 #undef TARGET_FN_ABI_VA_LIST
43190 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
43192 #undef TARGET_CANONICAL_VA_LIST_TYPE
43193 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
43195 #undef TARGET_EXPAND_BUILTIN_VA_START
43196 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
43198 #undef TARGET_MD_ASM_CLOBBERS
43199 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
43201 #undef TARGET_PROMOTE_PROTOTYPES
43202 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
43203 #undef TARGET_STRUCT_VALUE_RTX
43204 #define TARGET_STRUCT_VALUE_RTX ix86_struct_value_rtx
43205 #undef TARGET_SETUP_INCOMING_VARARGS
43206 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
43207 #undef TARGET_MUST_PASS_IN_STACK
43208 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
43209 #undef TARGET_FUNCTION_ARG_ADVANCE
43210 #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
43211 #undef TARGET_FUNCTION_ARG
43212 #define TARGET_FUNCTION_ARG ix86_function_arg
43213 #undef TARGET_FUNCTION_ARG_BOUNDARY
43214 #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
43215 #undef TARGET_PASS_BY_REFERENCE
43216 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
43217 #undef TARGET_INTERNAL_ARG_POINTER
43218 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
43219 #undef TARGET_UPDATE_STACK_BOUNDARY
43220 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
43221 #undef TARGET_GET_DRAP_RTX
43222 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
43223 #undef TARGET_STRICT_ARGUMENT_NAMING
43224 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
43225 #undef TARGET_STATIC_CHAIN
43226 #define TARGET_STATIC_CHAIN ix86_static_chain
43227 #undef TARGET_TRAMPOLINE_INIT
43228 #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
43229 #undef TARGET_RETURN_POPS_ARGS
43230 #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
43232 #undef TARGET_LEGITIMATE_COMBINED_INSN
43233 #define TARGET_LEGITIMATE_COMBINED_INSN ix86_legitimate_combined_insn
43235 #undef TARGET_ASAN_SHADOW_OFFSET
43236 #define TARGET_ASAN_SHADOW_OFFSET ix86_asan_shadow_offset
43238 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
43239 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
43241 #undef TARGET_SCALAR_MODE_SUPPORTED_P
43242 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
43244 #undef TARGET_VECTOR_MODE_SUPPORTED_P
43245 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
43247 #undef TARGET_C_MODE_FOR_SUFFIX
43248 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
43250 #ifdef HAVE_AS_TLS
43251 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
43252 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
43253 #endif
43255 #ifdef SUBTARGET_INSERT_ATTRIBUTES
43256 #undef TARGET_INSERT_ATTRIBUTES
43257 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
43258 #endif
43260 #undef TARGET_MANGLE_TYPE
43261 #define TARGET_MANGLE_TYPE ix86_mangle_type
43263 #if !TARGET_MACHO
43264 #undef TARGET_STACK_PROTECT_FAIL
43265 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
43266 #endif
43268 #undef TARGET_FUNCTION_VALUE
43269 #define TARGET_FUNCTION_VALUE ix86_function_value
43271 #undef TARGET_FUNCTION_VALUE_REGNO_P
43272 #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
43274 #undef TARGET_PROMOTE_FUNCTION_MODE
43275 #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
43277 #undef TARGET_MEMBER_TYPE_FORCES_BLK
43278 #define TARGET_MEMBER_TYPE_FORCES_BLK ix86_member_type_forces_blk
43280 #undef TARGET_INSTANTIATE_DECLS
43281 #define TARGET_INSTANTIATE_DECLS ix86_instantiate_decls
43283 #undef TARGET_SECONDARY_RELOAD
43284 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
43286 #undef TARGET_CLASS_MAX_NREGS
43287 #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
43289 #undef TARGET_PREFERRED_RELOAD_CLASS
43290 #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
43291 #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
43292 #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
43293 #undef TARGET_CLASS_LIKELY_SPILLED_P
43294 #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
43296 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
43297 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
43298 ix86_builtin_vectorization_cost
43299 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
43300 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
43301 ix86_vectorize_vec_perm_const_ok
43302 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
43303 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
43304 ix86_preferred_simd_mode
43305 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
43306 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
43307 ix86_autovectorize_vector_sizes
43308 #undef TARGET_VECTORIZE_INIT_COST
43309 #define TARGET_VECTORIZE_INIT_COST ix86_init_cost
43310 #undef TARGET_VECTORIZE_ADD_STMT_COST
43311 #define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
43312 #undef TARGET_VECTORIZE_FINISH_COST
43313 #define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
43314 #undef TARGET_VECTORIZE_DESTROY_COST_DATA
43315 #define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
43317 #undef TARGET_SET_CURRENT_FUNCTION
43318 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
43320 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
43321 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
43323 #undef TARGET_OPTION_SAVE
43324 #define TARGET_OPTION_SAVE ix86_function_specific_save
43326 #undef TARGET_OPTION_RESTORE
43327 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
43329 #undef TARGET_OPTION_PRINT
43330 #define TARGET_OPTION_PRINT ix86_function_specific_print
43332 #undef TARGET_OPTION_FUNCTION_VERSIONS
43333 #define TARGET_OPTION_FUNCTION_VERSIONS ix86_function_versions
43335 #undef TARGET_CAN_INLINE_P
43336 #define TARGET_CAN_INLINE_P ix86_can_inline_p
43338 #undef TARGET_EXPAND_TO_RTL_HOOK
43339 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
43341 #undef TARGET_LEGITIMATE_ADDRESS_P
43342 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
43344 #undef TARGET_LRA_P
43345 #define TARGET_LRA_P hook_bool_void_true
43347 #undef TARGET_REGISTER_PRIORITY
43348 #define TARGET_REGISTER_PRIORITY ix86_register_priority
43350 #undef TARGET_REGISTER_USAGE_LEVELING_P
43351 #define TARGET_REGISTER_USAGE_LEVELING_P hook_bool_void_true
43353 #undef TARGET_LEGITIMATE_CONSTANT_P
43354 #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
43356 #undef TARGET_FRAME_POINTER_REQUIRED
43357 #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
43359 #undef TARGET_CAN_ELIMINATE
43360 #define TARGET_CAN_ELIMINATE ix86_can_eliminate
43362 #undef TARGET_EXTRA_LIVE_ON_ENTRY
43363 #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
43365 #undef TARGET_ASM_CODE_END
43366 #define TARGET_ASM_CODE_END ix86_code_end
43368 #undef TARGET_CONDITIONAL_REGISTER_USAGE
43369 #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
43371 #if TARGET_MACHO
43372 #undef TARGET_INIT_LIBFUNCS
43373 #define TARGET_INIT_LIBFUNCS darwin_rename_builtins
43374 #endif
43376 #undef TARGET_SPILL_CLASS
43377 #define TARGET_SPILL_CLASS ix86_spill_class
43380 \f