| blob | e40750380cce202473da3cf572ebdbc28a4ecc06 |
1 /* Machine description for AArch64 architecture.
2 Copyright (C) 2009-2020 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
24 #define INCLUDE_STRING
78 /* This file should be included last. */
81 /* Defined for convenience. */
82 #define POINTER_BYTES (POINTER_SIZE / BITS_PER_UNIT)
84 /* Information about a legitimate vector immediate operand. */
85 struct simd_immediate_info
86 {
98 /* The mode of the elements. */
99 scalar_mode elt_mode;
101 /* The instruction to use to move the immediate into a vector. */
102 insn_type insn;
104 union
105 {
106 /* For MOV and MVN. */
107 struct
108 {
109 /* The value of each element. */
110 rtx value;
112 /* The kind of shift modifier to use, and the number of bits to shift.
113 This is (LSL, 0) if no shift is needed. */
114 modifier_type modifier;
118 /* For INDEX. */
119 struct
120 {
121 /* The value of the first element and the step to be added for each
122 subsequent element. */
126 /* For PTRUE. */
127 aarch64_svpattern pattern;
129 };
131 /* Construct a floating-point immediate in which each element has mode
132 ELT_MODE_IN and value VALUE_IN. */
133 inline simd_immediate_info
136 {
140 }
142 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
143 and value VALUE_IN. The other parameters are as for the structure
144 fields. */
145 inline simd_immediate_info
151 {
155 }
157 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
158 and where element I is equal to BASE_IN + I * STEP_IN. */
159 inline simd_immediate_info
162 {
165 }
167 /* Construct a predicate that controls elements of mode ELT_MODE_IN
168 and has PTRUE pattern PATTERN_IN. */
169 inline simd_immediate_info
171 aarch64_svpattern pattern_in)
173 {
175 }
177 /* The current code model. */
180 /* The number of 64-bit elements in an SVE vector. */
181 poly_uint16 aarch64_sve_vg;
183 #ifdef HAVE_AS_TLS
184 #undef TARGET_HAVE_TLS
185 #define TARGET_HAVE_TLS 1
186 #endif
190 const_tree,
199 const_tree type,
204 aarch64_addr_query_type);
207 /* Major revision number of the ARM Architecture implemented by the target. */
210 /* The processor for which instructions should be scheduled. */
213 /* Mask to specify which instruction scheduling options should be used. */
216 /* Global flag for PC relative loads. */
219 /* Global flag for whether frame pointer is enabled. */
222 #define BRANCH_PROTECT_STR_MAX 255
225 static enum aarch64_parse_opt_result
228 /* Support for command line parsing of boolean flags in the tuning
229 structures. */
230 struct aarch64_flag_desc
231 {
234 };
236 #define AARCH64_FUSION_PAIR(name, internal_name) \
237 { name, AARCH64_FUSE_##internal_name },
239 {
244 };
246 #define AARCH64_EXTRA_TUNING_OPTION(name, internal_name) \
247 { name, AARCH64_EXTRA_TUNE_##internal_name },
249 {
254 };
256 /* Tuning parameters. */
259 {
260 {
265 },
272 };
275 {
276 {
281 },
288 };
291 {
292 {
297 },
304 };
307 {
308 {
313 },
320 };
323 {
324 {
329 },
336 };
339 {
340 {
345 },
352 };
355 {
357 /* Avoid the use of slow int<->fp moves for spilling by setting
358 their cost higher than memmov_cost. */
362 };
365 {
367 /* Avoid the use of slow int<->fp moves for spilling by setting
368 their cost higher than memmov_cost. */
372 };
375 {
377 /* Avoid the use of slow int<->fp moves for spilling by setting
378 their cost higher than memmov_cost. */
382 };
385 {
387 /* Avoid the use of slow int<->fp moves for spilling by setting
388 their cost higher than memmov_cost (actual, 4 and 9). */
392 };
395 {
400 };
403 {
405 /* Avoid the use of slow int<->fp moves for spilling by setting
406 their cost higher than memmov_cost. */
410 };
413 {
415 /* Avoid the use of int<->fp moves for spilling. */
419 };
422 {
424 /* Avoid the use of int<->fp moves for spilling. */
428 };
431 {
433 /* Avoid the use of slow int<->fp moves for spilling by setting
434 their cost higher than memmov_cost. */
438 };
440 /* Generic costs for vector insn classes. */
442 {
458 };
460 /* QDF24XX costs for vector insn classes. */
462 {
478 };
480 /* ThunderX costs for vector insn classes. */
482 {
498 };
501 {
517 };
519 /* Generic costs for vector insn classes. */
521 {
537 };
540 {
556 };
558 /* Generic costs for vector insn classes. */
560 {
576 };
578 /* Costs for vector insn classes for Vulcan. */
580 {
596 };
598 /* Generic costs for branch instructions. */
600 {
603 };
605 /* Generic approximation modes. */
607 {
610 AARCH64_APPROX_NONE /* recip_sqrt */
611 };
613 /* Approximation modes for Exynos M1. */
615 {
618 AARCH64_APPROX_ALL /* recip_sqrt */
619 };
621 /* Approximation modes for X-Gene 1. */
623 {
626 AARCH64_APPROX_ALL /* recip_sqrt */
627 };
629 /* Generic prefetch settings (which disable prefetch). */
631 {
639 };
642 {
650 };
653 {
661 };
664 {
672 };
675 {
683 };
686 {
694 };
697 {
705 };
708 {
716 };
719 {
741 &generic_prefetch_tune
742 };
745 {
768 &generic_prefetch_tune
769 };
772 {
795 &generic_prefetch_tune
796 };
799 {
822 &generic_prefetch_tune
823 };
826 {
849 &generic_prefetch_tune
850 };
853 {
876 &generic_prefetch_tune
877 };
882 {
904 &exynosm1_prefetch_tune
905 };
908 {
930 &thunderxt88_prefetch_tune
931 };
934 {
955 (AARCH64_EXTRA_TUNE_SLOW_UNALIGNED_LDPW
957 &thunderx_prefetch_tune
958 };
961 {
984 &tsv110_prefetch_tune
985 };
988 {
1010 &xgene1_prefetch_tune
1011 };
1014 {
1021 SVE_NOT_IMPLEMENTED,
1036 &xgene1_prefetch_tune
1037 };
1040 {
1063 &qdf24xx_prefetch_tune
1064 };
1066 /* Tuning structure for the Qualcomm Saphira core. Default to falkor values
1067 for now. */
1069 {
1092 &generic_prefetch_tune
1093 };
1096 {
1119 &thunderx2t99_prefetch_tune
1120 };
1123 {
1145 &generic_prefetch_tune
1146 };
1148 /* Support for fine-grained override of the tuning structures. */
1149 struct aarch64_tuning_override_function
1150 {
1153 };
1159 static const struct aarch64_tuning_override_function
1160 aarch64_tuning_override_functions[] =
1161 {
1166 };
1168 /* A processor implementing AArch64. */
1169 struct processor
1170 {
1178 };
1180 /* Architectures implementing AArch64. */
1182 {
1183 #define AARCH64_ARCH(NAME, CORE, ARCH_IDENT, ARCH_REV, FLAGS) \
1184 {NAME, CORE, CORE, AARCH64_ARCH_##ARCH_IDENT, ARCH_REV, FLAGS, NULL},
1187 };
1189 /* Processor cores implementing AArch64. */
1191 {
1192 #define AARCH64_CORE(NAME, IDENT, SCHED, ARCH, FLAGS, COSTS, IMP, PART, VARIANT) \
1193 {NAME, IDENT, SCHED, AARCH64_ARCH_##ARCH, \
1194 all_architectures[AARCH64_ARCH_##ARCH].architecture_version, \
1195 FLAGS, &COSTS##_tunings},
1200 };
1203 /* Target specification. These are populated by the -march, -mtune, -mcpu
1204 handling code or by target attributes. */
1211 /* The current tuning set. */
1214 /* Check whether an 'aarch64_vector_pcs' attribute is valid. */
1216 static tree
1219 {
1220 /* Since we set fn_type_req to true, the caller should have checked
1221 this for us. */
1224 {
1231 name);
1238 }
1240 }
1242 /* Table of machine attributes. */
1244 {
1245 /* { name, min_len, max_len, decl_req, type_req, fn_type_req,
1246 affects_type_identity, handler, exclude } */
1251 };
1253 #define AARCH64_CPU_DEFAULT_FLAGS ((selected_cpu) ? selected_cpu->flags : 0)
1255 /* An ISA extension in the co-processor and main instruction set space. */
1256 struct aarch64_option_extension
1257 {
1261 };
1264 {
1268 }
1269 aarch64_cc;
1271 #define AARCH64_INVERSE_CONDITION_CODE(X) ((aarch64_cc) (((int) X) ^ 1))
1273 struct aarch64_branch_protect_type
1274 {
1275 /* The type's name that the user passes to the branch-protection option
1276 string. */
1278 /* Function to handle the protection type and set global variables.
1279 First argument is the string token corresponding with this type and the
1280 second argument is the next token in the option string.
1281 Return values:
1282 * AARCH64_PARSE_OK: Handling was sucessful.
1283 * AARCH64_INVALID_ARG: The type is invalid in this context and the caller
1284 should print an error.
1285 * AARCH64_INVALID_FEATURE: The type is invalid and the handler prints its
1286 own error. */
1288 /* A list of types that can follow this type in the option string. */
1291 };
1293 static enum aarch64_parse_opt_result
1295 {
1299 {
1302 }
1304 }
1306 static enum aarch64_parse_opt_result
1308 {
1313 {
1316 }
1318 }
1320 static enum aarch64_parse_opt_result
1323 {
1327 }
1329 static enum aarch64_parse_opt_result
1332 {
1335 }
1337 static enum aarch64_parse_opt_result
1340 {
1343 }
1345 static enum aarch64_parse_opt_result
1348 {
1351 }
1357 };
1366 };
1368 /* The condition codes of the processor, and the inverse function. */
1370 {
1373 };
1375 /* The preferred condition codes for SVE conditions. */
1377 {
1380 };
1382 /* Return the assembly token for svpattern value VALUE. */
1386 {
1388 {
1389 #define CASE(UPPER, LOWER, VALUE) case AARCH64_SV_##UPPER: return #LOWER;
1391 #undef CASE
1394 }
1396 }
1398 /* Return the descriptor of the SIMD ABI. */
1402 {
1405 {
1406 HARD_REG_SET full_reg_clobbers
1412 }
1414 }
1416 /* Return the descriptor of the SVE PCS. */
1420 {
1423 {
1424 HARD_REG_SET full_reg_clobbers
1431 }
1433 }
1435 /* Generate code to enable conditional branches in functions over 1 MiB. */
1439 {
1456 }
1458 void
1460 {
1465 else
1468 else
1472 else
1475 }
1477 /* Report when we try to do something that requires SVE when SVE is disabled.
1478 This is an error of last resort and isn't very high-quality. It usually
1479 involves attempts to measure the vector length in some way. */
1480 static void
1482 {
1485 /* Avoid reporting a slew of messages for a single oversight. */
1491 " option %<-march%>, or by using the %<target%>"
1494 }
1496 /* Return true if REGNO is P0-P15 or one of the special FFR-related
1497 registers. */
1500 {
1502 }
1504 /* Implement TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS.
1505 The register allocator chooses POINTER_AND_FP_REGS if FP_REGS and
1506 GENERAL_REGS have the same cost - even if POINTER_AND_FP_REGS has a much
1507 higher cost. POINTER_AND_FP_REGS is also used if the cost of both FP_REGS
1508 and GENERAL_REGS is lower than the memory cost (in this case the best class
1509 is the lowest cost one). Using POINTER_AND_FP_REGS irrespectively of its
1510 cost results in bad allocations with many redundant int<->FP moves which
1511 are expensive on various cores.
1512 To avoid this we don't allow POINTER_AND_FP_REGS as the allocno class, but
1513 force a decision between FP_REGS and GENERAL_REGS. We use the allocno class
1514 if it isn't POINTER_AND_FP_REGS. Similarly, use the best class if it isn't
1515 POINTER_AND_FP_REGS. Otherwise set the allocno class depending on the mode.
1516 The result of this is that it is no longer inefficient to have a higher
1517 memory move cost than the register move cost.
1518 */
1520 static reg_class_t
1522 reg_class_t best_class)
1523 {
1524 machine_mode mode;
1536 }
1538 static unsigned int
1540 {
1544 }
1546 /* Return the reassociation width of treeop OPC with mode MODE. */
1547 static int
1549 {
1554 /* Avoid reassociating floating point addition so we emit more FMAs. */
1558 }
1560 /* Provide a mapping from gcc register numbers to dwarf register numbers. */
1561 unsigned
1563 {
1575 /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
1576 equivalent DWARF register. */
1578 }
1580 /* If X is a CONST_DOUBLE, return its bit representation as a constant
1581 integer, otherwise return X unmodified. */
1582 static rtx
1584 {
1588 }
1590 /* Return true if MODE is any of the Advanced SIMD structure modes. */
1591 static bool
1593 {
1596 }
1598 /* Return true if MODE is an SVE predicate mode. */
1599 static bool
1601 {
1607 }
1609 /* Three mutually-exclusive flags describing a vector or predicate type. */
1613 /* Can be used in combination with VEC_ADVSIMD or VEC_SVE_DATA to indicate
1614 a structure of 2, 3 or 4 vectors. */
1616 /* Can be used in combination with VEC_SVE_DATA to indicate that the
1617 vector has fewer significant bytes than a full SVE vector. */
1619 /* Useful combinations of the above. */
1623 /* Return a set of flags describing the vector properties of mode MODE.
1624 Ignore modes that are not supported by the current target. */
1625 static unsigned int
1627 {
1634 /* Make the decision based on the mode's enum value rather than its
1635 properties, so that we keep the correct classification regardless
1636 of -msve-vector-bits. */
1638 {
1639 /* Partial SVE QI vectors. */
1643 /* Partial SVE HI vectors. */
1646 /* Partial SVE SI vector. */
1648 /* Partial SVE HF vectors. */
1651 /* Partial SVE SF vector. */
1664 /* x2 SVE vectors. */
1672 /* x3 SVE vectors. */
1680 /* x4 SVE vectors. */
1690 /* 64-bit Advanced SIMD vectors. */
1694 /* ...E_V1DImode doesn't exist. */
1699 /* 128-bit Advanced SIMD vectors. */
1712 }
1713 }
1715 /* Return true if MODE is any of the data vector modes, including
1716 structure modes. */
1717 static bool
1719 {
1721 }
1723 /* Return true if MODE is any form of SVE mode, including predicates,
1724 vectors and structures. */
1725 bool
1727 {
1729 }
1731 /* Return true if MODE is an SVE data vector mode; either a single vector
1732 or a structure of vectors. */
1733 static bool
1735 {
1737 }
1739 /* Return the number of defined bytes in one constituent vector of
1740 SVE mode MODE, which has vector flags VEC_FLAGS. */
1741 static poly_int64
1743 {
1745 /* A single partial vector. */
1749 /* A single vector or a tuple. */
1752 /* A single predicate. */
1755 }
1757 /* Implement target hook TARGET_ARRAY_MODE. */
1758 static opt_machine_mode
1760 {
1767 }
1769 /* Implement target hook TARGET_ARRAY_MODE_SUPPORTED_P. */
1770 static bool
1773 {
1781 }
1783 /* MODE is some form of SVE vector mode. For data modes, return the number
1784 of vector register bits that each element of MODE occupies, such as 64
1785 for both VNx2DImode and VNx2SImode (where each 32-bit value is stored
1786 in a 64-bit container). For predicate modes, return the number of
1787 data bits controlled by each significant predicate bit. */
1789 static unsigned int
1791 {
1794 ? BITS_PER_SVE_VECTOR
1797 }
1799 /* Return the SVE predicate mode to use for elements that have
1800 ELEM_NBYTES bytes, if such a mode exists. */
1802 opt_machine_mode
1804 {
1806 {
1815 }
1817 }
1819 /* Return the SVE predicate mode that should be used to control
1820 SVE mode MODE. */
1822 machine_mode
1824 {
1827 }
1829 /* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
1831 static opt_machine_mode
1833 {
1839 }
1841 /* Return the SVE vector mode that has NUNITS elements of mode INNER_MODE. */
1843 opt_machine_mode
1845 {
1848 machine_mode mode;
1855 }
1857 /* Return the integer element mode associated with SVE mode MODE. */
1859 static scalar_int_mode
1861 {
1863 ? BITS_PER_SVE_VECTOR
1868 }
1870 /* Return an integer element mode that contains exactly
1871 aarch64_sve_container_bits (MODE) bits. This is wider than
1872 aarch64_sve_element_int_mode if MODE is a partial vector,
1873 otherwise it's the same. */
1875 static scalar_int_mode
1877 {
1879 }
1881 /* Return the integer vector mode associated with SVE mode MODE.
1882 Unlike related_int_vector_mode, this can handle the case in which
1883 MODE is a predicate (and thus has a different total size). */
1885 machine_mode
1887 {
1890 }
1892 /* Implement TARGET_VECTORIZE_RELATED_MODE. */
1894 static opt_machine_mode
1896 scalar_mode element_mode,
1897 poly_uint64 nunits)
1898 {
1901 /* If we're operating on SVE vectors, try to return an SVE mode. */
1902 poly_uint64 sve_nunits;
1906 {
1907 machine_mode sve_mode;
1909 {
1910 /* Try to find a full or partial SVE mode with exactly
1911 NUNITS units. */
1916 }
1917 else
1918 {
1919 /* Take the preferred number of units from the number of bytes
1920 that fit in VECTOR_MODE. We always start by "autodetecting"
1921 a full vector mode with preferred_simd_mode, so vectors
1922 chosen here will also be full vector modes. Then
1923 autovectorize_vector_modes tries smaller starting modes
1924 and thus smaller preferred numbers of units. */
1929 }
1930 }
1932 /* Prefer to use 1 128-bit vector instead of 2 64-bit vectors. */
1938 {
1942 }
1945 }
1947 /* Implement TARGET_PREFERRED_ELSE_VALUE. For binary operations,
1948 prefer to use the first arithmetic operand as the else value if
1949 the else value doesn't matter, since that exactly matches the SVE
1950 destructive merging form. For ternary operations we could either
1951 pick the first operand and use FMAD-like instructions or the last
1952 operand and use FMLA-like instructions; the latter seems more
1953 natural. */
1955 static tree
1957 {
1959 }
1961 /* Implement TARGET_HARD_REGNO_NREGS. */
1963 static unsigned int
1965 {
1966 /* ??? Logically we should only need to provide a value when
1967 HARD_REGNO_MODE_OK says that the combination is valid,
1968 but at the moment we need to handle all modes. Just ignore
1969 any runtime parts for registers that can't store them. */
1972 {
1976 {
1982 }
1991 }
1993 }
1995 /* Implement TARGET_HARD_REGNO_MODE_OK. */
1997 static bool
1999 {
2004 /* This must have the same size as _Unwind_Word. */
2015 /* The purpose of comparing with ptr_mode is to support the
2016 global register variable associated with the stack pointer
2017 register via the syntax of asm ("wsp") in ILP32. */
2024 {
2031 }
2033 {
2036 else
2038 }
2041 }
2043 /* Return true if TYPE is a type that should be passed or returned in
2044 SVE registers, assuming enough registers are available. When returning
2045 true, set *NUM_ZR and *NUM_PR to the number of required Z and P registers
2046 respectively. */
2048 /* Return true if a function with type FNTYPE returns its value in
2049 SVE vector or predicate registers. */
2051 static bool
2053 {
2057 }
2059 /* Return true if a function with type FNTYPE takes arguments in
2060 SVE vector or predicate registers. */
2062 static bool
2064 {
2065 CUMULATIVE_ARGS args_so_far_v;
2073 {
2084 }
2086 }
2088 /* Implement TARGET_FNTYPE_ABI. */
2092 {
2101 }
2103 /* Implement TARGET_COMPATIBLE_VECTOR_TYPES_P. */
2105 static bool
2107 {
2110 }
2112 /* Return true if we should emit CFI for register REGNO. */
2114 static bool
2116 {
2119 }
2121 /* Return the mode we should use to save and restore register REGNO. */
2123 static machine_mode
2125 {
2131 {
2133 /* Only the low 64 bits are saved by the base PCS. */
2137 /* The vector PCS saves the low 128 bits (which is the full
2138 register on non-SVE targets). */
2142 /* Use vectors of DImode for registers that need frame
2143 information, so that the first 64 bytes of the save slot
2144 are always the equivalent of what storing D<n> would give. */
2148 /* Use vectors of bytes otherwise, so that the layout is
2149 endian-agnostic, and so that we can use LDR and STR for
2150 big-endian targets. */
2156 }
2159 /* Save the full predicate register. */
2163 }
2165 /* Implement TARGET_INSN_CALLEE_ABI. */
2169 {
2176 }
2178 /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The callee only saves
2179 the lower 64 bits of a 128-bit register. Tell the compiler the callee
2180 clobbers the top 64 bits when restoring the bottom 64 bits. */
2182 static bool
2185 machine_mode mode)
2186 {
2188 {
2196 }
2198 }
2200 /* Implement REGMODE_NATURAL_SIZE. */
2201 poly_uint64
2203 {
2204 /* The natural size for SVE data modes is one SVE data vector,
2205 and similarly for predicates. We can't independently modify
2206 anything smaller than that. */
2207 /* ??? For now, only do this for variable-width SVE registers.
2208 Doing it for constant-sized registers breaks lower-subreg.c. */
2209 /* ??? And once that's fixed, we should probably have similar
2210 code for Advanced SIMD. */
2212 {
2218 }
2220 }
2222 /* Implement HARD_REGNO_CALLER_SAVE_MODE. */
2223 machine_mode
2225 machine_mode mode)
2226 {
2227 /* The predicate mode determines which bits are significant and
2228 which are "don't care". Decreasing the number of lanes would
2229 lose data while increasing the number of lanes would make bits
2230 unnecessarily significant. */
2235 else
2237 }
2239 /* Return true if I's bits are consecutive ones from the MSB. */
2240 bool
2242 {
2244 }
2246 /* Implement TARGET_CONSTANT_ALIGNMENT. Make strings word-aligned so
2247 that strcpy from constants will be faster. */
2249 static HOST_WIDE_INT
2251 {
2255 }
2257 /* Return true if calls to DECL should be treated as
2258 long-calls (ie called via a register). */
2259 static bool
2261 {
2263 }
2265 /* Return true if calls to symbol-ref SYM should be treated as
2266 long-calls (ie called via a register). */
2267 bool
2269 {
2271 }
2273 /* Return true if calls to symbol-ref SYM should not go through
2274 plt stubs. */
2276 bool
2278 {
2282 && decl
2283 && (!flag_plt
2289 }
2291 /* Return true if the offsets to a zero/sign-extract operation
2292 represent an expression that matches an extend operation. The
2293 operands represent the paramters from
2295 (extract:MODE (mult (reg) (MULT_IMM)) (EXTRACT_IMM) (const_int 0)). */
2296 bool
2298 rtx extract_imm)
2299 {
2316 }
2318 /* Emit an insn that's a simple single-set. Both the operands must be
2319 known to be valid. */
2322 {
2324 }
2326 /* X and Y are two things to compare using CODE. Emit the compare insn and
2327 return the rtx for register 0 in the proper mode. */
2328 rtx
2330 {
2332 machine_mode cc_mode;
2333 rtx cc_reg;
2336 {
2351 }
2352 else
2353 {
2357 }
2359 }
2361 /* Similarly, but maybe zero-extend Y if Y_MODE < SImode. */
2363 static rtx
2365 machine_mode y_mode)
2366 {
2368 {
2371 else
2372 {
2374 machine_mode cc_mode;
2382 }
2383 }
2389 }
2391 /* Build the SYMBOL_REF for __tls_get_addr. */
2395 rtx
2397 {
2401 }
2403 /* Return the TLS model to use for ADDR. */
2405 static enum tls_model
2407 {
2410 {
2411 poly_int64 addend;
2415 }
2420 }
2422 /* We'll allow lo_sum's in addresses in our legitimate addresses
2423 so that combine would take care of combining addresses where
2424 necessary, but for generation purposes, we'll generate the address
2425 as :
2426 RTL Absolute
2427 tmp = hi (symbol_ref); adrp x1, foo
2428 dest = lo_sum (tmp, symbol_ref); add dest, x1, :lo_12:foo
2429 nop
2431 PIC TLS
2432 adrp x1, :got:foo adrp tmp, :tlsgd:foo
2433 ldr x1, [:got_lo12:foo] add dest, tmp, :tlsgd_lo12:foo
2434 bl __tls_get_addr
2435 nop
2437 Load TLS symbol, depending on TLS mechanism and TLS access model.
2439 Global Dynamic - Traditional TLS:
2440 adrp tmp, :tlsgd:imm
2441 add dest, tmp, #:tlsgd_lo12:imm
2442 bl __tls_get_addr
2444 Global Dynamic - TLS Descriptors:
2445 adrp dest, :tlsdesc:imm
2446 ldr tmp, [dest, #:tlsdesc_lo12:imm]
2447 add dest, dest, #:tlsdesc_lo12:imm
2448 blr tmp
2449 mrs tp, tpidr_el0
2450 add dest, dest, tp
2452 Initial Exec:
2453 mrs tp, tpidr_el0
2454 adrp tmp, :gottprel:imm
2455 ldr dest, [tmp, #:gottprel_lo12:imm]
2456 add dest, dest, tp
2458 Local Exec:
2459 mrs tp, tpidr_el0
2460 add t0, tp, #:tprel_hi12:imm, lsl #12
2461 add t0, t0, #:tprel_lo12_nc:imm
2462 */
2464 static void
2467 {
2469 {
2471 {
2472 /* In ILP32, the mode of dest can be either SImode or DImode. */
2484 }
2491 {
2494 rtx insn;
2495 rtx mem;
2497 /* NOTE: pic_offset_table_rtx can be NULL_RTX, because we can reach
2498 here before rtl expand. Tree IVOPT will generate rtl pattern to
2499 decide rtx costs, in which case pic_offset_table_rtx is not
2500 initialized. For that case no need to generate the first adrp
2501 instruction as the final cost for global variable access is
2502 one instruction. */
2504 {
2505 /* -fpic for -mcmodel=small allow 32K GOT table size (but we are
2506 using the page base as GOT base, the first page may be wasted,
2507 in the worst scenario, there is only 28K space for GOT).
2509 The generate instruction sequence for accessing global variable
2510 is:
2512 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym]
2514 Only one instruction needed. But we must initialize
2515 pic_offset_table_rtx properly. We generate initialize insn for
2516 every global access, and allow CSE to remove all redundant.
2518 The final instruction sequences will look like the following
2519 for multiply global variables access.
2521 adrp pic_offset_table_rtx, _GLOBAL_OFFSET_TABLE_
2523 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym1]
2524 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym2]
2525 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym3]
2526 ... */
2535 }
2538 {
2541 else
2545 }
2546 else
2547 {
2552 }
2554 /* The operand is expected to be MEM. Whenever the related insn
2555 pattern changed, above code which calculate mem should be
2556 updated. */
2562 }
2565 {
2566 /* In ILP32, the mode of dest can be either SImode or DImode,
2567 while the got entry is always of SImode size. The mode of
2568 dest depends on how dest is used: if dest is assigned to a
2569 pointer (e.g. in the memory), it has SImode; it may have
2570 DImode if dest is dereferenced to access the memeory.
2571 This is why we have to handle three different ldr_got_small
2572 patterns here (two patterns for ILP32). */
2574 rtx insn;
2575 rtx mem;
2584 {
2587 else
2591 }
2592 else
2593 {
2598 }
2605 }
2608 {
2616 else
2624 }
2627 {
2630 rtx tp;
2634 /* In ILP32, the got entry is always of SImode size. Unlike
2635 small GOT, the dest is fixed at reg 0. */
2638 else
2649 }
2652 {
2653 /* In ILP32, the mode of dest can be either SImode or DImode,
2654 while the got entry is always of SImode size. The mode of
2655 dest depends on how dest is used: if dest is assigned to a
2656 pointer (e.g. in the memory), it has SImode; it may have
2657 DImode if dest is dereferenced to access the memeory.
2658 This is why we have to handle three different tlsie_small
2659 patterns here (two patterns for ILP32). */
2665 {
2668 else
2669 {
2672 }
2673 }
2674 else
2675 {
2678 }
2684 }
2690 {
2698 {
2721 }
2726 }
2733 {
2738 {
2741 else
2742 {
2745 }
2746 }
2747 else
2748 {
2751 }
2756 }
2760 }
2761 }
2763 /* Emit a move from SRC to DEST. Assume that the move expanders can
2764 handle all moves if !can_create_pseudo_p (). The distinction is
2765 important because, unlike emit_move_insn, the move expanders know
2766 how to force Pmode objects into the constant pool even when the
2767 constant pool address is not itself legitimate. */
2768 static rtx
2770 {
2774 }
2776 /* Apply UNOPTAB to OP and store the result in DEST. */
2778 static void
2780 {
2784 }
2786 /* Apply BINOPTAB to OP0 and OP1 and store the result in DEST. */
2788 static void
2790 {
2792 OPTAB_DIRECT);
2795 }
2797 /* Split a 128-bit move operation into two 64-bit move operations,
2798 taking care to handle partial overlap of register to register
2799 copies. Special cases are needed when moving between GP regs and
2800 FP regs. SRC can be a register, constant or memory; DST a register
2801 or memory. If either operand is memory it must not have any side
2802 effects. */
2803 void
2805 {
2816 {
2820 /* Handle FP <-> GP regs. */
2822 {
2829 }
2831 {
2838 }
2839 }
2846 /* At most one pairing may overlap. */
2848 {
2851 }
2852 else
2853 {
2856 }
2857 }
2859 bool
2861 {
2864 }
2866 /* Split a complex SIMD combine. */
2868 void
2870 {
2881 }
2883 /* Split a complex SIMD move. */
2885 void
2887 {
2894 {
2897 }
2898 }
2900 bool
2903 {
2907 }
2909 /* Return TARGET if it is nonnull and a register of mode MODE.
2910 Otherwise, return a fresh register of mode MODE if we can,
2911 or TARGET reinterpreted as MODE if we can't. */
2913 static rtx
2915 {
2919 {
2922 }
2924 }
2926 /* Return a register that contains the constant in BUILDER, given that
2927 the constant is a legitimate move operand. Use TARGET as the register
2928 if it is nonnull and convenient. */
2930 static rtx
2932 {
2937 }
2939 static rtx
2941 {
2944 else
2945 {
2949 }
2950 }
2952 /* Return true if predicate value X is a constant in which every element
2953 is a CONST_INT. When returning true, describe X in BUILDER as a VNx16BI
2954 value, i.e. as a predicate in which all bits are significant. */
2956 static bool
2958 {
2970 {
2978 }
2981 }
2983 /* BUILDER contains a predicate constant of mode VNx16BI. Return the
2984 widest predicate element size it can have (that is, the largest size
2985 for which each element would still be 0 or 1). */
2987 unsigned int
2989 {
2990 /* Start with the most optimistic assumption: that we only need
2991 one bit per pattern. This is what we will use if only the first
2992 bit in each pattern is ever set. */
2996 /* Look for set bits. */
3000 {
3004 }
3006 }
3008 /* If VNx16BImode rtx X is a canonical PTRUE for a predicate mode,
3009 return that predicate mode, otherwise return opt_machine_mode (). */
3011 opt_machine_mode
3013 {
3027 }
3029 /* BUILDER is a predicate constant of mode VNx16BI. Consider the value
3030 that the constant would have with predicate element size ELT_SIZE
3031 (ignoring the upper bits in each element) and return:
3033 * -1 if all bits are set
3034 * N if the predicate has N leading set bits followed by all clear bits
3035 * 0 if the predicate does not have any of these forms. */
3037 int
3040 {
3041 /* If nelts_per_pattern is 3, we have set bits followed by clear bits
3042 followed by set bits. */
3046 /* Skip over leading set bits. */
3054 /* Check for the all-true case. */
3058 /* If nelts_per_pattern is 1, then either VL is zero, or we have a
3059 repeating pattern of set bits followed by clear bits. */
3063 /* We have a "foreground" value and a duplicated "background" value.
3064 If the background might repeat and the last set bit belongs to it,
3065 we might have set bits followed by clear bits followed by set bits. */
3069 /* Make sure that the rest are all clear. */
3075 }
3077 /* See if there is an svpattern that encodes an SVE predicate of mode
3078 PRED_MODE in which the first VL bits are set and the rest are clear.
3079 Return the pattern if so, otherwise return AARCH64_NUM_SVPATTERNS.
3080 A VL of -1 indicates an all-true vector. */
3082 aarch64_svpattern
3084 {
3099 {
3102 /* These would only trigger for non-power-of-2 lengths. */
3109 }
3111 }
3113 /* Return a VNx16BImode constant in which every sequence of ELT_SIZE
3114 bits has the lowest bit set and the upper bits clear. This is the
3115 VNx16BImode equivalent of a PTRUE for controlling elements of
3116 ELT_SIZE bytes. However, because the constant is VNx16BImode,
3117 all bits are significant, even the upper zeros. */
3119 rtx
3121 {
3127 }
3129 /* Return an all-true predicate register of mode MODE. */
3131 rtx
3133 {
3137 }
3139 /* Return an all-false predicate register of mode MODE. */
3141 rtx
3143 {
3147 }
3149 /* Return true if predicate PRED1[0] is true whenever predicate PRED2 is
3150 true, or alternatively if we know that the operation predicated by
3151 PRED1[0] is safe to perform whenever PRED2 is true. PRED1[1] is a
3152 aarch64_sve_gp_strictness operand that describes the operation
3153 predicated by PRED1[0]. */
3155 bool
3157 {
3165 }
3167 /* PRED1[0] is a PTEST predicate and PRED1[1] is an aarch64_sve_ptrue_flag
3168 for it. PRED2[0] is the predicate for the instruction whose result
3169 is tested by the PTEST and PRED2[1] is again an aarch64_sve_ptrue_flag
3170 for it. Return true if we can prove that the two predicates are
3171 equivalent for PTEST purposes; that is, if we can replace PRED2[0]
3172 with PRED1[0] without changing behavior. */
3174 bool
3176 {
3188 }
3190 /* Emit a comparison CMP between OP0 and OP1, both of which have mode
3191 DATA_MODE, and return the result in a predicate of mode PRED_MODE.
3192 Use TARGET as the target register if nonnull and convenient. */
3194 static rtx
3197 {
3207 }
3209 /* Use a comparison to convert integer vector SRC into MODE, which is
3210 the corresponding SVE predicate mode. Use TARGET for the result
3211 if it's nonnull and convenient. */
3213 rtx
3215 {
3219 }
3221 /* Return the assembly token for svprfop value PRFOP. */
3225 {
3227 {
3228 #define CASE(UPPER, LOWER, VALUE) case AARCH64_SV_##UPPER: return #LOWER;
3230 #undef CASE
3233 }
3235 }
3237 /* Return the assembly string for an SVE prefetch operation with
3238 mnemonic MNEMONIC, given that PRFOP_RTX is the prefetch operation
3239 and that SUFFIX is the format for the remaining operands. */
3244 {
3251 }
3253 /* Check whether we can calculate the number of elements in PATTERN
3254 at compile time, given that there are NELTS_PER_VQ elements per
3255 128-bit block. Return the value if so, otherwise return -1. */
3257 HOST_WIDE_INT
3259 {
3266 {
3267 /* There are two vector granules per quadword. */
3270 {
3276 }
3277 }
3278 else
3281 /* There are two vector granules per quadword. */
3286 /* Requesting more elements than are available results in a PFALSE. */
3291 }
3293 /* Return true if we can move VALUE into a register using a single
3294 CNT[BHWD] instruction. */
3296 static bool
3298 {
3300 /* The coefficient must be [1, 16] * {2, 4, 8, 16}. */
3305 }
3307 /* Likewise for rtx X. */
3309 bool
3311 {
3312 poly_int64 value;
3314 }
3316 /* Return the asm string for an instruction with a CNT-like vector size
3317 operand (a vector pattern followed by a multiplier in the range [1, 16]).
3318 PREFIX is the mnemonic without the size suffix and OPERANDS is the
3319 first part of the operands template (the part that comes before the
3320 vector size itself). PATTERN is the pattern to use. FACTOR is the
3321 number of quadwords. NELTS_PER_VQ, if nonzero, is the number of elements
3322 in each quadword. If it is zero, we can use any element size. */
3326 aarch64_svpattern pattern,
3329 {
3333 /* There is some overlap in the ranges of the four CNT instructions.
3334 Here we always use the smallest possible element size, so that the
3335 multiplier is 1 whereever possible. */
3349 else
3352 factor);
3355 }
3357 /* Return the asm string for an instruction with a CNT-like vector size
3358 operand (a vector pattern followed by a multiplier in the range [1, 16]).
3359 PREFIX is the mnemonic without the size suffix and OPERANDS is the
3360 first part of the operands template (the part that comes before the
3361 vector size itself). X is the value of the vector size operand,
3362 as a polynomial integer rtx; we need to convert this into an "all"
3363 pattern with a multiplier. */
3367 rtx x)
3368 {
3373 }
3375 /* Return the asm string for an instruction with a CNT-like vector size
3376 operand (a vector pattern followed by a multiplier in the range [1, 16]).
3377 PREFIX is the mnemonic without the size suffix and OPERANDS is the
3378 first part of the operands template (the part that comes before the
3379 vector size itself). CNT_PAT[0..2] are the operands of the
3380 UNSPEC_SVE_CNT_PAT; see aarch64_sve_cnt_pat for details. */
3385 {
3391 }
3393 /* Return true if we can add X using a single SVE INC or DEC instruction. */
3395 bool
3397 {
3398 poly_int64 value;
3402 }
3404 /* Return the asm string for adding SVE INC/DEC immediate OFFSET to
3405 operand 0. */
3409 {
3415 else
3418 }
3420 /* Return true if we can add VALUE to a register using a single ADDVL
3421 or ADDPL instruction. */
3423 static bool
3425 {
3429 /* FACTOR counts VG / 2, so a value of 2 is one predicate width
3430 and a value of 16 is one vector width. */
3433 }
3435 /* Likewise for rtx X. */
3437 bool
3439 {
3440 poly_int64 value;
3443 }
3445 /* Return the asm string for adding ADDVL or ADDPL immediate OFFSET
3446 to operand 1 and storing the result in operand 0. */
3450 {
3458 else
3461 }
3463 /* Return true if X is a valid immediate for an SVE vector INC or DEC
3464 instruction. If it is, store the number of elements in each vector
3465 quadword in *NELTS_PER_VQ_OUT (if nonnull) and store the multiplication
3466 factor in *FACTOR_OUT (if nonnull). */
3468 bool
3471 {
3472 rtx elt;
3473 poly_int64 value;
3481 /* There's no vector INCB. */
3488 /* The coefficient must be [1, 16] * NELTS_PER_VQ. */
3498 }
3500 /* Return true if X is a valid immediate for an SVE vector INC or DEC
3501 instruction. */
3503 bool
3505 {
3507 }
3509 /* Return the asm template for an SVE vector INC or DEC instruction.
3510 OPERANDS gives the operands before the vector count and X is the
3511 value of the vector count operand itself. */
3515 {
3523 else
3526 }
3528 static int
3530 scalar_int_mode mode)
3531 {
3540 {
3544 }
3546 /* Check to see if the low 32 bits are either 0xffffXXXX or 0xXXXXffff
3547 (with XXXX non-zero). In that case check to see if the move can be done in
3548 a smaller mode. */
3553 {
3557 /* Check if we have to emit a second instruction by checking to see
3558 if any of the upper 32 bits of the original DI mode value is set. */
3569 }
3572 {
3574 {
3579 else
3582 }
3584 }
3586 /* Remaining cases are all for DImode. */
3595 {
3596 /* Try emitting a bitmask immediate with a movk replacing 16 bits.
3597 For a 64-bit bitmask try whether changing 16 bits to all ones or
3598 zeroes creates a valid bitmask. To check any repeated bitmask,
3599 try using 16 bits from the other 32-bit half of val. */
3602 {
3613 }
3615 {
3617 {
3621 }
3623 }
3624 }
3626 /* Generate 2-4 instructions, skipping 16 bits of all zeroes or ones which
3627 are emitted by the initial mov. If one_match > zero_match, skip set bits,
3628 otherwise skip zero bits. */
3640 {
3646 num_insns ++;
3647 }
3650 }
3652 /* Return whether imm is a 128-bit immediate which is simple enough to
3653 expand inline. */
3654 bool
3656 {
3667 }
3670 /* Return the number of temporary registers that aarch64_add_offset_1
3671 would need to add OFFSET to a register. */
3673 static unsigned int
3675 {
3677 }
3679 /* A subroutine of aarch64_add_offset. Set DEST to SRC + OFFSET for
3680 a non-polynomial OFFSET. MODE is the mode of the addition.
3681 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
3682 be set and CFA adjustments added to the generated instructions.
3684 TEMP1, if nonnull, is a register of mode MODE that can be used as a
3685 temporary if register allocation is already complete. This temporary
3686 register may overlap DEST but must not overlap SRC. If TEMP1 is known
3687 to hold abs (OFFSET), EMIT_MOVE_IMM can be set to false to avoid emitting
3688 the immediate again.
3690 Since this function may be used to adjust the stack pointer, we must
3691 ensure that it cannot cause transient stack deallocation (for example
3692 by first incrementing SP and then decrementing when adjusting by a
3693 large immediate). */
3695 static void
3699 {
3707 {
3709 {
3712 }
3714 }
3716 /* Single instruction adjustment. */
3718 {
3722 }
3724 /* Emit 2 additions/subtractions if the adjustment is less than 24 bits
3725 and either:
3727 a) the offset cannot be loaded by a 16-bit move or
3728 b) there is no spare register into which we can move it. */
3732 {
3741 }
3743 /* Emit a move immediate if required and an addition/subtraction. */
3745 {
3748 }
3753 {
3757 }
3758 }
3760 /* Return the number of temporary registers that aarch64_add_offset
3761 would need to move OFFSET into a register or add OFFSET to a register;
3762 ADD_P is true if we want the latter rather than the former. */
3764 static unsigned int
3766 {
3767 /* This follows the same structure as aarch64_add_offset. */
3776 /* Need one register for the ADDVL/ADDPL result. */
3779 {
3782 /* Need one register for the CNT result and one for the multiplication
3783 factor. If necessary, the second temporary can be reused for the
3784 constant part of the offset. */
3786 /* Need one register for the CNT result (which might then
3787 be shifted). */
3789 }
3791 }
3793 /* If X can be represented as a poly_int64, return the number
3794 of temporaries that are required to add it to a register.
3795 Return -1 otherwise. */
3797 int
3799 {
3800 poly_int64 offset;
3804 }
3806 /* Set DEST to SRC + OFFSET. MODE is the mode of the addition.
3807 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
3808 be set and CFA adjustments added to the generated instructions.
3810 TEMP1, if nonnull, is a register of mode MODE that can be used as a
3811 temporary if register allocation is already complete. This temporary
3812 register may overlap DEST if !FRAME_RELATED_P but must not overlap SRC.
3813 If TEMP1 is known to hold abs (OFFSET), EMIT_MOVE_IMM can be set to
3814 false to avoid emitting the immediate again.
3816 TEMP2, if nonnull, is a second temporary register that doesn't
3817 overlap either DEST or REG.
3819 Since this function may be used to adjust the stack pointer, we must
3820 ensure that it cannot cause transient stack deallocation (for example
3821 by first incrementing SP and then decrementing when adjusting by a
3822 large immediate). */
3824 static void
3828 {
3832 || !frame_related_p
3836 /* Try using ADDVL or ADDPL to add the whole value. */
3838 {
3843 }
3845 /* Coefficient 1 is multiplied by the number of 128-bit blocks in an
3846 SVE vector register, over and above the minimum size of 128 bits.
3847 This is equivalent to half the value returned by CNTD with a
3848 vector shape of ALL. */
3852 /* Try using ADDVL or ADDPL to add the VG-based part. */
3856 {
3859 {
3863 }
3864 else
3865 {
3870 }
3871 }
3872 /* Otherwise use a CNT-based sequence. */
3874 {
3875 /* Use a subtraction if we have a negative factor. */
3878 {
3881 }
3883 /* Calculate CNTD * FACTOR / 2. First try to fold the division
3884 into the multiplication. */
3885 rtx val;
3888 /* Use a right shift by 1. */
3890 else
3894 {
3896 {
3897 /* "CNTB Xn, ALL, MUL #FACTOR" is out of range, so calculate
3898 the value with the minimum multiplier and shift it into
3899 position. */
3903 }
3905 }
3906 else
3907 {
3908 /* Base the factor on LOW_BIT if we can calculate LOW_BIT
3909 directly, since that should increase the chances of being
3910 able to use a shift and add sequence. If LOW_BIT itself
3911 is out of range, just use CNTD. */
3914 else
3921 {
3924 }
3925 else
3926 {
3927 /* Go back to using a negative multiplication factor if we have
3928 no register from which to subtract. */
3930 {
3933 }
3937 }
3938 }
3941 {
3942 /* Multiply by 1 << SHIFT. */
3945 }
3947 {
3948 /* Divide by 2. */
3951 }
3953 /* Calculate SRC +/- CNTD * FACTOR / 2. */
3955 {
3958 }
3960 {
3963 }
3966 {
3969 {
3973 poly_offset)));
3974 }
3978 }
3979 else
3980 {
3984 }
3987 }
3991 }
3993 /* Like aarch64_add_offset, but the offset is given as an rtx rather
3994 than a poly_int64. */
3996 void
3999 {
4002 }
4004 /* Add DELTA to the stack pointer, marking the instructions frame-related.
4005 TEMP1 is available as a temporary if nonnull. EMIT_MOVE_IMM is false
4006 if TEMP1 already contains abs (DELTA). */
4010 {
4013 }
4015 /* Subtract DELTA from the stack pointer, marking the instructions
4016 frame-related if FRAME_RELATED_P. TEMP1 is available as a temporary
4017 if nonnull. */
4022 {
4025 }
4027 /* Set DEST to (vec_series BASE STEP). */
4029 static void
4031 {
4035 /* Each operand can be a register or an immediate in the range [-16, 15]. */
4042 }
4044 /* Duplicate 128-bit Advanced SIMD vector SRC so that it fills an SVE
4045 register of mode MODE. Use TARGET for the result if it's nonnull
4046 and convenient.
4048 The two vector modes must have the same element mode. The behavior
4049 is to duplicate architectural lane N of SRC into architectural lanes
4050 N + I * STEP of the result. On big-endian targets, architectural
4051 lane 0 of an Advanced SIMD vector is the last element of the vector
4052 in memory layout, so for big-endian targets this operation has the
4053 effect of reversing SRC before duplicating it. Callers need to
4054 account for this. */
4056 rtx
4058 {
4061 insn_code icode = (BYTES_BIG_ENDIAN
4070 {
4071 /* Create a PARALLEL describing the reversal of SRC. */
4076 }
4079 }
4081 /* Try to force 128-bit vector value SRC into memory and use LD1RQ to fetch
4082 the memory image into DEST. Return true on success. */
4084 static bool
4086 {
4091 /* Make sure that the address is legitimate. */
4093 {
4096 }
4103 }
4105 /* Return a register containing CONST_VECTOR SRC, given that SRC has an
4106 SVE data mode and isn't a legitimate constant. Use TARGET for the
4107 result if convenient.
4109 The returned register can have whatever mode seems most natural
4110 given the contents of SRC. */
4112 static rtx
4114 {
4126 {
4127 /* We have a partial vector mode and a constant whose full-vector
4128 equivalent would occupy a repeating 128-bit sequence. Build that
4129 full-vector equivalent instead, so that we have the option of
4130 using LD1RQ and Advanced SIMD operations. */
4139 }
4142 {
4143 /* The constant is a duplicated quadword but can't be narrowed
4144 beyond a quadword. Get the memory image of the first quadword
4145 as a 128-bit vector and try using LD1RQ to load it from memory.
4147 The effect for both endiannesses is to load memory lane N into
4148 architectural lanes N + I * STEP of the result. On big-endian
4149 targets, the layout of the 128-bit vector in an Advanced SIMD
4150 register would be different from its layout in an SVE register,
4151 but this 128-bit vector is a memory value only. */
4156 }
4159 {
4160 /* The vector is a repeating sequence of 64 bits or fewer.
4161 See if we can load them using an Advanced SIMD move and then
4162 duplicate it to fill a vector. This is better than using a GPR
4163 move because it keeps everything in the same register file. */
4167 {
4168 /* We want memory lane N to go into architectural lane N,
4169 so reverse for big-endian targets. The DUP .Q pattern
4170 has a compensating reverse built-in. */
4173 }
4176 {
4179 }
4181 /* Get an integer representation of the repeating part of Advanced
4182 SIMD vector VQ_SRC. This preserves the endianness of VQ_SRC,
4183 which for big-endian targets is lane-swapped wrt a normal
4184 Advanced SIMD vector. This means that for both endiannesses,
4185 memory lane N of SVE vector SRC corresponds to architectural
4186 lane N of a register holding VQ_SRC. This in turn means that
4187 memory lane 0 of SVE vector SRC is in the lsb of VQ_SRC (viewed
4188 as a single 128-bit value) and thus that memory lane 0 of SRC is
4189 in the lsb of the integer. Duplicating the integer therefore
4190 ensures that memory lane N of SRC goes into architectural lane
4191 N + I * INDEX of the SVE register. */
4195 {
4196 /* Pretend that we had a vector of INT_MODE to start with. */
4200 /* If the integer can be moved into a general register by a
4201 single instruction, do that and duplicate the result. */
4204 {
4207 }
4208 }
4210 /* We're duplicating a single value, but can't do better than
4211 force it to memory and load from there. This handles things
4212 like symbolic constants. */
4216 {
4217 /* Load the element from memory if we can, otherwise move it into
4218 a register and use a DUP. */
4223 }
4224 }
4226 /* Try using INDEX. */
4229 {
4232 }
4234 /* From here on, it's better to force the whole constant to memory
4235 if we can. */
4239 /* Expand each pattern individually. */
4241 rtx_vector_builder builder;
4244 {
4249 }
4251 /* Use permutes to interleave the separate vectors. */
4253 {
4256 {
4261 }
4262 }
4265 }
4267 /* Use WHILE to set a predicate register of mode MODE in which the first
4268 VL bits are set and the rest are clear. Use TARGET for the register
4269 if it's nonnull and convenient. */
4271 static rtx
4274 {
4280 }
4282 static rtx
4285 /* BUILDER is a constant predicate in which the index of every set bit
4286 is a multiple of ELT_SIZE (which is <= 8). Try to load the constant
4287 by inverting every element at a multiple of ELT_SIZE and EORing the
4288 result with an ELT_SIZE PTRUE.
4290 Return a register that contains the constant on success, otherwise
4291 return null. Use TARGET as the register if it is nonnull and
4292 convenient. */
4294 static rtx
4297 {
4298 /* Invert every element at a multiple of ELT_SIZE, keeping the
4299 other bits zero. */
4305 else
4309 /* See if we can load the constant cheaply. */
4314 /* EOR the result with an ELT_SIZE PTRUE. */
4320 }
4322 /* BUILDER is a constant predicate in which the index of every set bit
4323 is a multiple of ELT_SIZE (which is <= 8). Try to load the constant
4324 using a TRN1 of size PERMUTE_SIZE, which is >= ELT_SIZE. Return the
4325 register on success, otherwise return null. Use TARGET as the register
4326 if nonnull and convenient. */
4328 static rtx
4332 {
4333 /* We're going to split the constant into two new constants A and B,
4334 with element I of BUILDER going into A if (I & PERMUTE_SIZE) == 0
4335 and into B otherwise. E.g. for PERMUTE_SIZE == 4 && ELT_SIZE == 1:
4337 A: { 0, 1, 2, 3, _, _, _, _, 8, 9, 10, 11, _, _, _, _ }
4338 B: { 4, 5, 6, 7, _, _, _, _, 12, 13, 14, 15, _, _, _, _ }
4340 where _ indicates elements that will be discarded by the permute.
4342 First calculate the ELT_SIZEs for A and B. */
4347 {
4350 else
4352 }
4356 /* Now construct the vectors themselves. */
4364 {
4367 }
4369 {
4370 /* The A and B elements are significant. */
4373 }
4374 else
4375 {
4376 /* The A and B elements are going to be discarded, so pick whatever
4377 is likely to give a nice constant. We are targeting element
4378 sizes A_ELT_SIZE and B_ELT_SIZE for A and B respectively,
4379 with the aim of each being a sequence of ones followed by
4380 a sequence of zeros. So:
4382 * if X_ELT_SIZE <= PERMUTE_SIZE, the best approach is to
4383 duplicate the last X_ELT_SIZE element, to extend the
4384 current sequence of ones or zeros.
4386 * if X_ELT_SIZE > PERMUTE_SIZE, the best approach is to add a
4387 zero, so that the constant really does have X_ELT_SIZE and
4388 not a smaller size. */
4391 else
4395 else
4397 }
4401 /* Try loading A into a register. */
4407 /* Try loading B into a register. */
4410 {
4413 {
4416 }
4417 }
4419 /* Emit the TRN1 itself. */
4426 }
4428 /* Subroutine of aarch64_expand_sve_const_pred. Try to load the VNx16BI
4429 constant in BUILDER into an SVE predicate register. Return the register
4430 on success, otherwise return null. Use TARGET for the register if
4431 nonnull and convenient.
4433 ALLOW_RECURSE_P is true if we can use methods that would call this
4434 function recursively. */
4436 static rtx
4439 {
4441 /* A PFALSE or a PTRUE .B ALL. */
4446 {
4447 /* If we can load the constant using PTRUE, use it as-is. */
4452 /* Otherwise use WHILE to set the first VL bits. */
4454 }
4459 /* Try inverting the vector in element size ELT_SIZE and then EORing
4460 the result with an ELT_SIZE PTRUE. */
4463 elt_size))
4466 /* Try using TRN1 to permute two simpler constants. */
4473 }
4475 /* Return an SVE predicate register that contains the VNx16BImode
4476 constant in BUILDER, without going through the move expanders.
4478 The returned register can have whatever mode seems most natural
4479 given the contents of BUILDER. Use TARGET for the result if
4480 convenient. */
4482 static rtx
4484 {
4485 /* Try loading the constant using pure predicate operations. */
4489 /* Try forcing the constant to memory. */
4492 {
4496 }
4498 /* The last resort is to load the constant as an integer and then
4499 compare it against zero. Use -1 for set bits in order to increase
4500 the changes of using SVE DUPM or an Advanced SIMD byte mask. */
4508 }
4510 /* Set DEST to immediate IMM. */
4512 void
4514 {
4517 /* Check on what type of symbol it is. */
4518 scalar_int_mode int_mode;
4524 {
4525 rtx mem;
4526 poly_int64 offset;
4527 HOST_WIDE_INT const_offset;
4530 /* If we have (const (plus symbol offset)), separate out the offset
4531 before we start classifying the symbol. */
4534 /* We must always add an offset involving VL separately, rather than
4535 folding it into the relocation. */
4537 {
4539 {
4542 }
4545 else
4546 {
4547 /* Do arithmetic on 32-bit values if the result is smaller
4548 than that. */
4550 {
4551 /* It is invalid to do symbol calculations in modes
4552 narrower than SImode. */
4556 }
4558 {
4562 }
4563 else
4566 }
4568 }
4572 {
4576 {
4582 }
4587 /* If we aren't generating PC relative literals, then
4588 we need to expand the literal pool access carefully.
4589 This is something that needs to be done in a number
4590 of places, so could well live as a separate function. */
4592 {
4599 }
4616 {
4622 }
4623 /* FALLTHRU */
4636 }
4637 }
4640 {
4642 {
4643 /* Only the low bit of each .H, .S and .D element is defined,
4644 so we can set the upper bits to whatever we like. If the
4645 predicate is all-true in MODE, prefer to set all the undefined
4646 bits as well, so that we can share a single .B predicate for
4647 all modes. */
4651 /* All methods for constructing predicate modes wider than VNx16BI
4652 will set the upper bits of each element to zero. Expose this
4653 by moving such constants as a VNx16BI, so that all bits are
4654 significant and so that constants for different modes can be
4655 shared. The wider constant will still be available as a
4656 REG_EQUAL note. */
4657 rtx_vector_builder builder;
4659 {
4664 }
4665 }
4669 {
4672 }
4676 {
4680 }
4686 }
4690 }
4692 /* Emit an SVE predicated move from SRC to DEST. PRED is a predicate
4693 that is known to contain PTRUE. */
4695 void
4697 {
4705 }
4707 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one
4708 operand is in memory. In this case we need to use the predicated LD1
4709 and ST1 instead of LDR and STR, both for correctness on big-endian
4710 targets and because LD1 and ST1 support a wider range of addressing modes.
4711 PRED_MODE is the mode of the predicate.
4713 See the comment at the head of aarch64-sve.md for details about the
4714 big-endian handling. */
4716 void
4718 {
4723 {
4727 else
4730 }
4732 }
4734 /* Called only on big-endian targets. See whether an SVE vector move
4735 from SRC to DEST is effectively a REV[BHW] instruction, because at
4736 least one operand is a subreg of an SVE vector that has wider or
4737 narrower elements. Return true and emit the instruction if so.
4739 For example:
4741 (set (reg:VNx8HI R1) (subreg:VNx8HI (reg:VNx16QI R2) 0))
4743 represents a VIEW_CONVERT between the following vectors, viewed
4744 in memory order:
4746 R2: { [0].high, [0].low, [1].high, [1].low, ... }
4747 R1: { [0], [1], [2], [3], ... }
4749 The high part of lane X in R2 should therefore correspond to lane X*2
4750 of R1, but the register representations are:
4752 msb lsb
4753 R2: ...... [1].high [1].low [0].high [0].low
4754 R1: ...... [3] [2] [1] [0]
4756 where the low part of lane X in R2 corresponds to lane X*2 in R1.
4757 We therefore need a reverse operation to swap the high and low values
4758 around.
4760 This is purely an optimization. Without it we would spill the
4761 subreg operand to the stack in one mode and reload it in the
4762 other mode, which has the same effect as the REV. */
4764 bool
4766 {
4773 /* The optimization handles two single SVE REGs with different element
4774 sizes. */
4783 /* Generate *aarch64_sve_mov<mode>_subreg_be. */
4786 UNSPEC_REV_SUBREG);
4789 }
4791 /* Return a copy of X with mode MODE, without changing its other
4792 attributes. Unlike gen_lowpart, this doesn't care whether the
4793 mode change is valid. */
4795 rtx
4797 {
4804 }
4806 /* Return the SVE REV[BHW] unspec for reversing quantites of mode MODE
4807 stored in wider integer containers. */
4809 static unsigned int
4811 {
4813 {
4817 }
4819 }
4821 /* Split a *aarch64_sve_mov<mode>_subreg_be pattern with the given
4822 operands. */
4824 void
4826 {
4827 /* Decide which REV operation we need. The mode with wider elements
4828 determines the mode of the operands and the mode with the narrower
4829 elements determines the reverse width. */
4839 /* Get the operands in the appropriate modes and emit the instruction. */
4845 }
4847 static bool
4849 {
4854 }
4856 /* Implement TARGET_PASS_BY_REFERENCE. */
4858 static bool
4861 {
4863 HOST_WIDE_INT size;
4864 machine_mode dummymode;
4869 {
4871 /* We can't gracefully recover at this point, so make this a
4872 fatal error. */
4876 /* Variadic SVE types are passed by reference. Normal non-variadic
4877 arguments are too if we've run out of registers. */
4881 }
4883 /* GET_MODE_SIZE (BLKmode) is useless since it is 0. */
4886 else
4887 /* No frontends can create types with variable-sized modes, so we
4888 shouldn't be asked to pass or return them. */
4891 /* Aggregates are passed by reference based on their size. */
4895 /* Variable sized arguments are always returned by reference. */
4899 /* Can this be a candidate to be passed in fp/simd register(s)? */
4902 NULL))
4905 /* Arguments which are variable sized or larger than 2 registers are
4906 passed by reference unless they are a homogenous floating point
4907 aggregate. */
4909 }
4911 /* Return TRUE if VALTYPE is padded to its least significant bits. */
4912 static bool
4914 {
4915 machine_mode dummy_mode;
4918 /* Never happens in little-endian mode. */
4922 /* Only composite types smaller than or equal to 16 bytes can
4923 be potentially returned in registers. */
4929 /* But not a composite that is an HFA (Homogeneous Floating-point Aggregate)
4930 or an HVA (Homogeneous Short-Vector Aggregate); such a special composite
4931 is always passed/returned in the least significant bits of fp/simd
4932 register(s). */
4938 }
4940 /* Subroutine of aarch64_function_value. MODE is the mode of the argument
4941 after promotion, and after partial SVE types have been replaced by
4942 their integer equivalents. */
4943 static rtx
4945 {
4948 {
4949 /* Don't raise an error here if we're called when SVE is disabled,
4950 since this is really just a query function. Other code must
4951 do that where appropriate. */
4963 }
4965 /* Generic vectors that map to SVE modes with -msve-vector-bits=N are
4966 returned in memory, not by value. */
4970 {
4974 {
4977 }
4978 }
4981 machine_mode ag_mode;
4984 {
4986 {
4989 }
4990 else
4991 {
4993 rtx par;
4997 {
5002 }
5004 }
5005 }
5006 else
5008 }
5010 /* Implement TARGET_FUNCTION_VALUE.
5011 Define how to find the value returned by a function. */
5013 static rtx
5016 {
5017 machine_mode mode;
5024 /* Vector types can acquire a partial SVE mode using things like
5025 __attribute__((vector_size(N))), and this is potentially useful.
5026 However, the choice of mode doesn't affect the type's ABI identity,
5027 so we should treat the types as though they had the associated
5028 integer mode, just like they did before SVE was introduced.
5030 We know that the vector must be 128 bits or smaller, otherwise we'd
5031 have returned it in memory instead. */
5034 {
5037 /* Vector types are never returned in the MSB and are never split. */
5041 }
5044 }
5046 /* Implements TARGET_FUNCTION_VALUE_REGNO_P.
5047 Return true if REGNO is the number of a hard register in which the values
5048 of called function may come back. */
5050 static bool
5052 {
5053 /* Maximum of 16 bytes can be returned in the general registers. Examples
5054 of 16-byte return values are: 128-bit integers and 16-byte small
5055 structures (excluding homogeneous floating-point aggregates). */
5059 /* Up to four fp/simd registers can return a function value, e.g. a
5060 homogeneous floating-point aggregate having four members. */
5065 }
5067 /* Implement TARGET_RETURN_IN_MEMORY.
5069 If the type T of the result of a function is such that
5070 void func (T arg)
5071 would require that arg be passed as a value in a register (or set of
5072 registers) according to the parameter passing rules, then the result
5073 is returned in the same registers as would be used for such an
5074 argument. */
5076 static bool
5078 {
5079 HOST_WIDE_INT size;
5080 machine_mode ag_mode;
5086 /* Simple scalar types always returned in registers. */
5091 {
5092 /* All SVE types we support fit in registers. For example, it isn't
5093 yet possible to define an aggregate of 9+ SVE vectors or 5+ SVE
5094 predicates. */
5097 }
5100 type,
5103 NULL))
5106 /* Types larger than 2 registers returned in memory. */
5109 }
5111 static bool
5114 {
5117 type,
5119 nregs,
5120 NULL);
5121 }
5123 /* Given MODE and TYPE of a function argument, return the alignment in
5124 bits. The idea is to suppress any stronger alignment requested by
5125 the user and opt for the natural alignment (specified in AAPCS64 \S
5126 4.1). ABI_BREAK is set to true if the alignment was incorrectly
5127 calculated in versions of GCC prior to GCC-9. This is a helper
5128 function for local use only. */
5130 static unsigned int
5133 {
5153 {
5156 bitfield_alignment
5159 }
5162 {
5165 }
5168 }
5170 /* Layout a function argument according to the AAPCS64 rules. The rule
5171 numbers refer to the rule numbers in the AAPCS64. ORIG_MODE is the
5172 mode that was originally given to us by the target hook, whereas the
5173 mode in ARG might be the result of replacing partial SVE modes with
5174 the equivalent integer mode. */
5176 static void
5178 machine_mode orig_mode)
5179 {
5185 HOST_WIDE_INT size;
5188 /* We need to do this once per argument. */
5192 /* Vector types can acquire a partial SVE mode using things like
5193 __attribute__((vector_size(N))), and this is potentially useful.
5194 However, the choice of mode doesn't affect the type's ABI identity,
5195 so we should treat the types as though they had the associated
5196 integer mode, just like they did before SVE was introduced.
5198 We know that the vector must be 128 bits or smaller, otherwise we'd
5199 have passed it by reference instead. */
5202 {
5207 {
5211 }
5213 }
5219 {
5220 /* The PCS says that it is invalid to pass an SVE value to an
5221 unprototyped function. There is no ABI-defined location we
5222 can return in this case, so we have no real choice but to raise
5223 an error immediately, even though this is only a query function. */
5225 {
5229 /* Avoid repeating the message, and avoid tripping the assert
5230 below. */
5232 }
5234 /* We would have converted the argument into pass-by-reference
5235 form if it didn't fit in registers. */
5248 else
5251 }
5253 /* Generic vectors that map to SVE modes with -msve-vector-bits=N are
5254 passed by reference, not by value. */
5257 /* Size in bytes, rounded to the nearest multiple of 8 bytes. */
5260 else
5261 /* No frontends can create types with variable-sized modes, so we
5262 shouldn't be asked to pass or return them. */
5268 mode,
5269 type,
5272 /* allocate_ncrn may be false-positive, but allocate_nvrn is quite reliable.
5273 The following code thus handles passing by SIMD/FP registers first. */
5277 /* C1 - C5 for floating point, homogenous floating point aggregates (HFA)
5278 and homogenous short-vector aggregates (HVA). */
5280 {
5285 {
5288 {
5291 }
5292 else
5293 {
5294 rtx par;
5298 {
5301 rtx offset = gen_int_mode
5305 }
5307 }
5309 }
5310 else
5311 {
5312 /* C.3 NSRN is set to 8. */
5315 }
5316 }
5321 /* C6 - C9. though the sign and zero extension semantics are
5322 handled elsewhere. This is the case where the argument fits
5323 entirely general registers. */
5325 {
5328 /* C.8 if the argument has an alignment of 16 then the NGRN is
5329 rounded up to the next even number. */
5332 /* The == 16 * BITS_PER_UNIT instead of >= 16 * BITS_PER_UNIT
5333 comparison is there because for > 16 * BITS_PER_UNIT
5334 alignment nregs should be > 2 and therefore it should be
5335 passed by reference rather than value. */
5338 {
5344 }
5346 /* NREGS can be 0 when e.g. an empty structure is to be passed.
5347 A reg is still generated for it, but the caller should be smart
5348 enough not to use it. */
5351 else
5352 {
5353 rtx par;
5358 {
5363 }
5365 }
5369 }
5371 /* C.11 */
5374 /* The argument is passed on stack; record the needed number of words for
5375 this argument and align the total size if necessary. */
5376 on_stack:
5381 {
5384 {
5389 }
5390 }
5392 }
5394 /* Implement TARGET_FUNCTION_ARG. */
5396 static rtx
5398 {
5409 }
5411 void
5413 const_tree fntype,
5414 rtx libname ATTRIBUTE_UNUSED,
5415 const_tree fndecl ATTRIBUTE_UNUSED,
5418 {
5427 else
5436 && !TARGET_FLOAT
5439 {
5446 }
5449 && !TARGET_SVE
5451 {
5452 /* We can't gracefully recover at this point, so make this a
5453 fatal error. */
5456 fndecl);
5457 else
5460 }
5461 }
5463 static void
5466 {
5471 {
5482 }
5483 }
5485 bool
5487 {
5490 }
5492 /* Implement FUNCTION_ARG_BOUNDARY. Every parameter gets at least
5493 PARM_BOUNDARY bits of alignment, but will be given anything up
5494 to STACK_BOUNDARY bits if the type requires it. This makes sure
5495 that both before and after the layout of each argument, the Next
5496 Stacked Argument Address (NSAA) will have a minimum alignment of
5497 8 bytes. */
5499 static unsigned int
5501 {
5510 }
5512 /* Implement TARGET_GET_RAW_RESULT_MODE and TARGET_GET_RAW_ARG_MODE. */
5514 static fixed_size_mode
5516 {
5518 /* Don't use the SVE part of the register for __builtin_apply and
5519 __builtin_return. The SVE registers aren't used by the normal PCS,
5520 so using them there would be a waste of time. The PCS extensions
5521 for SVE types are fundamentally incompatible with the
5522 __builtin_return/__builtin_apply interface. */
5525 }
5527 /* Implement TARGET_FUNCTION_ARG_PADDING.
5529 Small aggregate types are placed in the lowest memory address.
5531 The related parameter passing rules are B.4, C.3, C.5 and C.14. */
5533 static pad_direction
5535 {
5536 /* On little-endian targets, the least significant byte of every stack
5537 argument is passed at the lowest byte address of the stack slot. */
5541 /* Otherwise, integral, floating-point and pointer types are padded downward:
5542 the least significant byte of a stack argument is passed at the highest
5543 byte address of the stack slot. */
5550 /* Everything else padded upward, i.e. data in first byte of stack slot. */
5552 }
5554 /* Similarly, for use by BLOCK_REG_PADDING (MODE, TYPE, FIRST).
5556 It specifies padding for the last (may also be the only)
5557 element of a block move between registers and memory. If
5558 assuming the block is in the memory, padding upward means that
5559 the last element is padded after its highest significant byte,
5560 while in downward padding, the last element is padded at the
5561 its least significant byte side.
5563 Small aggregates and small complex types are always padded
5564 upwards.
5566 We don't need to worry about homogeneous floating-point or
5567 short-vector aggregates; their move is not affected by the
5568 padding direction determined here. Regardless of endianness,
5569 each element of such an aggregate is put in the least
5570 significant bits of a fp/simd register.
5572 Return !BYTES_BIG_ENDIAN if the least significant byte of the
5573 register has useful data, and return the opposite if the most
5574 significant byte does. */
5576 bool
5579 {
5581 /* Small composite types are always padded upward. */
5583 {
5584 HOST_WIDE_INT size;
5587 else
5588 /* No frontends can create types with variable-sized modes, so we
5589 shouldn't be asked to pass or return them. */
5593 }
5595 /* Otherwise, use the default padding. */
5597 }
5599 static scalar_int_mode
5601 {
5603 }
5605 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
5607 /* We use the 12-bit shifted immediate arithmetic instructions so values
5608 must be multiple of (1 << 12), i.e. 4096. */
5609 #define ARITH_FACTOR 4096
5611 #if (PROBE_INTERVAL % ARITH_FACTOR) != 0
5612 #error Cannot use simple address calculation for stack probing
5613 #endif
5615 /* The pair of scratch registers used for stack probing. */
5616 #define PROBE_STACK_FIRST_REG R9_REGNUM
5617 #define PROBE_STACK_SECOND_REG R10_REGNUM
5619 /* Emit code to probe a range of stack addresses from FIRST to FIRST+POLY_SIZE,
5620 inclusive. These are offsets from the current stack pointer. */
5622 static void
5624 {
5625 HOST_WIDE_INT size;
5627 {
5630 }
5634 /* See the same assertion on PROBE_INTERVAL above. */
5637 /* See if we have a constant small number of probes to generate. If so,
5638 that's the easy case. */
5640 {
5647 }
5649 /* The run-time loop is made up of 8 insns in the generic case while the
5650 compile-time loop is made up of 4+2*(n-2) insns for n # of intervals. */
5652 {
5657 stack_pointer_rtx,
5661 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
5662 it exceeds SIZE. If only two probes are needed, this will not
5663 generate any code. Then probe at FIRST + SIZE. */
5665 {
5669 }
5673 {
5678 }
5679 else
5681 }
5683 /* Otherwise, do the same as above, but in a loop. Note that we must be
5684 extra careful with variables wrapping around because we might be at
5685 the very top (or the very bottom) of the address space and we have
5686 to be able to handle this case properly; in particular, we use an
5687 equality test for the loop condition. */
5688 else
5689 {
5692 /* Step 1: round SIZE to the previous multiple of the interval. */
5697 /* Step 2: compute initial and final value of the loop counter. */
5699 /* TEST_ADDR = SP + FIRST. */
5703 /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE. */
5706 {
5710 }
5711 else
5715 /* Step 3: the loop
5717 do
5718 {
5719 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
5720 probe at TEST_ADDR
5721 }
5722 while (TEST_ADDR != LAST_ADDR)
5724 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
5725 until it is equal to ROUNDED_SIZE. */
5730 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
5731 that SIZE is equal to ROUNDED_SIZE. */
5734 {
5738 {
5743 }
5744 else
5746 }
5747 }
5749 /* Make sure nothing is scheduled before we are done. */
5751 }
5753 /* Probe a range of stack addresses from REG1 to REG2 inclusive. These are
5754 absolute addresses. */
5758 {
5765 /* Loop. */
5768 HOST_WIDE_INT stack_clash_probe_interval
5771 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
5773 HOST_WIDE_INT interval;
5776 else
5784 /* If doing stack clash protection then we probe up by the ABI specified
5785 amount. We do this because we're dropping full pages at a time in the
5786 loop. But if we're doing non-stack clash probing, probe at SP 0. */
5789 else
5792 /* Probe at TEST_ADDR. If we're inside the loop it is always safe to probe
5793 by this amount for each iteration. */
5796 /* Test if TEST_ADDR == LAST_ADDR. */
5800 /* Branch. */
5806 }
5808 /* Emit the probe loop for doing stack clash probes and stack adjustments for
5809 SVE. This emits probes from BASE to BASE - ADJUSTMENT based on a guard size
5810 of GUARD_SIZE. When a probe is emitted it is done at most
5811 MIN_PROBE_THRESHOLD bytes from the current BASE at an interval of
5812 at most MIN_PROBE_THRESHOLD. By the end of this function
5813 BASE = BASE - ADJUSTMENT. */
5818 {
5819 /* This function is not allowed to use any instruction generation function
5820 like gen_ and friends. If you do you'll likely ICE during CFG validation,
5821 so instead emit the code you want using output_asm_insn. */
5826 /* The minimum required allocation before the residual requires probing. */
5829 /* Clamp the value down to the nearest value that can be used with a cmp. */
5844 /* Emit loop start label. */
5847 /* ADJUSTMENT < RESIDUAL_PROBE_GUARD. */
5852 /* Branch to end if not enough adjustment to probe. */
5857 /* BASE = BASE - RESIDUAL_PROBE_GUARD. */
5862 /* Probe at BASE. */
5866 /* ADJUSTMENT = ADJUSTMENT - RESIDUAL_PROBE_GUARD. */
5871 /* Branch to start if still more bytes to allocate. */
5876 /* No probe leave. */
5879 /* BASE = BASE - ADJUSTMENT. */
5884 }
5886 /* Determine whether a frame chain needs to be generated. */
5887 static bool
5889 {
5890 /* Force a frame chain for EH returns so the return address is at FP+8. */
5894 /* A leaf function cannot have calls or write LR. */
5897 /* Don't use a frame chain in leaf functions if leaf frame pointers
5898 are disabled. */
5903 }
5905 /* Mark the registers that need to be saved by the callee and calculate
5906 the size of the callee-saved registers area and frame record (both FP
5907 and LR may be omitted). */
5908 static void
5910 {
5920 /* Adjust the outgoing arguments size if required. Keep it in sync with what
5921 the mid-end is doing. */
5924 #define SLOT_NOT_REQUIRED (-2)
5925 #define SLOT_REQUIRED (-1)
5931 /* First mark all the registers that really need to be saved... */
5935 /* ... that includes the eh data registers (if needed)... */
5940 /* ... and any callee saved register that dataflow says is live. */
5952 {
5957 }
5959 /* Big-endian SVE frames need a spare predicate register in order
5960 to save Z8-Z15. Decide which register they should use. Prefer
5961 an unused argument register if possible, so that we don't force P4
5962 to be saved unnecessarily. */
5966 {
5975 }
5983 /* With stack-clash, LR must be saved in non-leaf functions. */
5987 /* Now assign stack slots for the registers. Start with the predicate
5988 registers, since predicate LDR and STR have a relatively small
5989 offset range. These saves happen below the hard frame pointer. */
5992 {
5995 }
5997 /* We save a maximum of 8 predicate registers, and since vector
5998 registers are 8 times the size of a predicate register, all the
5999 saved predicates fit within a single vector. Doing this also
6000 rounds the offset to a 128-bit boundary. */
6002 {
6005 }
6007 /* If we need to save any SVE vector registers, add them next. */
6011 {
6014 }
6016 /* OFFSET is now the offset of the hard frame pointer from the bottom
6017 of the callee save area. */
6021 {
6022 /* FP and LR are placed in the linkage record. */
6028 }
6032 {
6039 }
6047 {
6048 /* If there is an alignment gap between integer and fp callee-saves,
6049 allocate the last fp register to it if possible. */
6051 && has_align_gap
6054 {
6057 }
6066 }
6074 poly_int64 above_outgoing_args
6079 frame.hard_fp_offset
6082 /* Both these values are already aligned. */
6102 HOST_WIDE_INT const_saved_regs_size;
6106 {
6107 /* Simple, small frame with no outgoing arguments:
6109 stp reg1, reg2, [sp, -frame_size]!
6110 stp reg3, reg4, [sp, 16] */
6112 }
6116 /* We could handle this case even with outgoing args, provided
6117 that the number of args left us with valid offsets for all
6118 predicate and vector save slots. It's such a rare case that
6119 it hardly seems worth the effort though. */
6124 {
6125 /* Frame with small outgoing arguments:
6127 sub sp, sp, frame_size
6128 stp reg1, reg2, [sp, outgoing_args_size]
6129 stp reg3, reg4, [sp, outgoing_args_size + 16] */
6132 }
6136 {
6137 /* Frame in which all saves are SVE saves:
6139 sub sp, sp, hard_fp_offset + below_hard_fp_saved_regs_size
6140 save SVE registers relative to SP
6141 sub sp, sp, outgoing_args_size */
6145 }
6148 {
6149 /* Frame with large outgoing arguments or SVE saves, but with
6150 a small local area:
6152 stp reg1, reg2, [sp, -hard_fp_offset]!
6153 stp reg3, reg4, [sp, 16]
6154 [sub sp, sp, below_hard_fp_saved_regs_size]
6155 [save SVE registers relative to SP]
6156 sub sp, sp, outgoing_args_size */
6160 }
6161 else
6162 {
6163 /* Frame with large local area and outgoing arguments or SVE saves,
6164 using frame pointer:
6166 sub sp, sp, hard_fp_offset
6167 stp x29, x30, [sp, 0]
6168 add x29, sp, 0
6169 stp reg3, reg4, [sp, 16]
6170 [sub sp, sp, below_hard_fp_saved_regs_size]
6171 [save SVE registers relative to SP]
6172 sub sp, sp, outgoing_args_size */
6176 }
6178 /* Make sure the individual adjustments add up to the full frame size. */
6185 }
6187 /* Return true if the register REGNO is saved on entry to
6188 the current function. */
6190 static bool
6192 {
6194 }
6196 /* Return the next register up from REGNO up to LIMIT for the callee
6197 to save. */
6199 static unsigned
6201 {
6203 regno ++;
6205 }
6207 /* Push the register number REGNO of mode MODE to the stack with write-back
6208 adjusting the stack by ADJUSTMENT. */
6210 static void
6212 HOST_WIDE_INT adjustment)
6213 {
6224 }
6226 /* Generate and return an instruction to store the pair of registers
6227 REG and REG2 of mode MODE to location BASE with write-back adjusting
6228 the stack location BASE by ADJUSTMENT. */
6230 static rtx
6232 HOST_WIDE_INT adjustment)
6233 {
6235 {
6250 }
6251 }
6253 /* Push registers numbered REGNO1 and REGNO2 to the stack, adjusting the
6254 stack pointer by ADJUSTMENT. */
6256 static void
6258 {
6273 }
6275 /* Load the pair of register REG, REG2 of mode MODE from stack location BASE,
6276 adjusting it by ADJUSTMENT afterwards. */
6278 static rtx
6280 HOST_WIDE_INT adjustment)
6281 {
6283 {
6295 }
6296 }
6298 /* Pop the two registers numbered REGNO1, REGNO2 from the stack, adjusting it
6299 afterwards by ADJUSTMENT and writing the appropriate REG_CFA_RESTORE notes
6300 into CFI_OPS. */
6302 static void
6305 {
6312 {
6316 }
6317 else
6318 {
6323 }
6324 }
6326 /* Generate and return a store pair instruction of mode MODE to store
6327 register REG1 to MEM1 and register REG2 to MEM2. */
6329 static rtx
6331 rtx reg2)
6332 {
6334 {
6346 }
6347 }
6349 /* Generate and regurn a load pair isntruction of mode MODE to load register
6350 REG1 from MEM1 and register REG2 from MEM2. */
6352 static rtx
6354 rtx mem2)
6355 {
6357 {
6369 }
6370 }
6372 /* Return TRUE if return address signing should be enabled for the current
6373 function, otherwise return FALSE. */
6375 bool
6377 {
6378 /* This function should only be called after frame laid out. */
6381 /* If signing scope is AARCH64_FUNCTION_NON_LEAF, we only sign a leaf function
6382 if its LR is pushed onto stack. */
6386 }
6388 /* Return TRUE if Branch Target Identification Mechanism is enabled. */
6389 bool
6391 {
6393 }
6395 /* The caller is going to use ST1D or LD1D to save or restore an SVE
6396 register in mode MODE at BASE_RTX + OFFSET, where OFFSET is in
6397 the range [1, 16] * GET_MODE_SIZE (MODE). Prepare for this by:
6399 (1) updating BASE_RTX + OFFSET so that it is a legitimate ST1D
6400 or LD1D address
6402 (2) setting PRED to a valid predicate register for the ST1D or LD1D,
6403 if the variable isn't already nonnull
6405 (1) is needed when OFFSET is in the range [8, 16] * GET_MODE_SIZE (MODE).
6406 Handle this case using a temporary base register that is suitable for
6407 all offsets in that range. Use ANCHOR_REG as this base register if it
6408 is nonnull, otherwise create a new register and store it in ANCHOR_REG. */
6414 {
6416 {
6417 /* This is the maximum valid offset of the anchor from the base.
6418 Lower values would be valid too. */
6421 {
6425 }
6428 }
6430 {
6435 }
6436 }
6438 /* Add a REG_CFA_EXPRESSION note to INSN to say that register REG
6439 is saved at BASE + OFFSET. */
6441 static void
6444 {
6448 }
6450 /* Emit code to save the callee-saved registers from register number START
6451 to LIMIT to the stack at the location starting at offset START_OFFSET,
6452 skipping any write-back candidates if SKIP_WB is true. HARD_FP_VALID_P
6453 is true if the hard frame pointer has been set up. */
6455 static void
6459 {
6468 {
6470 poly_int64 offset;
6487 HOST_WIDE_INT const_offset;
6493 {
6495 poly_int64 fp_offset
6499 else
6500 {
6502 {
6506 }
6508 }
6510 }
6520 {
6522 rtx mem2;
6527 reg2));
6529 /* The first part of a frame-related parallel insn is
6530 always assumed to be relevant to the frame
6531 calculations; subsequent parts, are only
6532 frame-related if explicitly marked. */
6534 {
6538 else
6540 }
6543 }
6545 {
6548 }
6551 else
6557 }
6558 }
6560 /* Emit code to restore the callee registers from register number START
6561 up to and including LIMIT. Restore from the stack offset START_OFFSET,
6562 skipping any write-back candidates if SKIP_WB is true. Write the
6563 appropriate REG_CFA_RESTORE notes into CFI_OPS. */
6565 static void
6568 {
6571 poly_int64 offset;
6577 {
6604 {
6606 rtx mem2;
6614 }
6619 else
6623 }
6624 }
6626 /* Return true if OFFSET is a signed 4-bit value multiplied by the size
6627 of MODE. */
6631 {
6632 HOST_WIDE_INT multiple;
6635 }
6637 /* Return true if OFFSET is a unsigned 6-bit value multiplied by the size
6638 of MODE. */
6642 {
6643 HOST_WIDE_INT multiple;
6646 }
6648 /* Return true if OFFSET is a signed 7-bit value multiplied by the size
6649 of MODE. */
6651 bool
6653 {
6654 HOST_WIDE_INT multiple;
6657 }
6659 /* Return true if OFFSET is a signed 9-bit value. */
6661 bool
6663 poly_int64 offset)
6664 {
6665 HOST_WIDE_INT const_offset;
6668 }
6670 /* Return true if OFFSET is a signed 9-bit value multiplied by the size
6671 of MODE. */
6675 {
6676 HOST_WIDE_INT multiple;
6679 }
6681 /* Return true if OFFSET is an unsigned 12-bit value multiplied by the size
6682 of MODE. */
6686 {
6687 HOST_WIDE_INT multiple;
6690 }
6692 /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
6694 static sbitmap
6696 {
6700 /* The registers we need saved to the frame. */
6703 {
6704 /* Punt on saves and restores that use ST1D and LD1D. We could
6705 try to be smarter, but it would involve making sure that the
6706 spare predicate register itself is safe to use at the save
6707 and restore points. Also, when a frame pointer is being used,
6708 the slots are often out of reach of ST1D and LD1D anyway. */
6715 /* If the register is saved in the first SVE save slot, we use
6716 it as a stack probe for -fstack-clash-protection. */
6722 /* Get the offset relative to the register we'll use. */
6725 else
6728 /* Check that we can access the stack slot of the register with one
6729 direct load with no adjustments needed. */
6734 }
6736 /* Don't mess with the hard frame pointer. */
6740 /* If the spare predicate register used by big-endian SVE code
6741 is call-preserved, it must be saved in the main prologue
6742 before any saves that use it. */
6748 /* If registers have been chosen to be stored/restored with
6749 writeback don't interfere with them to avoid having to output explicit
6750 stack adjustment instructions. */
6760 }
6762 /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
6764 static sbitmap
6766 {
6774 /* Clobbered registers don't generate values in any meaningful sense,
6775 since nothing after the clobber can rely on their value. And we can't
6776 say that partially-clobbered registers are unconditionally killed,
6777 because whether they're killed or not depends on the mode of the
6778 value they're holding. Thus partially call-clobbered registers
6779 appear in neither the kill set nor the gen set.
6781 Check manually for any calls that clobber more of a register than the
6782 current function can. */
6783 function_abi_aggregator callee_abis;
6790 /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
6798 {
6801 /* If there is a callee-save at an adjacent offset, add it too
6802 to increase the use of LDP/STP. */
6807 {
6813 }
6814 }
6817 }
6819 /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS.
6820 Nothing to do for aarch64. */
6822 static void
6824 {
6825 }
6827 /* Return the next set bit in BMP from START onwards. Return the total number
6828 of bits in BMP if no set bit is found at or after START. */
6830 static unsigned int
6832 {
6843 }
6845 /* Do the work for aarch64_emit_prologue_components and
6846 aarch64_emit_epilogue_components. COMPONENTS is the bitmap of registers
6847 to save/restore, PROLOGUE_P indicates whether to emit the prologue sequence
6848 for these components or the epilogue sequence. That is, it determines
6849 whether we should emit stores or loads and what kind of CFA notes to attach
6850 to the insns. Otherwise the logic for the two sequences is very
6851 similar. */
6853 static void
6855 {
6857 ? HARD_FRAME_POINTER_REGNUM
6865 {
6873 else
6881 /* No more registers to handle after REGNO.
6882 Emit a single save/restore and exit. */
6884 {
6887 {
6891 else
6893 }
6895 }
6898 /* The next register is not of the same class or its offset is not
6899 mergeable with the current one into a pair. */
6906 {
6909 {
6913 else
6915 }
6919 }
6923 /* REGNO2 can be saved/restored in a pair with REGNO. */
6927 else
6936 else
6940 {
6943 {
6948 }
6949 else
6950 {
6955 }
6956 }
6959 }
6960 }
6962 /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
6964 static void
6966 {
6968 }
6970 /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
6972 static void
6974 {
6976 }
6978 /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
6980 static void
6982 {
6986 }
6988 /* On AArch64 we have an ABI defined safe buffer. This constant is used to
6989 determining the probe offset for alloca. */
6991 static HOST_WIDE_INT
6993 {
6995 }
6998 /* Allocate POLY_SIZE bytes of stack space using TEMP1 and TEMP2 as scratch
6999 registers. If POLY_SIZE is not large enough to require a probe this function
7000 will only adjust the stack. When allocating the stack space
7001 FRAME_RELATED_P is then used to indicate if the allocation is frame related.
7002 FINAL_ADJUSTMENT_P indicates whether we are allocating the outgoing
7003 arguments. If we are then we ensure that any allocation larger than the ABI
7004 defined buffer needs a probe so that the invariant of having a 1KB buffer is
7005 maintained.
7007 We emit barriers after each stack adjustment to prevent optimizations from
7008 breaking the invariant that we never drop the stack more than a page. This
7009 invariant is needed to make it easier to correctly handle asynchronous
7010 events, e.g. if we were to allow the stack to be dropped by more than a page
7011 and then have multiple probes up and we take a signal somewhere in between
7012 then the signal handler doesn't know the state of the stack and can make no
7013 assumptions about which pages have been probed. */
7015 static void
7017 poly_int64 poly_size,
7020 {
7021 HOST_WIDE_INT guard_size
7024 HOST_WIDE_INT min_probe_threshold
7025 = (final_adjustment_p
7026 ? guard_used_by_caller
7028 /* When doing the final adjustment for the outgoing arguments, take into
7029 account any unprobed space there is above the current SP. There are
7030 two cases:
7032 - When saving SVE registers below the hard frame pointer, we force
7033 the lowest save to take place in the prologue before doing the final
7034 adjustment (i.e. we don't allow the save to be shrink-wrapped).
7035 This acts as a probe at SP, so there is no unprobed space.
7037 - When there are no SVE register saves, we use the store of the link
7038 register as a probe. We can't assume that LR was saved at position 0
7039 though, so treat any space below it as unprobed. */
7042 {
7046 else
7048 }
7052 /* We should always have a positive probe threshold. */
7056 {
7062 {
7064 }
7068 {
7070 }
7071 }
7073 /* If SIZE is not large enough to require probing, just adjust the stack and
7074 exit. */
7077 {
7080 }
7082 HOST_WIDE_INT size;
7083 /* Handle the SVE non-constant case first. */
7085 {
7087 {
7091 }
7093 /* First calculate the amount of bytes we're actually spilling. */
7100 {
7101 /* This is done to provide unwinding information for the stack
7102 adjustments we're about to do, however to prevent the optimizers
7103 from removing the R11 move and leaving the CFA note (which would be
7104 very wrong) we tie the old and new stack pointer together.
7105 The tie will expand to nothing but the optimizers will not touch
7106 the instruction. */
7111 /* We want the CFA independent of the stack pointer for the
7112 duration of the loop. */
7115 }
7124 /* Now reset the CFA register if needed. */
7126 {
7131 }
7134 }
7138 "Stack clash AArch64 prologue: " HOST_WIDE_INT_PRINT_DEC
7141 /* Round size to the nearest multiple of guard_size, and calculate the
7142 residual as the difference between the original size and the rounded
7143 size. */
7147 /* We can handle a small number of allocations/probes inline. Otherwise
7148 punt to a loop. */
7150 {
7152 {
7155 guard_used_by_caller));
7157 }
7159 }
7160 else
7161 {
7162 /* Compute the ending address. */
7167 /* For the initial allocation, we don't have a frame pointer
7168 set up, so we always need CFI notes. If we're doing the
7169 final allocation, then we may have a frame pointer, in which
7170 case it is the CFA, otherwise we need CFI notes.
7172 We can determine which allocation we are doing by looking at
7173 the value of FRAME_RELATED_P since the final allocations are not
7174 frame related. */
7176 {
7177 /* We want the CFA independent of the stack pointer for the
7178 duration of the loop. */
7182 }
7184 /* This allocates and probes the stack. Note that this re-uses some of
7185 the existing Ada stack protection code. However we are guaranteed not
7186 to enter the non loop or residual branches of that code.
7188 The non-loop part won't be entered because if our allocation amount
7189 doesn't require a loop, the case above would handle it.
7191 The residual amount won't be entered because TEMP1 is a mutliple of
7192 the allocation size. The residual will always be 0. As such, the only
7193 part we are actually using from that code is the loop setup. The
7194 actual probing is done in aarch64_output_probe_stack_range. */
7198 /* Now reset the CFA register if needed. */
7200 {
7204 }
7208 }
7210 /* Handle any residuals. Residuals of at least MIN_PROBE_THRESHOLD have to
7211 be probed. This maintains the requirement that each page is probed at
7212 least once. For initial probing we probe only if the allocation is
7213 more than GUARD_SIZE - buffer, and for the outgoing arguments we probe
7214 if the amount is larger than buffer. GUARD_SIZE - buffer + buffer ==
7215 GUARD_SIZE. This works that for any allocation that is large enough to
7216 trigger a probe here, we'll have at least one, and if they're not large
7217 enough for this code to emit anything for them, The page would have been
7218 probed by the saving of FP/LR either by this function or any callees. If
7219 we don't have any callees then we won't have more stack adjustments and so
7220 are still safe. */
7222 {
7224 /* If we're doing final adjustments, and we've done any full page
7225 allocations then any residual needs to be probed. */
7228 /* If doing a small final adjustment, we always probe at offset 0.
7229 This is done to avoid issues when LR is not at position 0 or when
7230 the final adjustment is smaller than the probing offset. */
7236 {
7239 "Stack clash AArch64 prologue residuals: "
7240 HOST_WIDE_INT_PRINT_DEC " bytes, probing will be required."
7244 residual_probe_offset));
7246 }
7247 }
7248 }
7250 /* Return 1 if the register is used by the epilogue. We need to say the
7251 return register is used, but only after epilogue generation is complete.
7252 Note that in the case of sibcalls, the values "used by the epilogue" are
7253 considered live at the start of the called function.
7255 For SIMD functions we need to return 1 for FP registers that are saved and
7256 restored by a function but are not zero in call_used_regs. If we do not do
7257 this optimizations may remove the restore of the register. */
7259 int
7261 {
7263 {
7266 }
7268 }
7270 /* AArch64 stack frames generated by this compiler look like:
7272 +-------------------------------+
7273 | |
7274 | incoming stack arguments |
7275 | |
7276 +-------------------------------+
7277 | | <-- incoming stack pointer (aligned)
7278 | callee-allocated save area |
7279 | for register varargs |
7280 | |
7281 +-------------------------------+
7282 | local variables | <-- frame_pointer_rtx
7283 | |
7284 +-------------------------------+
7285 | padding | \
7286 +-------------------------------+ |
7287 | callee-saved registers | | frame.saved_regs_size
7288 +-------------------------------+ |
7289 | LR' | |
7290 +-------------------------------+ |
7291 | FP' | |
7292 +-------------------------------+ |<- hard_frame_pointer_rtx (aligned)
7293 | SVE vector registers | | \
7294 +-------------------------------+ | | below_hard_fp_saved_regs_size
7295 | SVE predicate registers | / /
7296 +-------------------------------+
7297 | dynamic allocation |
7298 +-------------------------------+
7299 | padding |
7300 +-------------------------------+
7301 | outgoing stack arguments | <-- arg_pointer
7302 | |
7303 +-------------------------------+
7304 | | <-- stack_pointer_rtx (aligned)
7306 Dynamic stack allocations via alloca() decrease stack_pointer_rtx
7307 but leave frame_pointer_rtx and hard_frame_pointer_rtx
7308 unchanged.
7310 By default for stack-clash we assume the guard is at least 64KB, but this
7311 value is configurable to either 4KB or 64KB. We also force the guard size to
7312 be the same as the probing interval and both values are kept in sync.
7314 With those assumptions the callee can allocate up to 63KB (or 3KB depending
7315 on the guard size) of stack space without probing.
7317 When probing is needed, we emit a probe at the start of the prologue
7318 and every PARAM_STACK_CLASH_PROTECTION_GUARD_SIZE bytes thereafter.
7320 We have to track how much space has been allocated and the only stores
7321 to the stack we track as implicit probes are the FP/LR stores.
7323 For outgoing arguments we probe if the size is larger than 1KB, such that
7324 the ABI specified buffer is maintained for the next callee.
7326 The following registers are reserved during frame layout and should not be
7327 used for any other purpose:
7329 - r11: Used by stack clash protection when SVE is enabled, and also
7330 as an anchor register when saving and restoring registers
7331 - r12(EP0) and r13(EP1): Used as temporaries for stack adjustment.
7332 - r14 and r15: Used for speculation tracking.
7333 - r16(IP0), r17(IP1): Used by indirect tailcalls.
7334 - r30(LR), r29(FP): Used by standard frame layout.
7336 These registers must be avoided in frame layout related code unless the
7337 explicit intention is to interact with one of the features listed above. */
7339 /* Generate the prologue instructions for entry into a function.
7340 Establish the stack frame by decreasing the stack pointer with a
7341 properly calculated size and, if necessary, create a frame record
7342 filled with the values of LR and previous frame pointer. The
7343 current FP is also set up if it is in use. */
7345 void
7347 {
7354 poly_int64 below_hard_fp_saved_regs_size
7362 {
7363 /* Fold the SVE allocation into the initial allocation.
7364 We don't do this in aarch64_layout_arg to avoid pessimizing
7365 the epilogue code. */
7368 }
7370 /* Sign return address for functions. */
7372 {
7374 {
7383 }
7386 }
7392 {
7394 {
7398 (frame_size
7400 }
7403 }
7408 /* In theory we should never have both an initial adjustment
7409 and a callee save adjustment. Verify that is the case since the
7410 code below does not handle it for -fstack-clash-protection. */
7413 /* Will only probe if the initial adjustment is larger than the guard
7414 less the amount of the guard reserved for use by the caller's
7415 outgoing args. */
7422 /* The offset of the frame chain record (if any) from the current SP. */
7427 /* The offset of the bottom of the save area from the current SP. */
7431 {
7433 {
7438 }
7439 else
7445 {
7446 /* Variable-sized frames need to describe the save slot
7447 address using DW_CFA_expression rather than DW_CFA_offset.
7448 This means that, without taking further action, the
7449 locations of the registers that we've already saved would
7450 remain based on the stack pointer even after we redefine
7451 the CFA based on the frame pointer. We therefore need new
7452 DW_CFA_expressions to re-express the save slots with addresses
7453 based on the frame pointer. */
7457 /* Add an explicit CFA definition if this was previously
7458 implicit. */
7460 {
7462 callee_offset);
7465 }
7467 /* Change the save slot expressions for the registers that
7468 we've already saved. */
7473 }
7475 }
7479 emit_frame_chain);
7481 {
7485 sve_callee_adjust,
7488 }
7493 emit_frame_chain);
7495 /* We may need to probe the final adjustment if it is larger than the guard
7496 that is assumed by the called. */
7499 }
7501 /* Return TRUE if we can use a simple_return insn.
7503 This function checks whether the callee saved stack is empty, which
7504 means no restore actions are need. The pro_and_epilogue will use
7505 this to check whether shrink-wrapping opt is feasible. */
7507 bool
7509 {
7517 }
7519 /* Generate the epilogue instructions for returning from a function.
7520 This is almost exactly the reverse of the prolog sequence, except
7521 that we need to insert barriers to avoid scheduling loads that read
7522 from a deallocated stack, and we optimize the unwind records by
7523 emitting them all together if possible. */
7524 void
7526 {
7532 poly_int64 below_hard_fp_saved_regs_size
7538 /* A stack clash protection prologue may not have left EP0_REGNUM or
7539 EP1_REGNUM in a usable state. The same is true for allocations
7540 with an SVE component, since we then need both temporary registers
7541 for each allocation. For stack clash we are in a usable state if
7542 the adjustment is less than GUARD_SIZE - GUARD_USED_BY_CALLER. */
7543 HOST_WIDE_INT guard_size
7547 /* We can re-use the registers when:
7549 (a) the deallocation amount is the same as the corresponding
7550 allocation amount (which is false if we combine the initial
7551 and SVE callee save allocations in the prologue); and
7553 (b) the allocation amount doesn't need a probe (which is false
7554 if the amount is guard_size - guard_used_by_caller or greater).
7556 In such situations the register should remain live with the correct
7557 value. */
7560 && (!flag_stack_clash_protection
7565 /* We need to add memory barrier to prevent read from deallocated stack. */
7566 bool need_barrier_p
7570 /* Emit a barrier to prevent loads from a deallocated stack. */
7574 {
7577 }
7579 /* Restore the stack pointer from the frame pointer if it may not
7580 be the same as the stack pointer. */
7585 /* If writeback is used when restoring callee-saves, the CFA
7586 is restored on the instruction doing the writeback. */
7588 hard_frame_pointer_rtx,
7591 else
7592 /* The case where we need to re-use the register here is very rare, so
7593 avoid the complicated condition and just always emit a move if the
7594 immediate doesn't fit. */
7597 /* Restore the vector registers before the predicate registers,
7598 so that we can use P4 as a temporary for big-endian SVE frames. */
7616 {
7617 /* Emit delayed restores and set the CFA to be SP + initial_adjust. */
7623 }
7625 /* Liveness of EP0_REGNUM can not be trusted across function calls either, so
7626 add restriction on emit_move optimization to leaf functions. */
7632 {
7633 /* Emit delayed restores and reset the CFA to be SP. */
7638 }
7640 /* We prefer to emit the combined return/authenticate instruction RETAA,
7641 however there are three cases in which we must instead emit an explicit
7642 authentication instruction.
7644 1) Sibcalls don't return in a normal way, so if we're about to call one
7645 we must authenticate.
7647 2) The RETAA instruction is not available before ARMv8.3-A, so if we are
7648 generating code for !TARGET_ARMV8_3 we can't use it and must
7649 explicitly authenticate.
7651 3) On an eh_return path we make extra stack adjustments to update the
7652 canonical frame address to be the exception handler's CFA. We want
7653 to authenticate using the CFA of the function which calls eh_return.
7654 */
7657 {
7659 {
7668 }
7671 }
7673 /* Stack adjustment for exception handler. */
7675 {
7676 /* We need to unwind the stack by the offset computed by
7677 EH_RETURN_STACKADJ_RTX. We have already reset the CFA
7678 to be SP; letting the CFA move during this adjustment
7679 is just as correct as retaining the CFA from the body
7680 of the function. Therefore, do nothing special. */
7682 }
7687 }
7689 /* Implement EH_RETURN_HANDLER_RTX. EH returns need to either return
7690 normally or return to a previous frame after unwinding.
7692 An EH return uses a single shared return sequence. The epilogue is
7693 exactly like a normal epilogue except that it has an extra input
7694 register (EH_RETURN_STACKADJ_RTX) which contains the stack adjustment
7695 that must be applied after the frame has been destroyed. An extra label
7696 is inserted before the epilogue which initializes this register to zero,
7697 and this is the entry point for a normal return.
7699 An actual EH return updates the return address, initializes the stack
7700 adjustment and jumps directly into the epilogue (bypassing the zeroing
7701 of the adjustment). Since the return address is typically saved on the
7702 stack when a function makes a call, the saved LR must be updated outside
7703 the epilogue.
7705 This poses problems as the store is generated well before the epilogue,
7706 so the offset of LR is not known yet. Also optimizations will remove the
7707 store as it appears dead, even after the epilogue is generated (as the
7708 base or offset for loading LR is different in many cases).
7710 To avoid these problems this implementation forces the frame pointer
7711 in eh_return functions so that the location of LR is fixed and known early.
7712 It also marks the store volatile, so no optimization is permitted to
7713 remove the store. */
7714 rtx
7716 {
7720 /* Mark the store volatile, so no optimization is permitted to remove it. */
7723 }
7725 /* Output code to add DELTA to the first argument, and then jump
7726 to FUNCTION. Used for C++ multiple inheritance. */
7727 static void
7729 HOST_WIDE_INT delta,
7730 HOST_WIDE_INT vcall_offset,
7731 tree function)
7732 {
7733 /* The this pointer is always in x0. Note that this differs from
7734 Arm where the this pointer maybe bumped to r1 if r0 is required
7735 to return a pointer to an aggregate. On AArch64 a result value
7736 pointer will be in x8. */
7754 else
7755 {
7760 {
7764 else
7767 }
7771 else
7778 else
7779 {
7781 Pmode);
7783 }
7787 else
7793 }
7795 /* Generate a tail call to the target function. */
7797 {
7800 }
7816 /* Stop pretending to be a post-reload pass. */
7818 }
7820 static bool
7822 {
7827 {
7831 /* Don't recurse into UNSPEC_TLS looking for TLS symbols; these are
7832 TLS offsets, not real symbol references. */
7835 }
7837 }
7840 /* Return true if val can be encoded as a 12-bit unsigned immediate with
7841 a left shift of 0 or 12 bits. */
7842 bool
7844 {
7847 );
7848 }
7850 /* Returns the nearest value to VAL that will fit as a 12-bit unsigned immediate
7851 that can be created with a left shift of 0 or 12. */
7852 static HOST_WIDE_INT
7854 {
7855 /* Check to see if the value fits in 24 bits, as that is the maximum we can
7856 handle correctly. */
7863 }
7865 /* Return true if val is an immediate that can be loaded into a
7866 register by a MOVZ instruction. */
7867 static bool
7869 {
7871 {
7875 }
7876 else
7877 {
7878 /* Ignore sign extension. */
7880 }
7883 }
7885 /* VAL is a value with the inner mode of MODE. Replicate it to fill a
7886 64-bit (DImode) integer. */
7888 static unsigned HOST_WIDE_INT
7890 {
7893 {
7897 }
7899 }
7901 /* Multipliers for repeating bitmasks of width 32, 16, 8, 4, and 2. */
7904 {
7910 };
7913 /* Return true if val is a valid bitmask immediate. */
7915 bool
7917 {
7921 /* Check for a single sequence of one bits and return quickly if so.
7922 The special cases of all ones and all zeroes returns false. */
7929 /* Replicate 32-bit immediates so we can treat them as 64-bit. */
7933 /* Invert if the immediate doesn't start with a zero bit - this means we
7934 only need to search for sequences of one bits. */
7938 /* Find the first set bit and set tmp to val with the first sequence of one
7939 bits removed. Return success if there is a single sequence of ones. */
7946 /* Find the next set bit and compute the difference in bit position. */
7951 /* Check the bit position difference is a power of 2, and that the first
7952 sequence of one bits fits within 'bits' bits. */
7956 /* Check the sequence of one bits is repeated 64/bits times. */
7958 }
7960 /* Create mask of ones, covering the lowest to highest bits set in VAL_IN.
7961 Assumed precondition: VAL_IN Is not zero. */
7963 unsigned HOST_WIDE_INT
7965 {
7972 }
7974 /* Create constant where bits outside of lowest bit set to highest bit set
7975 are set to 1. */
7977 unsigned HOST_WIDE_INT
7979 {
7981 }
7983 /* Return true if VAL_IN is a valid 'and' bitmask immediate. */
7985 bool
7987 {
7988 scalar_int_mode int_mode;
8001 }
8003 /* Return true if val is an immediate that can be loaded into a
8004 register in a single instruction. */
8005 bool
8007 {
8008 scalar_int_mode int_mode;
8015 }
8017 static bool
8019 {
8025 /* There's no way to calculate VL-based values using relocations. */
8033 {
8037 else
8038 /* Avoid generating a 64-bit relocation in ILP32; leave
8039 to aarch64_expand_mov_immediate to handle it properly. */
8041 }
8044 }
8046 /* Implement TARGET_CASE_VALUES_THRESHOLD.
8047 The expansion for a table switch is quite expensive due to the number
8048 of instructions, the table lookup and hard to predict indirect jump.
8049 When optimizing for speed, and -O3 enabled, use the per-core tuning if
8050 set, otherwise use tables for > 16 cases as a tradeoff between size and
8051 performance. When optimizing for size, use the default setting. */
8053 static unsigned int
8055 {
8056 /* Use the specified limit for the number of cases before using jump
8057 tables at higher optimization levels. */
8061 else
8063 }
8065 /* Return true if register REGNO is a valid index register.
8066 STRICT_P is true if REG_OK_STRICT is in effect. */
8068 bool
8070 {
8072 {
8080 }
8082 }
8084 /* Return true if register REGNO is a valid base register for mode MODE.
8085 STRICT_P is true if REG_OK_STRICT is in effect. */
8087 bool
8089 {
8091 {
8099 }
8101 /* The fake registers will be eliminated to either the stack or
8102 hard frame pointer, both of which are usually valid base registers.
8103 Reload deals with the cases where the eliminated form isn't valid. */
8108 }
8110 /* Return true if X is a valid base register for mode MODE.
8111 STRICT_P is true if REG_OK_STRICT is in effect. */
8113 static bool
8115 {
8122 }
8124 /* Return true if address offset is a valid index. If it is, fill in INFO
8125 appropriately. STRICT_P is true if REG_OK_STRICT is in effect. */
8127 static bool
8130 {
8132 rtx index;
8135 /* (reg:P) */
8138 {
8142 }
8143 /* (sign_extend:DI (reg:SI)) */
8148 {
8153 }
8154 /* (mult:DI (sign_extend:DI (reg:SI)) (const_int scale)) */
8161 {
8166 }
8167 /* (ashift:DI (sign_extend:DI (reg:SI)) (const_int shift)) */
8174 {
8179 }
8180 /* (sign_extract:DI (mult:DI (reg:DI) (const_int scale)) 32+shift 0) */
8187 {
8195 }
8196 /* (and:DI (mult:DI (reg:DI) (const_int scale))
8197 (const_int 0xffffffff<<shift)) */
8204 {
8210 }
8211 /* (sign_extract:DI (ashift:DI (reg:DI) (const_int shift)) 32+shift 0) */
8218 {
8226 }
8227 /* (and:DI (ashift:DI (reg:DI) (const_int shift))
8228 (const_int 0xffffffff<<shift)) */
8235 {
8241 }
8242 /* (mult:P (reg:P) (const_int scale)) */
8247 {
8251 }
8252 /* (ashift:P (reg:P) (const_int shift)) */
8257 {
8261 }
8262 else
8271 {
8275 }
8276 else
8277 {
8282 }
8286 {
8291 }
8294 }
8296 /* Return true if MODE is one of the modes for which we
8297 support LDP/STP operations. */
8299 static bool
8301 {
8309 }
8311 /* Return true if REGNO is a virtual pointer register, or an eliminable
8312 "soft" frame register. Like REGNO_PTR_FRAME_P except that we don't
8313 include stack_pointer or hard_frame_pointer. */
8314 static bool
8316 {
8321 }
8323 /* Return true if X is a valid address of type TYPE for machine mode MODE.
8324 If it is, fill in INFO appropriately. STRICT_P is true if
8325 REG_OK_STRICT is in effect. */
8327 bool
8330 aarch64_addr_query_type type)
8331 {
8334 poly_int64 offset;
8336 HOST_WIDE_INT const_size;
8338 /* Whether a vector mode is partial doesn't affect address legitimacy.
8339 Partial vectors like VNx8QImode allow the same indexed addressing
8340 mode and MUL VL addressing mode as full vectors like VNx16QImode;
8341 in both cases, MUL VL counts multiples of GET_MODE_SIZE. */
8345 /* On BE, we use load/store pair for all large int mode load/stores.
8346 TI/TFmode may also use a load/store pair. */
8354 /* If we are dealing with ADDR_QUERY_LDP_STP_N that means the incoming mode
8355 corresponds to the actual size of the memory being loaded/stored and the
8356 mode of the corresponding addressing mode is half of that. */
8366 /* For SVE, only accept [Rn], [Rn, Rm, LSL #shift] and
8367 [Rn, #offset, MUL VL]. */
8372 /* On LE, for AdvSIMD, don't support anything other than POST_INC or
8373 REG addressing. */
8375 && !BYTES_BIG_ENDIAN
8383 {
8400 {
8407 }
8412 {
8418 /* TImode and TFmode values are allowed in both pairs of X
8419 registers and individual Q registers. The available
8420 address modes are:
8421 X,X: 7-bit signed scaled offset
8422 Q: 9-bit signed offset
8423 We conservatively require an offset representable in either mode.
8424 When performing the check for pairs of X registers i.e. LDP/STP
8425 pass down DImode since that is the natural size of the LDP/STP
8426 instruction memory accesses. */
8432 /* A 7bit offset check because OImode will emit a ldp/stp
8433 instruction (only big endian will get here).
8434 For ldp/stp instructions, the offset is scaled for the size of a
8435 single element of the pair. */
8439 /* Three 9/12 bit offsets checks because CImode will emit three
8440 ldr/str instructions (only big endian will get here). */
8448 /* Two 7bit offsets checks because XImode will emit two ldp/stp
8449 instructions (only big endian will get here). */
8455 /* Make "m" use the LD1 offset range for SVE data modes, so
8456 that pre-RTL optimizers like ivopts will work to that
8457 instead of the wider LDR/STR range. */
8464 {
8465 poly_int64 end_offset = (offset
8472 end_offset)));
8473 }
8483 else
8486 }
8489 {
8490 /* Look for base + (scaled/extended) index register. */
8493 {
8496 }
8499 {
8502 }
8503 }
8524 {
8528 /* TImode and TFmode values are allowed in both pairs of X
8529 registers and individual Q registers. The available
8530 address modes are:
8531 X,X: 7-bit signed scaled offset
8532 Q: 9-bit signed offset
8533 We conservatively require an offset representable in either mode.
8534 */
8544 else
8546 }
8552 /* load literal: pc-relative constant pool entry. Only supported
8553 for SI mode or larger. */
8559 {
8567 }
8576 {
8582 {
8583 /* The symbol and offset must be aligned to the access size. */
8589 {
8593 }
8599 else
8608 }
8609 }
8614 }
8615 }
8617 /* Return true if the address X is valid for a PRFM instruction.
8618 STRICT_P is true if we should do strict checking with
8619 aarch64_classify_address. */
8621 bool
8623 {
8626 /* PRFM accepts the same addresses as DImode... */
8631 /* ... except writeback forms. */
8633 }
8635 bool
8637 {
8638 rtx offset;
8642 }
8644 /* Classify the base of symbolic expression X. */
8646 enum aarch64_symbol_type
8648 {
8649 rtx offset;
8653 }
8656 /* Return TRUE if X is a legitimate address for accessing memory in
8657 mode MODE. */
8658 static bool
8660 {
8664 }
8666 /* Return TRUE if X is a legitimate address of type TYPE for accessing
8667 memory in mode MODE. STRICT_P is true if REG_OK_STRICT is in effect. */
8668 bool
8670 aarch64_addr_query_type type)
8671 {
8675 }
8677 /* Implement TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT. */
8679 static bool
8681 poly_int64 orig_offset,
8682 machine_mode mode)
8683 {
8684 HOST_WIDE_INT size;
8686 {
8689 /* A general SVE offset is A * VQ + B. Remove the A component from
8690 coefficient 0 in order to get the constant B. */
8693 /* Split an out-of-range address displacement into a base and
8694 offset. Use 4KB range for 1- and 2-byte accesses and a 16KB
8695 range otherwise to increase opportunities for sharing the base
8696 address of different sizes. Unaligned accesses use the signed
8697 9-bit range, TImode/TFmode use the intersection of signed
8698 scaled 7-bit and signed 9-bit offset. */
8703 else
8709 /* Split the offset into second_offset and the rest. */
8713 }
8714 else
8715 {
8716 /* Get the mode we should use as the basis of the range. For structure
8717 modes this is the mode of one vector. */
8719 machine_mode step_mode
8722 /* Get the "mul vl" multiplier we'd like to use. */
8726 /* LDR supports a 9-bit range, but the move patterns for
8727 structure modes require all vectors to be in range of the
8728 same base. The simplest way of accomodating that while still
8729 promoting reuse of anchor points between different modes is
8730 to use an 8-bit range unconditionally. */
8732 else
8733 /* Predicates are only handled singly, so we might as well use
8734 the full range. */
8739 /* Convert the "mul vl" multiplier into a byte offset. */
8744 /* Split the offset into second_offset and the rest. */
8748 }
8749 }
8751 /* Return the binary representation of floating point constant VALUE in INTVAL.
8752 If the value cannot be converted, return false without setting INTVAL.
8753 The conversion is done in the given MODE. */
8754 bool
8756 {
8758 /* We make a general exception for 0. */
8760 {
8763 }
8765 scalar_float_mode mode;
8769 /* Only support up to DF mode. */
8781 {
8785 }
8786 else
8791 }
8793 /* Return TRUE if rtx X is an immediate constant that can be moved using a
8794 single MOV(+MOVK) followed by an FMOV. */
8795 bool
8797 {
8802 /* Determine whether it's cheaper to write float constants as
8803 mov/movk pairs over ldr/adrp pairs. */
8809 {
8811 ? SImode
8816 }
8819 }
8821 /* Return TRUE if rtx X is immediate constant 0.0 */
8822 bool
8824 {
8831 }
8833 /* Return TRUE if rtx X is immediate constant that fits in a single
8834 MOVI immediate operation. */
8835 bool
8837 {
8841 machine_mode vmode;
8842 scalar_int_mode imode;
8847 {
8851 /* We make a general exception for 0. */
8856 }
8860 else
8863 /* use a 64 bit mode for everything except for DI/DF mode, where we use
8864 a 128 bit vector mode. */
8871 }
8874 /* Return the fixed registers used for condition codes. */
8876 static bool
8878 {
8882 }
8884 /* This function is used by the call expanders of the machine description.
8885 RESULT is the register in which the result is returned. It's NULL for
8886 "call" and "sibcall".
8887 MEM is the location of the function call.
8888 CALLEE_ABI is a const_int that gives the arm_pcs of the callee.
8889 SIBCALL indicates whether this function call is normal call or sibling call.
8890 It will generate different pattern accordingly. */
8892 void
8894 {
8896 rtvec vec;
8897 machine_mode mode;
8904 /* Decide if we should generate indirect calls by loading the
8905 address of the callee into a register before performing
8906 the branch-and-link. */
8920 else
8925 UNSPEC_CALLEE_ABI);
8931 }
8933 /* Emit call insn with PAT and do aarch64-specific handling. */
8935 void
8937 {
8943 }
8945 machine_mode
8947 {
8951 /* All floating point compares return CCFP if it is an equality
8952 comparison, and CCFPE otherwise. */
8954 {
8956 {
8977 }
8978 }
8980 /* Equality comparisons of short modes against zero can be performed
8981 using the TST instruction with the appropriate bitmask. */
8987 /* Similarly, comparisons of zero_extends from shorter modes can
8988 be performed using an ANDS with an immediate mask. */
9004 /* A compare with a shifted operand. Because of canonicalization,
9005 the comparison will have to be swapped when we emit the assembly
9006 code. */
9014 /* Similarly for a negated operand, but we can only do this for
9015 equalities. */
9022 /* A test for unsigned overflow from an addition. */
9029 /* A test for unsigned overflow from an add with carry. */
9039 /* A test for signed overflow. */
9046 /* For everything else, return CCmode. */
9048 }
9050 static int
9053 int
9055 {
9062 }
9064 static int
9066 {
9068 {
9072 {
9086 }
9091 {
9103 }
9108 {
9120 }
9125 {
9135 }
9140 {
9146 }
9151 {
9155 }
9160 {
9164 }
9169 {
9173 }
9178 {
9182 }
9187 }
9190 }
9192 bool
9194 HOST_WIDE_INT minval,
9195 HOST_WIDE_INT maxval)
9196 {
9197 rtx elt;
9201 }
9203 bool
9205 {
9207 }
9209 /* Return true if VEC is a constant in which every element is in the range
9210 [MINVAL, MAXVAL]. The elements do not need to have the same value. */
9212 static bool
9214 HOST_WIDE_INT minval,
9215 HOST_WIDE_INT maxval)
9216 {
9228 {
9233 }
9235 }
9237 /* N Z C V. */
9238 #define AARCH64_CC_V 1
9239 #define AARCH64_CC_C (1 << 1)
9240 #define AARCH64_CC_Z (1 << 2)
9241 #define AARCH64_CC_N (1 << 3)
9243 /* N Z C V flags for ccmp. Indexed by AARCH64_COND_CODE. */
9245 {
9262 };
9264 /* Print floating-point vector immediate operand X to F, negating it
9265 first if NEGATE is true. Return true on success, false if it isn't
9266 a constant we can handle. */
9268 static bool
9270 {
9271 rtx elt;
9280 /* Handle the SVE single-bit immediates specially, since they have a
9281 fixed form in the assembly syntax. */
9290 else
9291 {
9297 }
9300 }
9302 /* Return the equivalent letter for size. */
9303 static char
9305 {
9307 {
9313 }
9314 }
9316 /* Print operand X to file F in a target specific manner according to CODE.
9317 The acceptable formatting commands given by CODE are:
9318 'c': An integer or symbol address without a preceding #
9319 sign.
9320 'C': Take the duplicated element in a vector constant
9321 and print it in hex.
9322 'D': Take the duplicated element in a vector constant
9323 and print it as an unsigned integer, in decimal.
9324 'e': Print the sign/zero-extend size as a character 8->b,
9325 16->h, 32->w. Can also be used for masks:
9326 0xff->b, 0xffff->h, 0xffffffff->w.
9327 'I': If the operand is a duplicated vector constant,
9328 replace it with the duplicated scalar. If the
9329 operand is then a floating-point constant, replace
9330 it with the integer bit representation. Print the
9331 transformed constant as a signed decimal number.
9332 'p': Prints N such that 2^N == X (X must be power of 2 and
9333 const int).
9334 'P': Print the number of non-zero bits in X (a const_int).
9335 'H': Print the higher numbered register of a pair (TImode)
9336 of regs.
9337 'm': Print a condition (eq, ne, etc).
9338 'M': Same as 'm', but invert condition.
9339 'N': Take the duplicated element in a vector constant
9340 and print the negative of it in decimal.
9341 'b/h/s/d/q': Print a scalar FP/SIMD register name.
9342 'S/T/U/V': Print a FP/SIMD register name for a register list.
9343 The register printed is the FP/SIMD register name
9344 of X + 0/1/2/3 for S/T/U/V.
9345 'R': Print a scalar Integer/FP/SIMD register name + 1.
9346 'X': Print bottom 16 bits of integer constant in hex.
9347 'w/x': Print a general register name or the zero register
9348 (32-bit or 64-bit).
9349 '0': Print a normal operand, if it's a general register,
9350 then we assume DImode.
9351 'k': Print NZCV for conditional compare instructions.
9352 'A': Output address constant representing the first
9353 argument of X, specifying a relocation offset
9354 if appropriate.
9355 'L': Output constant address specified by X
9356 with a relocation offset if appropriate.
9357 'G': Prints address of X, specifying a PC relative
9358 relocation mode if appropriate.
9359 'y': Output address of LDP or STP - this is used for
9360 some LDP/STPs which don't use a PARALLEL in their
9361 pattern (so the mode needs to be adjusted).
9362 'z': Output address of a typical LDP or STP. */
9364 static void
9366 {
9367 rtx elt;
9369 {
9372 {
9384 {
9387 }
9388 /* Fall through. */
9392 }
9396 {
9399 {
9402 }
9411 else
9412 {
9415 }
9416 }
9420 {
9424 {
9427 }
9430 }
9435 {
9438 }
9445 {
9448 }
9451 {
9454 }
9460 {
9464 else
9465 {
9468 }
9470 }
9474 {
9476 /* CONST_TRUE_RTX means al/nv (al is the default, don't print it). */
9478 {
9482 }
9485 {
9488 }
9496 else
9498 }
9503 {
9506 }
9512 ;
9513 else
9514 {
9517 }
9526 {
9527 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
9529 }
9538 {
9539 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
9541 }
9552 else
9554 code);
9559 {
9562 }
9567 {
9568 /* Print a replicated constant in hex. */
9570 {
9573 }
9576 }
9580 {
9581 /* Print a replicated constant in decimal, treating it as
9582 unsigned. */
9584 {
9587 }
9590 }
9597 {
9600 }
9603 {
9606 }
9609 {
9612 }
9614 /* Fall through */
9618 {
9621 }
9624 {
9627 {
9630 else
9631 {
9632 char suffix
9637 }
9638 }
9639 else
9658 {
9661 }
9662 /* fall through */
9666 {
9669 }
9675 ;
9676 else
9677 {
9680 }
9684 /* Since we define TARGET_SUPPORTS_WIDE_INT we shouldn't ever
9685 be getting CONST_DOUBLEs holding integers. */
9688 {
9691 }
9693 {
9694 #define buf_size 20
9702 #undef buf_size
9703 }
9709 }
9717 {
9744 }
9750 {
9785 }
9791 {
9797 }
9802 {
9803 HOST_WIDE_INT cond_code;
9806 {
9809 }
9814 }
9819 {
9824 {
9827 }
9831 ? ADDR_QUERY_LDP_STP_N
9834 }
9840 }
9841 }
9843 /* Print address 'x' of a memory access with mode 'mode'.
9844 'op' is the context required by aarch64_classify_address. It can either be
9845 MEM for a normal memory access or PARALLEL for LDP/STP. */
9846 static bool
9848 aarch64_addr_query_type type)
9849 {
9853 /* Check all addresses are Pmode - including ILP32. */
9857 {
9860 }
9864 {
9867 {
9870 }
9874 {
9875 HOST_WIDE_INT vnum
9881 }
9891 else
9900 else
9909 else
9915 /* Writeback is only supported for fixed-width modes. */
9918 {
9941 }
9953 }
9956 }
9958 /* Print address 'x' of a memory access with mode 'mode'. */
9959 static void
9961 {
9964 }
9966 bool
9968 {
9975 /* UNSPEC_TLS entries for a symbol include a LABEL_REF for the
9976 referencing instruction, but they are constant offsets, not
9977 symbols. */
9983 {
9985 {
9991 }
9994 }
9997 }
9999 /* Implement REGNO_REG_CLASS. */
10001 enum reg_class
10003 {
10025 }
10027 /* OFFSET is an address offset for mode MODE, which has SIZE bytes.
10028 If OFFSET is out of range, return an offset of an anchor point
10029 that is in range. Return 0 otherwise. */
10031 static HOST_WIDE_INT
10033 machine_mode mode)
10034 {
10035 /* Does it look like we'll need a 16-byte load/store-pair operation? */
10039 /* For offsets that aren't a multiple of the access size, the limit is
10040 -256...255. */
10042 {
10043 /* BLKmode typically uses LDP of X-registers. */
10047 }
10049 /* Small negative offsets are supported. */
10056 /* Use 12-bit offset by access size. */
10058 }
10060 static rtx
10062 {
10063 /* Try to split X+CONST into Y=X+(CONST & ~mask), Y+(CONST&mask),
10064 where mask is selected by alignment and size of the offset.
10065 We try to pick as large a range for the offset as possible to
10066 maximize the chance of a CSE. However, for aligned addresses
10067 we limit the range to 4k so that structures with different sized
10068 elements are likely to use the same base. We need to be careful
10069 not to split a CONST for some forms of address expression, otherwise
10070 it will generate sub-optimal code. */
10073 {
10079 {
10083 /* Force any scaling into a temp for CSE. */
10087 /* Let the pointer register be in op0. */
10091 /* If the pointer is virtual or frame related, then we know that
10092 virtual register instantiation or register elimination is going
10093 to apply a second constant. We want the two constants folded
10094 together easily. Therefore, emit as (OP0 + CONST) + OP1. */
10096 {
10100 }
10102 /* Otherwise, in order to encourage CSE (and thence loop strength
10103 reduce) scaled addresses, emit as (OP0 + OP1) + CONST. */
10107 }
10109 HOST_WIDE_INT size;
10111 {
10113 mode);
10115 {
10119 }
10120 }
10121 }
10124 }
10126 static reg_class_t
10128 reg_class_t rclass,
10129 machine_mode mode,
10131 {
10132 /* Use aarch64_sve_reload_mem for SVE memory reloads that cannot use
10133 LDR and STR. See the comment at the head of aarch64-sve.md for
10134 more details about the big-endian handling. */
10139 {
10143 {
10146 }
10147 }
10149 /* If we have to disable direct literal pool loads and stores because the
10150 function is too big, then we need a scratch register. */
10155 {
10158 }
10160 /* Without the TARGET_SIMD instructions we cannot move a Q register
10161 to a Q register directly. We need a scratch. */
10165 {
10168 }
10170 /* A TFmode or TImode memory access should be handled via an FP_REGS
10171 because AArch64 has richer addressing modes for LDR/STR instructions
10172 than LDP/STP instructions. */
10181 }
10183 static bool
10185 {
10188 /* If we need a frame pointer, ARG_POINTER_REGNUM and FRAME_POINTER_REGNUM
10189 can only eliminate to HARD_FRAME_POINTER_REGNUM. */
10193 }
10195 poly_int64
10197 {
10199 {
10206 }
10209 {
10213 }
10216 }
10218 /* Implement RETURN_ADDR_RTX. We do not support moving back to a
10219 previous frame. */
10221 rtx
10223 {
10227 }
10230 static void
10232 {
10237 {
10241 }
10244 {
10247 offset1);
10248 }
10249 else
10250 {
10253 offset2);
10254 }
10257 /* The trampoline needs an extra padding instruction. In case if BTI is
10258 enabled the padding instruction is replaced by the BTI instruction at
10259 the beginning. */
10265 }
10267 static void
10269 {
10273 /* Don't need to copy the trailing D-words, we fill those in below. */
10285 /* XXX We should really define a "clear_cache" pattern and use
10286 gen_clear_cache(). */
10291 ptr_mode);
10292 }
10294 static unsigned char
10296 {
10297 /* ??? Logically we should only need to provide a value when
10298 HARD_REGNO_MODE_OK says that at least one register in REGCLASS
10299 can hold MODE, but at the moment we need to handle all modes.
10300 Just ignore any runtime parts for registers that can't store them. */
10304 {
10334 }
10336 }
10338 static reg_class_t
10340 {
10345 {
10351 }
10353 /* Register eliminiation can result in a request for
10354 SP+constant->FP_REGS. We cannot support such operations which
10355 use SP as source and an FP_REG as destination, so reject out
10356 right now. */
10358 {
10361 /* Look through a possible SUBREG introduced by ILP32. */
10367 POINTER_REGS));
10369 }
10372 }
10374 void
10376 {
10378 }
10380 static void
10382 {
10385 else
10386 {
10388 /* While priority is known to be in range [0, 65535], so 18 bytes
10389 would be enough, the compiler might not know that. To avoid
10390 -Wformat-truncation false positive, use a larger size. */
10397 }
10398 }
10400 static void
10402 {
10405 else
10406 {
10408 /* While priority is known to be in range [0, 65535], so 18 bytes
10409 would be enough, the compiler might not know that. To avoid
10410 -Wformat-truncation false positive, use a larger size. */
10417 }
10418 }
10422 {
10428 {
10429 {
10432 },
10433 {
10436 },
10437 {
10440 },
10441 /* We assume that DImode is only generated when not optimizing and
10442 that we don't really need 64-bit address offsets. That would
10443 imply an object file with 8GB of code in a single function! */
10444 {
10447 }
10448 };
10457 /* Need to implement table size reduction, by chaning the code below. */
10467 }
10470 /* Return size in bits of an arithmetic operand which is shifted/scaled and
10471 masked such that it is suitable for a UXTB, UXTH, or UXTW extend
10472 operator. */
10474 int
10476 {
10478 {
10481 {
10485 }
10486 }
10488 }
10490 /* Constant pools are per function only when PC relative
10491 literal loads are true or we are in the large memory
10492 model. */
10496 {
10499 }
10501 static bool
10503 {
10504 /* We can't use blocks for constants when we're using a per-function
10505 constant pool. */
10507 }
10509 /* Select appropriate section for constants depending
10510 on where we place literal pools. */
10514 rtx x,
10516 {
10521 }
10523 /* Implement ASM_OUTPUT_POOL_EPILOGUE. */
10524 void
10526 HOST_WIDE_INT offset)
10527 {
10528 /* When using per-function literal pools, we must ensure that any code
10529 section is aligned to the minimal instruction length, lest we get
10530 errors from the assembler re "unaligned instructions". */
10533 }
10535 /* Costs. */
10537 /* Helper function for rtx cost calculation. Strip a shift expression
10538 from X. Returns the inner operand if successful, or the original
10539 expression on failure. */
10540 static rtx
10542 {
10545 /* We accept both ROTATERT and ROTATE: since the RHS must be a constant
10546 we can convert both to ROR during final output. */
10561 }
10563 /* Helper function for rtx cost calculation. Strip an extend
10564 expression from X. Returns the inner operand if successful, or the
10565 original expression on failure. We deal with a number of possible
10566 canonicalization variations here. If STRIP_SHIFT is true, then
10567 we can strip off a shift also. */
10568 static rtx
10570 {
10571 scalar_int_mode mode;
10577 /* Zero and sign extraction of a widened value. */
10585 /* It can also be represented (for zero-extend) as an AND with an
10586 immediate. */
10595 /* Now handle extended register, as this may also have an optional
10596 left shift by 1..4. */
10611 }
10613 /* Return true iff CODE is a shift supported in combination
10614 with arithmetic instructions. */
10616 static bool
10618 {
10620 }
10623 /* Return true iff X is a cheap shift without a sign extend. */
10625 static bool
10627 {
10653 }
10655 /* Helper function for rtx cost calculation. Calculate the cost of
10656 a MULT or ASHIFT, which may be part of a compound PLUS/MINUS rtx.
10657 Return the calculated cost of the expression, recursing manually in to
10658 operands where needed. */
10660 static int
10662 {
10678 /* Integer multiply/fma. */
10680 {
10681 /* The multiply will be canonicalized as a shift, cost it as such. */
10685 {
10689 {
10691 {
10692 /* If the shift is considered cheap,
10693 then don't add any cost. */
10695 ;
10697 /* ARITH + shift-by-register. */
10700 /* ARITH + extended register. We don't have a cost field
10701 for ARITH+EXTEND+SHIFT, so use extend_arith here. */
10703 else
10704 /* ARITH + shift-by-immediate. */
10706 }
10707 else
10708 /* LSL (immediate). */
10711 }
10712 /* Strip extends as we will have costed them in the case above. */
10719 }
10721 /* MNEG or [US]MNEGL. Extract the NEG operand and indicate that it's a
10722 compound and let the below cases handle it. After all, MNEG is a
10723 special-case alias of MSUB. */
10725 {
10728 }
10730 /* Integer multiplies or FMAs have zero/sign extending variants. */
10735 {
10740 {
10742 /* SMADDL/UMADDL/UMSUBL/SMSUBL. */
10744 else
10745 /* MUL/SMULL/UMULL. */
10747 }
10750 }
10752 /* This is either an integer multiply or a MADD. In both cases
10753 we want to recurse and cost the operands. */
10758 {
10760 /* MADD/MSUB. */
10762 else
10763 /* MUL. */
10765 }
10768 }
10769 else
10770 {
10772 {
10773 /* Floating-point FMA/FMUL can also support negations of the
10774 operands, unless the rounding mode is upward or downward in
10775 which case FNMUL is different than FMUL with operand negation. */
10779 {
10784 }
10787 /* FMADD/FNMADD/FNMSUB/FMSUB. */
10789 else
10790 /* FMUL/FNMUL. */
10792 }
10797 }
10798 }
10800 static int
10802 machine_mode mode,
10803 addr_space_t as ATTRIBUTE_UNUSED,
10805 {
10813 {
10815 {
10816 /* This is a CONST or SYMBOL ref which will be split
10817 in a different way depending on the code model in use.
10818 Cost it through the generic infrastructure. */
10820 /* Divide through by the cost of one instruction to
10821 bring it to the same units as the address costs. */
10823 /* The cost is then the cost of preparing the address,
10824 followed by an immediate (possibly 0) offset. */
10826 }
10827 else
10828 {
10829 /* This is most likely a jump table from a case
10830 statement. */
10832 }
10833 }
10836 {
10848 else
10867 }
10871 {
10872 /* For the sake of calculating the cost of the shifted register
10873 component, we can treat same sized modes in the same way. */
10880 else
10881 /* We can't tell, or this is a 128-bit vector. */
10883 }
10886 }
10888 /* Return the cost of a branch. If SPEED_P is true then the compiler is
10889 optimizing for speed. If PREDICTABLE_P is true then the branch is predicted
10890 to be taken. */
10892 int
10894 {
10895 /* When optimizing for speed, use the cost of unpredictable branches. */
10901 else
10903 }
10905 /* Return true if the RTX X in mode MODE is a zero or sign extract
10906 usable in an ADD or SUB (extended register) instruction. */
10907 static bool
10909 {
10910 /* Catch add with a sign extract.
10911 This is add_<optab><mode>_multp2. */
10914 {
10925 op1))
10926 {
10928 }
10929 }
10930 /* The simple case <ARITH>, XD, XN, XM, [us]xt.
10931 No shift. */
10937 }
10939 static bool
10941 {
10943 {
10955 }
10956 }
10958 /* Return true iff X is an rtx that will match an extr instruction
10959 i.e. as described in the *extr<mode>5_insn family of patterns.
10960 OP0 and OP1 will be set to the operands of the shifts involved
10961 on success and will be NULL_RTX otherwise. */
10963 static bool
10965 {
10967 scalar_int_mode mode;
10982 {
10983 /* Canonicalise locally to ashift in op0, lshiftrt in op1. */
10995 {
10999 }
11000 }
11003 }
11005 /* Calculate the cost of calculating (if_then_else (OP0) (OP1) (OP2)),
11006 storing it in *COST. Result is true if the total cost of the operation
11007 has now been calculated. */
11008 static bool
11010 {
11011 rtx inner;
11012 rtx comparator;
11016 {
11020 }
11021 else
11022 {
11026 }
11029 {
11030 /* Conditional branch. */
11033 else
11034 {
11036 {
11038 {
11039 /* TBZ/TBNZ/CBZ/CBNZ. */
11041 /* TBZ/TBNZ. */
11044 else
11045 /* CBZ/CBNZ. */
11049 }
11050 }
11052 {
11053 /* TBZ/TBNZ. */
11056 }
11057 }
11058 }
11060 {
11061 /* CCMP. */
11063 {
11064 /* Increase cost of CCMP reg, 0, imm, CC to prefer CMP reg, 0. */
11068 {
11075 else
11077 }
11079 }
11081 /* It's a conditional operation based on the status flags,
11082 so it must be some flavor of CSEL. */
11084 /* CSNEG, CSINV, and CSINC are handled for free as part of CSEL. */
11090 {
11091 /* CSEL with zero-extension (*cmovdi_insn_uxtw). */
11094 }
11099 }
11101 /* We don't know what this is, cost all operands. */
11103 }
11105 /* Check whether X is a bitfield operation of the form shift + extend that
11106 maps down to a UBFIZ/SBFIZ/UBFX/SBFX instruction. If so, return the
11107 operand to which the bitfield operation is applied. Otherwise return
11108 NULL_RTX. */
11110 static rtx
11112 {
11126 {
11144 }
11147 }
11149 /* Return true if the mask and a shift amount from an RTX of the form
11150 (x << SHFT_AMNT) & MASK are valid to combine into a UBFIZ instruction of
11151 mode MODE. See the *andim_ashift<mode>_bfiz pattern. */
11153 bool
11155 rtx shft_amnt)
11156 {
11162 }
11164 /* Return true if the masks and a shift amount from an RTX of the form
11165 ((x & MASK1) | ((y << SHIFT_AMNT) & MASK2)) are valid to combine into
11166 a BFI instruction of mode MODE. See *arch64_bfi patterns. */
11168 bool
11173 {
11176 /* Verify that there is no overlap in what bits are set in the two masks. */
11180 /* Verify that mask2 is not all zeros or ones. */
11184 /* The shift amount should always be less than the mode size. */
11187 /* Verify that the mask being shifted is contiguous and would be in the
11188 least significant bits after shifting by shft_amnt. */
11191 }
11193 /* Calculate the cost of calculating X, storing it in *COST. Result
11194 is true if the total cost of the operation has now been calculated. */
11195 static bool
11198 {
11203 scalar_int_mode int_mode;
11205 /* By default, assume that everything has equivalent cost to the
11206 cheapest instruction. Any additional costs are applied as a delta
11207 above this default. */
11211 {
11213 /* The cost depends entirely on the operands to SET. */
11219 {
11222 {
11236 }
11245 /* Fall through. */
11247 /* The cost is one per vector-register copied. */
11249 {
11252 }
11253 /* const0_rtx is in general free, but we will use an
11254 instruction to set a register to 0. */
11256 {
11257 /* The cost is 1 per register copied. */
11260 }
11261 else
11262 /* Cost is just the cost of the RHS of the set. */
11268 /* Bit-field insertion. Strip any redundant widening of
11269 the RHS to meet the width of the target. */
11280 {
11281 /* MOV immediate is assumed to always be cheap. */
11283 }
11284 else
11285 {
11286 /* BFM. */
11290 }
11295 /* We can't make sense of this, assume default cost. */
11298 }
11302 /* If an instruction can incorporate a constant within the
11303 instruction, the instruction's expression avoids calling
11304 rtx_cost() on the constant. If rtx_cost() is called on a
11305 constant, then it is usually because the constant must be
11306 moved into a register by one or more instructions.
11308 The exception is constant 0, which can be expressed
11309 as XZR/WZR and is therefore free. The exception to this is
11310 if we have (set (reg) (const0_rtx)) in which case we must cost
11311 the move. However, we can catch that when we cost the SET, so
11312 we don't need to consider that here. */
11315 else
11316 {
11317 /* To an approximation, building any other constant is
11318 proportionally expensive to the number of instructions
11319 required to build that constant. This is true whether we
11320 are compiling for SPEED or otherwise. */
11325 }
11330 /* First determine number of instructions to do the move
11331 as an integer constant. */
11335 {
11341 ? SImode
11347 }
11350 {
11351 /* mov[df,sf]_aarch64. */
11353 /* FMOV (scalar immediate). */
11356 {
11357 /* This will be a load from memory. */
11360 else
11362 }
11363 else
11364 /* Otherwise this is +0.0. We get this using MOVI d0, #0
11365 or MOV v0.s[0], wzr - neither of which are modeled by the
11366 cost tables. Just use the default cost. */
11367 {
11368 }
11369 }
11375 {
11376 /* For loads we want the base cost of a load, plus an
11377 approximation for the additional cost of the addressing
11378 mode. */
11392 }
11400 {
11402 {
11403 /* FNEG. */
11405 }
11407 }
11410 {
11413 {
11414 /* CSETM. */
11417 }
11419 /* Cost this as SUB wzr, X. */
11423 }
11426 {
11427 /* Support (neg(fma...)) as a single instruction only if
11428 sign of zeros is unimportant. This matches the decision
11429 making in aarch64.md. */
11431 {
11432 /* FNMADD. */
11435 }
11437 {
11438 /* FNMUL. */
11441 }
11443 /* FNEG. */
11446 }
11453 {
11456 else
11458 }
11468 {
11472 }
11475 {
11476 /* TODO: A write to the CC flags possibly costs extra, this
11477 needs encoding in the cost tables. */
11480 /* ANDS. */
11482 {
11485 }
11488 {
11489 /* ADDS (and CMN alias). */
11492 }
11495 {
11496 /* SUBS. */
11499 }
11504 {
11505 /* COMPARE of ZERO_EXTRACT form of TST-immediate.
11506 Handle it here directly rather than going to cost_logic
11507 since we know the immediate generated for the TST is valid
11508 so we can avoid creating an intermediate rtx for it only
11509 for costing purposes. */
11516 }
11519 {
11520 /* CMN. */
11527 }
11529 /* CMP.
11531 Compare can freely swap the order of operands, and
11532 canonicalization puts the more complex operation first.
11533 But the integer MINUS logic expects the shift/extend
11534 operation in op1. */
11537 {
11540 }
11542 }
11545 {
11546 /* FCMP. */
11551 {
11553 /* FCMP supports constant 0.0 for no extra cost. */
11555 }
11557 }
11560 {
11561 /* Vector compare. */
11566 {
11567 /* Vector cm (eq|ge|gt|lt|le) supports constant 0.0 for no extra
11568 cost. */
11570 }
11572 }
11576 {
11580 cost_minus:
11583 /* Detect valid immediates. */
11589 {
11591 /* SUB(S) (immediate). */
11594 }
11596 /* Look for SUB (extended register). */
11599 {
11607 }
11611 /* Cost this as an FMA-alike operation. */
11615 {
11618 speed);
11620 }
11625 {
11627 {
11628 /* Vector SUB. */
11630 }
11632 {
11633 /* SUB(S). */
11635 }
11637 {
11638 /* FSUB. */
11640 }
11641 }
11643 }
11646 {
11647 rtx new_op0;
11652 cost_plus:
11655 {
11656 /* CSINC. */
11660 }
11665 {
11669 /* ADD (immediate). */
11672 }
11676 /* Look for ADD (extended register). */
11679 {
11687 }
11689 /* Strip any extend, leave shifts behind as we will
11690 cost them through mult_cost. */
11695 {
11697 speed);
11699 }
11704 {
11706 {
11707 /* Vector ADD. */
11709 }
11711 {
11712 /* ADD. */
11714 }
11716 {
11717 /* FADD. */
11719 }
11720 }
11722 }
11728 {
11731 else
11733 }
11738 {
11742 {
11745 else
11747 }
11749 }
11752 {
11759 }
11760 /* Fall through. */
11763 cost_logic:
11768 {
11772 }
11780 {
11781 /* This is a UBFM/SBFM. */
11786 }
11789 {
11791 {
11792 /* We have a mask + shift version of a UBFIZ
11793 i.e. the *andim_ashift<mode>_bfiz pattern. */
11797 {
11804 }
11806 {
11807 /* We possibly get the immediate for free, this is not
11808 modelled. */
11815 }
11816 }
11817 else
11818 {
11821 /* Handle ORN, EON, or BIC. */
11827 /* If we had a shift on op0 then this is a logical-shift-
11828 by-register/immediate operation. Otherwise, this is just
11829 a logical operation. */
11831 {
11833 {
11834 /* Shift by immediate. */
11837 else
11839 }
11840 else
11842 }
11844 /* In both cases we want to cost both operands. */
11851 }
11852 }
11860 {
11861 /* Vector NOT. */
11864 }
11866 /* MVN-shifted-reg. */
11868 {
11875 }
11876 /* EON can have two forms: (xor (not a) b) but also (not (xor a b)).
11877 Handle the second form here taking care that 'a' in the above can
11878 be a shift. */
11880 {
11889 {
11892 else
11894 }
11897 }
11898 /* MVN. */
11907 /* If a value is written in SI mode, then zero extended to DI
11908 mode, the operation will in general be free as a write to
11909 a 'w' register implicitly zeroes the upper bits of an 'x'
11910 register. However, if this is
11912 (set (reg) (zero_extend (reg)))
11914 we must cost the explicit register move. */
11918 {
11921 /* If OP_COST is non-zero, then the cost of the zero extend
11922 is effectively the cost of the inner operation. Otherwise
11923 we have a MOV instruction and we take the cost from the MOV
11924 itself. This is true independently of whether we are
11925 optimizing for space or time. */
11930 }
11932 {
11933 /* All loads can zero extend to any size for free. */
11936 }
11940 {
11945 }
11948 {
11950 {
11951 /* UMOV. */
11953 }
11954 else
11955 {
11956 /* We generate an AND instead of UXTB/UXTH. */
11958 }
11959 }
11964 {
11965 /* LDRSH. */
11967 {
11974 }
11976 }
11980 {
11985 }
11988 {
11991 else
11993 }
12001 {
12003 {
12005 {
12006 /* Vector shift (immediate). */
12008 }
12009 else
12010 {
12011 /* LSL (immediate), UBMF, UBFIZ and friends. These are all
12012 aliases. */
12014 }
12015 }
12017 /* We can incorporate zero/sign extend for free. */
12024 }
12025 else
12026 {
12028 {
12030 /* Vector shift (register). */
12032 }
12033 else
12034 {
12036 /* LSLV. */
12043 {
12045 /* We already demanded XEXP (op1, 0) to be REG_P, so
12046 don't recurse into it. */
12048 }
12049 }
12051 }
12061 {
12062 /* ASR (immediate) and friends. */
12064 {
12067 else
12069 }
12073 }
12074 else
12075 {
12077 {
12079 /* Vector shift (register). */
12081 }
12082 else
12083 {
12085 /* ASR (register) and friends. */
12092 {
12094 /* We already demanded XEXP (op1, 0) to be REG_P, so
12095 don't recurse into it. */
12097 }
12098 }
12100 }
12106 {
12107 /* LDR. */
12110 }
12113 {
12114 /* ADRP, followed by ADD. */
12118 }
12121 {
12122 /* ADR. */
12125 }
12128 {
12129 /* One extra load instruction, after accessing the GOT. */
12133 }
12138 /* ADRP/ADD (immediate). */
12145 /* UBFX/SBFX. */
12147 {
12150 else
12152 }
12154 /* We can trust that the immediates used will be correct (there
12155 are no by-register forms), so we need only cost op0. */
12161 /* aarch64_rtx_mult_cost always handles recursion to its
12162 operands. */
12166 /* We can expand signed mod by power of 2 using a NEGS, two parallel
12167 ANDs and a CSNEG. Assume here that CSNEG is the same as the cost of
12168 an unconditional negate. This case should only ever be reached through
12169 the set_smod_pow2_cheap check in expmed.c. */
12173 {
12174 /* We expand to 4 instructions. Reset the baseline. */
12182 }
12184 /* Fall-through. */
12187 {
12188 /* Slighly prefer UMOD over SMOD. */
12195 }
12202 {
12206 /* There is no integer SQRT, so only DIV and UDIV can get
12207 here. */
12209 /* Slighly prefer UDIV over SDIV. */
12211 else
12213 }
12239 {
12242 else
12244 }
12246 /* FMSUB, FNMADD, and FNMSUB are free. */
12253 /* aarch64_fnma4_elt_to_64v2df has the NEG as operand 1,
12254 and the by-element operand as operand 0. */
12258 /* Catch vector-by-element operations. The by-element operand can
12259 either be (vec_duplicate (vec_select (x))) or just
12260 (vec_select (x)), depending on whether we are multiplying by
12261 a vector or a scalar.
12263 Canonicalization is not very good in these cases, FMA4 will put the
12264 by-element operand as operand 0, FNMA4 will have it as operand 1. */
12275 /* If the remaining parameters are not registers,
12276 get the cost to put them into registers. */
12290 {
12292 {
12293 /*Vector truncate. */
12295 }
12296 else
12298 }
12303 {
12305 {
12306 /*Vector conversion. */
12308 }
12309 else
12311 }
12317 /* Strip the rounding part. They will all be implemented
12318 by the fcvt* family of instructions anyway. */
12320 {
12329 }
12332 {
12335 else
12337 }
12339 /* We can combine fmul by a power of 2 followed by a fcvt into a single
12340 fixed-point fcvt. */
12345 {
12349 }
12356 {
12357 /* ABS (vector). */
12360 }
12362 {
12365 /* FABD, which is analogous to FADD. */
12367 {
12374 }
12375 /* Simple FABS is analogous to FNEG. */
12378 }
12379 else
12380 {
12381 /* Integer ABS will either be split to
12382 two arithmetic instructions, or will be an ABS
12383 (scalar), which we don't model. */
12387 }
12393 {
12396 else
12397 {
12398 /* FMAXNM/FMINNM/FMAX/FMIN.
12399 TODO: This may not be accurate for all implementations, but
12400 we do not model this in the cost tables. */
12402 }
12403 }
12407 /* The floating point round to integer frint* instructions. */
12409 {
12414 }
12417 {
12422 }
12427 /* Decompose <su>muldi3_highpart. */
12429 mode == DImode
12430 /* (lshiftrt:TI */
12433 /* (mult:TI */
12435 /* (ANY_EXTEND:TI (reg:DI))
12436 (ANY_EXTEND:TI (reg:DI))) */
12443 /* (const_int 64) */
12446 {
12447 /* UMULH/SMULH. */
12455 }
12457 /* Fall through. */
12460 }
12468 }
12470 /* Wrapper around aarch64_rtx_costs, dumps the partial, or total cost
12471 calculated for X. This cost is stored in *COST. Returns true
12472 if the total cost of X was calculated. */
12473 static bool
12476 {
12481 {
12486 }
12489 }
12491 static int
12494 {
12500 /* Caller save and pointer regs are equivalent to GENERAL_REGS. */
12507 /* Make RDFFR very expensive. In particular, if we know that the FFR
12508 contains a PTRUE (e.g. after a SETFFR), we must never use RDFFR
12509 as a way of obtaining a PTRUE. */
12515 /* Moving between GPR and stack cost is the same as GP2GP. */
12520 /* To/From the stack register, we move via the gprs. */
12526 {
12527 /* 128-bit operations on general registers require 2 instructions. */
12535 /* When AdvSIMD instructions are disabled it is not possible to move
12536 a 128-bit value directly between Q registers. This is handled in
12537 secondary reload. A general register is used as a scratch to move
12538 the upper DI value and the lower DI value is moved directly,
12539 hence the cost is the sum of three moves. */
12544 }
12554 }
12556 static int
12558 reg_class_t rclass ATTRIBUTE_UNUSED,
12560 {
12562 }
12564 /* Implement TARGET_INIT_BUILTINS. */
12565 static void
12567 {
12570 }
12572 /* Implement TARGET_FOLD_BUILTIN. */
12573 static tree
12575 {
12580 {
12586 }
12588 }
12590 /* Implement TARGET_GIMPLE_FOLD_BUILTIN. */
12591 static bool
12593 {
12600 {
12608 }
12615 }
12617 /* Implement TARGET_EXPAND_BUILTIN. */
12618 static rtx
12620 {
12625 {
12631 }
12633 }
12635 /* Implement TARGET_BUILTIN_DECL. */
12636 static tree
12638 {
12641 {
12647 }
12649 }
12651 /* Return true if it is safe and beneficial to use the approximate rsqrt optabs
12652 to optimize 1.0/sqrt. */
12654 static bool
12656 {
12658 && flag_unsafe_math_optimizations
12662 }
12664 /* Function to decide when to use the approximate reciprocal square root
12665 builtin. */
12667 static tree
12669 {
12677 {
12683 }
12685 }
12687 /* Emit instruction sequence to compute either the approximate square root
12688 or its approximate reciprocal, depending on the flag RECP, and return
12689 whether the sequence was emitted or not. */
12691 bool
12693 {
12697 {
12700 }
12703 {
12710 || flag_trapping_math
12711 || !flag_unsafe_math_optimizations
12714 }
12715 else
12716 /* Caller assumes we cannot fail. */
12724 /* When calculating the approximate square root, compare the
12725 argument with 0.0 and create a mask. */
12731 /* Estimate the approximate reciprocal square root. */
12735 /* Iterate over the series twice for SF and thrice for DF. */
12738 /* Optionally iterate over the series once less for faster performance
12739 while sacrificing the accuracy. */
12742 iterations--;
12744 /* Iterate over the series to calculate the approximate reciprocal square
12745 root. */
12748 {
12756 }
12759 {
12760 /* Qualify the approximate reciprocal square root when the argument is
12761 0.0 by squashing the intermediary result to 0.0. */
12767 /* Calculate the approximate square root. */
12769 }
12771 /* Finalize the approximation. */
12775 }
12777 /* Emit the instruction sequence to compute the approximation for the division
12778 of NUM by DEN in QUO and return whether the sequence was emitted or not. */
12780 bool
12782 {
12793 || flag_trapping_math
12794 || !flag_unsafe_math_optimizations
12802 /* Estimate the approximate reciprocal. */
12806 /* Iterate over the series twice for SF and thrice for DF. */
12809 /* Optionally iterate over the series once less for faster performance,
12810 while sacrificing the accuracy. */
12812 iterations--;
12814 /* Iterate over the series to calculate the approximate reciprocal. */
12817 {
12822 }
12825 {
12826 /* As the approximate reciprocal of DEN is already calculated, only
12827 calculate the approximate division when NUM is not 1.0. */
12830 }
12832 /* Finalize the approximation. */
12835 }
12837 /* Return the number of instructions that can be issued per cycle. */
12838 static int
12840 {
12842 }
12844 /* Implement TARGET_SCHED_VARIABLE_ISSUE. */
12845 static int
12847 {
12859 }
12861 static int
12863 {
12867 }
12870 /* Implement TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD as
12871 autopref_multipass_dfa_lookahead_guard from haifa-sched.c. It only
12872 has an effect if PARAM_SCHED_AUTOPREF_QUEUE_DEPTH > 0. */
12874 static int
12877 {
12879 }
12882 /* Vectorizer cost model target hooks. */
12884 /* Implement targetm.vectorize.builtin_vectorization_cost. */
12885 static int
12887 tree vectype,
12889 {
12898 {
12949 }
12950 }
12952 /* Return true if STMT_INFO extends the result of a load. */
12953 static bool
12955 {
12972 }
12974 /* Return true if STMT_INFO is an integer truncation. */
12975 static bool
12977 {
12987 }
12989 /* STMT_COST is the cost calculated by aarch64_builtin_vectorization_cost
12990 for STMT_INFO, which has cost kind KIND. Adjust the cost as necessary
12991 for SVE targets. */
12992 static unsigned int
12995 {
12996 /* Unlike vec_promote_demote, vector_stmt conversions do not change the
12997 vector register size or number of units. Integer promotions of this
12998 type therefore map to SXT[BHW] or UXT[BHW].
13000 Most loads have extending forms that can do the sign or zero extension
13001 on the fly. Optimistically assume that a load followed by an extension
13002 will fold to this form during combine, and that the extension therefore
13003 comes for free. */
13007 /* For similar reasons, vector_stmt integer truncations are a no-op,
13008 because we can just ignore the unused upper bits of the source. */
13013 }
13015 /* Implement targetm.vectorize.add_stmt_cost. */
13016 static unsigned
13020 {
13025 {
13033 /* Statements in an inner loop relative to the loop being
13034 vectorized are weighted more heavily. The value here is
13035 arbitrary and could potentially be improved with analysis. */
13041 }
13044 }
13048 /* Parse the TO_PARSE string and put the architecture struct that it
13049 selects into RES and the architectural features into ISA_FLAGS.
13050 Return an aarch64_parse_opt_result describing the parse result.
13051 If there is an error parsing, RES and ISA_FLAGS are left unchanged.
13052 When the TO_PARSE string contains an invalid extension,
13053 a copy of the string is created and stored to INVALID_EXTENSION. */
13055 static enum aarch64_parse_opt_result
13058 {
13067 else
13074 /* Loop through the list of supported ARCHes to find a match. */
13076 {
13079 {
13083 {
13084 /* TO_PARSE string contains at least one extension. */
13085 enum aarch64_parse_opt_result ext_res
13090 }
13091 /* Extension parsing was successful. Confirm the result
13092 arch and ISA flags. */
13096 }
13097 }
13099 /* ARCH name not found in list. */
13101 }
13103 /* Parse the TO_PARSE string and put the result tuning in RES and the
13104 architecture flags in ISA_FLAGS. Return an aarch64_parse_opt_result
13105 describing the parse result. If there is an error parsing, RES and
13106 ISA_FLAGS are left unchanged.
13107 When the TO_PARSE string contains an invalid extension,
13108 a copy of the string is created and stored to INVALID_EXTENSION. */
13110 static enum aarch64_parse_opt_result
13113 {
13122 else
13129 /* Loop through the list of supported CPUs to find a match. */
13131 {
13133 {
13138 {
13139 /* TO_PARSE string contains at least one extension. */
13140 enum aarch64_parse_opt_result ext_res
13145 }
13146 /* Extension parsing was successfull. Confirm the result
13147 cpu and ISA flags. */
13151 }
13152 }
13154 /* CPU name not found in list. */
13156 }
13158 /* Parse the TO_PARSE string and put the cpu it selects into RES.
13159 Return an aarch64_parse_opt_result describing the parse result.
13160 If the parsing fails the RES does not change. */
13162 static enum aarch64_parse_opt_result
13164 {
13167 /* Loop through the list of supported CPUs to find a match. */
13169 {
13171 {
13174 }
13175 }
13177 /* CPU name not found in list. */
13179 }
13181 /* Parse TOKEN, which has length LENGTH to see if it is an option
13182 described in FLAG. If it is, return the index bit for that fusion type.
13183 If not, error (printing OPTION_NAME) and return zero. */
13185 static unsigned int
13190 {
13192 {
13196 }
13200 }
13202 /* Parse OPTION which is a comma-separated list of flags to enable.
13203 FLAGS gives the list of flags we understand, INITIAL_STATE gives any
13204 default state we inherit from the CPU tuning structures. OPTION_NAME
13205 gives the top-level option we are parsing in the -moverride string,
13206 for use in error messages. */
13208 static unsigned int
13213 {
13220 {
13223 token_length,
13224 flags,
13225 option_name);
13226 /* If we find "none" (or, for simplicity's sake, an error) anywhere
13227 in the token stream, reset the supported operations. So:
13229 adrp+add.cmp+branch.none.adrp+add
13231 would have the result of turning on only adrp+add fusion. */
13237 }
13239 /* We ended with a comma, print something. */
13241 {
13244 }
13246 /* We still have one more token to parse. */
13249 token_length,
13250 flags,
13251 option_name);
13257 }
13259 /* Support for overriding instruction fusion. */
13261 static void
13264 {
13266 aarch64_fusible_pairs,
13269 }
13271 /* Support for overriding other tuning flags. */
13273 static void
13276 {
13277 tune->extra_tuning_flags
13279 aarch64_tuning_flags,
13282 }
13284 /* Parse the sve_width tuning moverride string in TUNE_STRING.
13285 Accept the valid SVE vector widths allowed by
13286 aarch64_sve_vector_bits_enum and use it to override sve_width
13287 in TUNE. */
13289 static void
13292 {
13297 {
13300 }
13302 {
13311 }
13313 }
13315 /* Parse TOKEN, which has length LENGTH to see if it is a tuning option
13316 we understand. If it is, extract the option string and handoff to
13317 the appropriate function. */
13319 void
13323 {
13329 {
13332 }
13334 /* Get the length of the option name. */
13336 /* Skip the '=' to get to the option string. */
13337 option_part++;
13340 {
13342 {
13345 }
13346 }
13350 }
13352 /* A checking mechanism for the implementation of the tls size. */
13354 static void
13356 {
13361 {
13363 /* Both the default and maximum TLS size allowed under tiny is 1M which
13364 needs two instructions to address, so we clamp the size to 24. */
13369 /* The maximum TLS size allowed under small is 4G. */
13374 /* The maximum TLS size allowed under large is 16E.
13375 FIXME: 16E should be 64bit, we only support 48bit offset now. */
13381 }
13384 }
13386 /* Parse STRING looking for options in the format:
13387 string :: option:string
13388 option :: name=substring
13389 name :: {a-z}
13390 substring :: defined by option. */
13392 static void
13395 {
13406 {
13408 /* Make this substring look like a string. */
13412 }
13414 /* One last option to parse. */
13417 }
13420 static void
13422 {
13424 {
13425 opts->x_aarch64_branch_protection_string
13427 }
13429 /* PR 70044: We have to be careful about being called multiple times for the
13430 same function. This means all changes should be repeatable. */
13432 /* Set aarch64_use_frame_pointer based on -fno-omit-frame-pointer.
13433 Disable the frame pointer flag so the mid-end will not use a frame
13434 pointer in leaf functions in order to support -fomit-leaf-frame-pointer.
13435 Set x_flag_omit_frame_pointer to the special value 2 to differentiate
13436 between -fomit-frame-pointer (1) and -fno-omit-frame-pointer (2). */
13441 /* If not optimizing for size, set the default
13442 alignment to what the target wants. */
13444 {
13451 }
13453 /* We default to no pc-relative literal loads. */
13457 /* If -mpc-relative-literal-loads is set on the command line, this
13458 implies that the user asked for PC relative literal loads. */
13462 /* In the tiny memory model it makes no sense to disallow PC relative
13463 literal pool loads. */
13468 /* When enabling the lower precision Newton series for the square root, also
13469 enable it for the reciprocal square root, since the latter is an
13470 intermediary step for the former. */
13473 }
13475 /* 'Unpack' up the internal tuning structs and update the options
13476 in OPTS. The caller must have set up selected_tune and selected_arch
13477 as all the other target-specific codegen decisions are
13478 derived from them. */
13480 void
13482 {
13485 /* Make a copy of the tuning parameters attached to the core, which
13486 we may later overwrite. */
13494 /* This target defaults to strict volatile bitfields. */
13500 {
13503 aarch64_stack_protector_guard_offset_str);
13504 }
13509 {
13511 "%<-mstack-protector-guard-reg%> must be used "
13513 }
13516 {
13519 }
13522 {
13531 }
13538 {
13550 }
13552 /* We don't mind passing in global_options_set here as we don't use
13553 the *options_set structs anyway. */
13557 /* Set up parameters to be used in prefetching algorithm. Do not
13558 override the defaults unless we are tuning for a core we have
13559 researched values for. */
13562 param_simultaneous_prefetches,
13566 param_l1_cache_size,
13570 param_l1_cache_line_size,
13574 param_l2_cache_size,
13581 param_prefetch_minimum_stride,
13584 /* Use the alternative scheduling-pressure algorithm by default. */
13586 param_sched_pressure_algorithm,
13587 SCHED_PRESSURE_MODEL);
13589 /* Validate the guard size. */
13597 /* Enforce that interval is the same size as size so the mid-end does the
13598 right thing. */
13600 param_stack_clash_protection_probe_interval,
13601 guard_size);
13603 /* The maybe_set calls won't update the value if the user has explicitly set
13604 one. Which means we need to validate that probing interval and guard size
13605 are equal. */
13606 int probe_interval
13612 /* Enable sw prefetching at specified optimization level for
13613 CPUS that have prefetch. Lower optimization level threshold by 1
13614 when profiling is enabled. */
13629 }
13631 /* Print a hint with a suggestion for a core or architecture name that
13632 most closely resembles what the user passed in STR. ARCH is true if
13633 the user is asking for an architecture name. ARCH is false if the user
13634 is asking for a core name. */
13636 static void
13638 {
13644 #ifdef HAVE_LOCAL_CPU_DETECT
13645 /* Add also "native" as possible value. */
13648 #endif
13655 else
13659 }
13661 /* Print a hint with a suggestion for a core name that most closely resembles
13662 what the user passed in STR. */
13666 {
13668 }
13670 /* Print a hint with a suggestion for an architecture name that most closely
13671 resembles what the user passed in STR. */
13675 {
13677 }
13680 /* Print a hint with a suggestion for an extension name
13681 that most closely resembles what the user passed in STR. */
13683 void
13685 {
13693 else
13697 }
13699 /* Validate a command-line -mcpu option. Parse the cpu and extensions (if any)
13700 specified in STR and throw errors if appropriate. Put the results if
13701 they are valid in RES and ISA_FLAGS. Return whether the option is
13702 valid. */
13704 static bool
13707 {
13709 enum aarch64_parse_opt_result parse_res
13716 {
13731 }
13734 }
13736 /* Parses CONST_STR for branch protection features specified in
13737 aarch64_branch_protect_types, and set any global variables required. Returns
13738 the parsing result and assigns LAST_STR to the last processed token from
13739 CONST_STR so that it can be used for error reporting. */
13741 static enum
13744 {
13751 else
13752 {
13754 /* Reset the branch protection features to their defaults. */
13758 {
13761 /* Search for this type. */
13763 {
13765 {
13770 }
13771 else
13772 type++;
13773 }
13775 {
13777 /* Loop through each token until we find one that isn't a
13778 subtype. */
13780 {
13783 /* Search for the subtype. */
13786 {
13788 {
13793 }
13794 else
13795 subtype++;
13796 }
13797 }
13798 }
13801 }
13802 }
13803 /* Copy the last processed token into the argument to pass it back.
13804 Used by option and attribute validation to print the offending token. */
13806 {
13809 }
13811 {
13812 /* If needed, alloc the accepted string then copy in const_str.
13813 Used by override_option_after_change_1. */
13816 BRANCH_PROTECT_STR_MAX
13820 /* Forcibly null-terminate. */
13822 }
13824 }
13826 static bool
13828 {
13838 }
13840 /* Validate a command-line -march option. Parse the arch and extensions
13841 (if any) specified in STR and throw errors if appropriate. Put the
13842 results, if they are valid, in RES and ISA_FLAGS. Return whether the
13843 option is valid. */
13845 static bool
13848 {
13850 enum aarch64_parse_opt_result parse_res
13857 {
13872 }
13875 }
13877 /* Validate a command-line -mtune option. Parse the cpu
13878 specified in STR and throw errors if appropriate. Put the
13879 result, if it is valid, in RES. Return whether the option is
13880 valid. */
13882 static bool
13884 {
13885 enum aarch64_parse_opt_result parse_res
13892 {
13902 }
13904 }
13906 /* Return the CPU corresponding to the enum CPU.
13907 If it doesn't specify a cpu, return the default. */
13911 {
13915 /* The & 0x3f is to extract the bottom 6 bits that encode the
13916 default cpu as selected by the --with-cpu GCC configure option
13917 in config.gcc.
13918 ???: The whole TARGET_CPU_DEFAULT and AARCH64_CPU_DEFAULT_FLAGS
13919 flags mechanism should be reworked to make it more sane. */
13921 }
13923 /* Return the architecture corresponding to the enum ARCH.
13924 If it doesn't specify a valid architecture, return the default. */
13928 {
13935 }
13937 /* Return the VG value associated with -msve-vector-bits= value VALUE. */
13939 static poly_uint16
13941 {
13942 /* 128-bit SVE and Advanced SIMD modes use different register layouts
13943 on big-endian targets, so we would need to forbid subregs that convert
13944 from one to the other. By default a reinterpret sequence would then
13945 involve a store to memory in one mode and a load back in the other.
13946 Even if we optimize that sequence using reverse instructions,
13947 it would still be a significant potential overhead.
13949 For now, it seems better to generate length-agnostic code for that
13950 case instead. */
13954 else
13956 }
13958 /* Implement TARGET_OPTION_OVERRIDE. This is called once in the beginning
13959 and is used to parse the -m{cpu,tune,arch} strings and setup the initial
13960 tuning structs. In particular it must set selected_tune and
13961 aarch64_isa_flags that define the available ISA features and tuning
13962 decisions. It must also set selected_arch as this will be used to
13963 output the .arch asm tags for each function. */
13965 static void
13967 {
13983 /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
13984 If either of -march or -mtune is given, they override their
13985 respective component of -mcpu. */
13997 #ifdef SUBTARGET_OVERRIDE_OPTIONS
13998 SUBTARGET_OVERRIDE_OPTIONS;
13999 #endif
14001 /* If the user did not specify a processor, choose the default
14002 one for them. This will be the CPU set during configuration using
14003 --with-cpu, otherwise it is "generic". */
14005 {
14007 {
14011 }
14012 else
14013 {
14014 /* Get default configure-time CPU. */
14017 }
14021 }
14022 /* If both -mcpu and -march are specified check that they are architecturally
14023 compatible, warn if they're not and prefer the -march ISA flags. */
14025 {
14027 {
14031 }
14036 }
14037 else
14038 {
14039 /* -mcpu but no -march. */
14046 }
14048 /* Set the arch as well as we will need it when outputing
14049 the .arch directive in assembly. */
14051 {
14054 }
14060 {
14061 #ifdef TARGET_ENABLE_BTI
14063 #else
14065 #endif
14066 }
14068 /* Return address signing is currently not supported for ILP32 targets. For
14069 LP64 targets use the configured option in the absence of a command-line
14070 option for -mbranch-protection. */
14072 {
14073 #ifdef TARGET_ENABLE_PAC_RET
14075 #else
14077 #endif
14078 }
14080 #ifndef HAVE_AS_MABI_OPTION
14081 /* The compiler may have been configured with 2.23.* binutils, which does
14082 not have support for ILP32. */
14085 #endif
14087 /* Convert -msve-vector-bits to a VG count. */
14093 /* Make sure we properly set up the explicit options. */
14102 /* The pass to insert speculation tracking runs before
14103 shrink-wrapping and the latter does not know how to update the
14104 tracking status. So disable it in this case. */
14110 /* Save these options as the default ones in case we push and pop them later
14111 while processing functions with potential target attributes. */
14112 target_option_default_node = target_option_current_node
14114 }
14116 /* Implement targetm.override_options_after_change. */
14118 static void
14120 {
14122 }
14126 {
14130 }
14132 void
14134 {
14136 }
14138 /* A checking mechanism for the implementation of the various code models. */
14139 static void
14141 {
14143 {
14145 {
14150 #ifdef HAVE_AS_SMALL_PIC_RELOCS
14152 ? AARCH64_CMODEL_SMALL_PIC
14154 #else
14156 #endif
14164 }
14165 }
14166 else
14168 }
14170 /* Implement TARGET_OPTION_SAVE. */
14172 static void
14174 {
14176 ptr->x_aarch64_branch_protection_string
14178 }
14180 /* Implements TARGET_OPTION_RESTORE. Restore the backend codegen decisions
14181 using the information saved in PTR. */
14183 static void
14185 {
14191 opts->x_aarch64_branch_protection_string
14194 {
14196 NULL);
14197 }
14200 }
14202 /* Implement TARGET_OPTION_PRINT. */
14204 static void
14206 {
14217 }
14221 void
14223 {
14225 }
14227 /* Restore or save the TREE_TARGET_GLOBALS from or to NEW_TREE.
14228 Used by aarch64_set_current_function and aarch64_pragma_target_parse to
14229 make sure optab availability predicates are recomputed when necessary. */
14231 void
14233 {
14238 else
14240 }
14242 /* Implement TARGET_SET_CURRENT_FUNCTION. Unpack the codegen decisions
14243 like tuning and ISA features from the DECL_FUNCTION_SPECIFIC_TARGET
14244 of the function, if such exists. This function may be called multiple
14245 times on a single function so use aarch64_previous_fndecl to avoid
14246 setting up identical state. */
14248 static void
14250 {
14254 tree old_tree = (aarch64_previous_fndecl
14260 /* If current function has no attributes but the previous one did,
14261 use the default node. */
14265 /* If nothing to do, return. #pragma GCC reset or #pragma GCC pop to
14266 the default have been handled by aarch64_save_restore_target_globals from
14267 aarch64_pragma_target_parse. */
14273 /* First set the target options. */
14277 }
14279 /* Enum describing the various ways we can handle attributes.
14280 In many cases we can reuse the generic option handling machinery. */
14282 enum aarch64_attr_opt_type
14283 {
14287 aarch64_attr_custom /* Attribute requires a custom handling function. */
14288 };
14290 /* All the information needed to handle a target attribute.
14291 NAME is the name of the attribute.
14292 ATTR_TYPE specifies the type of behavior of the attribute as described
14293 in the definition of enum aarch64_attr_opt_type.
14294 ALLOW_NEG is true if the attribute supports a "no-" form.
14295 HANDLER is the function that takes the attribute string as an argument
14296 It is needed only when the ATTR_TYPE is aarch64_attr_custom.
14297 OPT_NUM is the enum specifying the option that the attribute modifies.
14298 This is needed for attributes that mirror the behavior of a command-line
14299 option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
14300 aarch64_attr_enum. */
14302 struct aarch64_attribute_info
14303 {
14309 };
14311 /* Handle the ARCH_STR argument to the arch= target attribute. */
14313 static bool
14315 {
14318 enum aarch64_parse_opt_result parse_res
14322 {
14327 }
14330 {
14345 }
14348 }
14350 /* Handle the argument CPU_STR to the cpu= target attribute. */
14352 static bool
14354 {
14357 enum aarch64_parse_opt_result parse_res
14361 {
14369 }
14372 {
14387 }
14390 }
14392 /* Handle the argument STR to the branch-protection= attribute. */
14394 static bool
14396 {
14402 {
14413 /* Fall through. */
14418 }
14421 }
14423 /* Handle the argument STR to the tune= target attribute. */
14425 static bool
14427 {
14429 enum aarch64_parse_opt_result parse_res
14433 {
14438 }
14441 {
14448 }
14451 }
14453 /* Parse an architecture extensions target attribute string specified in STR.
14454 For example "+fp+nosimd". Show any errors if needed. Return TRUE
14455 if successful. Update aarch64_isa_flags to reflect the ISA features
14456 modified. */
14458 static bool
14460 {
14464 /* We allow "+nothing" in the beginning to clear out all architectural
14465 features if the user wants to handpick specific features. */
14467 {
14470 }
14476 {
14479 }
14482 {
14494 }
14497 }
14499 /* The target attributes that we support. On top of these we also support just
14500 ISA extensions, like __attribute__ ((target ("+crc"))), but that case is
14501 handled explicitly in aarch64_process_one_target_attr. */
14504 {
14506 OPT_mgeneral_regs_only },
14508 OPT_mfix_cortex_a53_835769 },
14510 OPT_mfix_cortex_a53_843419 },
14514 OPT_momit_leaf_frame_pointer },
14517 OPT_march_ },
14520 OPT_mtune_ },
14524 OPT_msign_return_address_ },
14526 };
14528 /* Parse ARG_STR which contains the definition of one target attribute.
14529 Show appropriate errors if any or return true if the attribute is valid. */
14531 static bool
14533 {
14539 {
14542 }
14547 /* We have something like __attribute__ ((target ("+fp+nosimd"))).
14548 It is easier to detect and handle it explicitly here rather than going
14549 through the machinery for the rest of the target attributes in this
14550 function. */
14555 {
14558 }
14561 /* If we found opt=foo then terminate STR_TO_CHECK at the '='
14562 and point ARG to "foo". */
14564 {
14566 arg++;
14567 }
14571 {
14572 /* If the names don't match up, or the user has given an argument
14573 to an attribute that doesn't accept one, or didn't give an argument
14574 to an attribute that expects one, fail to match. */
14583 {
14586 }
14588 /* If the name matches but the attribute does not allow "no-" versions
14589 then we can't match. */
14591 {
14594 }
14597 {
14598 /* Has a custom handler registered.
14599 For example, cpu=, arch=, tune=. */
14606 /* Either set or unset a boolean option. */
14608 {
14616 }
14617 /* Set or unset a bit in the target_flags. aarch64_handle_option
14618 should know what mask to apply given the option number. */
14620 {
14622 /* We only need to specify the option number.
14623 aarch64_handle_option will know which mask to apply. */
14629 }
14630 /* Use the option setting machinery to set an option to an enum. */
14632 {
14639 {
14642 global_dc);
14643 }
14644 else
14645 {
14647 }
14649 }
14652 }
14653 }
14655 /* If we reached here we either have found an attribute and validated
14656 it or didn't match any. If we matched an attribute but its arguments
14657 were malformed we will have returned false already. */
14659 }
14661 /* Count how many times the character C appears in
14662 NULL-terminated string STR. */
14664 static unsigned int
14666 {
14669 {
14671 res++;
14673 str++;
14674 }
14677 }
14679 /* Parse the tree in ARGS that contains the target attribute information
14680 and update the global target options space. */
14682 bool
14684 {
14686 {
14687 do
14688 {
14691 {
14694 }
14699 }
14702 {
14705 }
14712 {
14715 }
14717 /* Used to catch empty spaces between commas i.e.
14718 attribute ((target ("attr1,,attr2"))). */
14721 /* Handle multiple target attributes separated by ','. */
14726 {
14727 num_attrs++;
14729 {
14732 }
14735 }
14738 {
14741 }
14744 }
14746 /* Implement TARGET_OPTION_VALID_ATTRIBUTE_P. This is used to
14747 process attribute ((target ("..."))). */
14749 static bool
14751 {
14754 tree old_optimize;
14758 /* If what we're processing is the current pragma string then the
14759 target option node is already stored in target_option_current_node
14760 by aarch64_pragma_target_parse in aarch64-c.c. Use that to avoid
14761 having to re-parse the string. This is especially useful to keep
14762 arm_neon.h compile times down since that header contains a lot
14763 of intrinsics enclosed in pragmas. */
14765 {
14768 }
14774 /* If the function changed the optimization levels as well as setting
14775 target options, start with the optimizations specified. */
14780 /* Save the current target options to restore at the end. */
14783 /* If fndecl already has some target attributes applied to it, unpack
14784 them so that we add this attribute on top of them, rather than
14785 overwriting them. */
14787 {
14793 }
14794 else
14800 /* Set up any additional state. */
14802 {
14804 /* Initialize SIMD builtins if we haven't already.
14805 Set current_target_pragma to NULL for the duration so that
14806 the builtin initialization code doesn't try to tag the functions
14807 being built with the attributes specified by any current pragma, thus
14808 going into an infinite recursion. */
14810 {
14815 }
14817 }
14818 else
14824 {
14829 }
14837 }
14839 /* Helper for aarch64_can_inline_p. In the case where CALLER and CALLEE are
14840 tri-bool options (yes, no, don't care) and the default value is
14841 DEF, determine whether to reject inlining. */
14843 static bool
14846 {
14847 /* If the callee doesn't care, always allow inlining. */
14851 /* If the caller doesn't care, always allow inlining. */
14855 /* Otherwise, allow inlining if either the callee and caller values
14856 agree, or if the callee is using the default value. */
14858 }
14860 /* Implement TARGET_CAN_INLINE_P. Decide whether it is valid
14861 to inline CALLEE into CALLER based on target-specific info.
14862 Make sure that the caller and callee have compatible architectural
14863 features. Then go through the other possible target attributes
14864 and see if they can block inlining. Try not to reject always_inline
14865 callees unless they are incompatible architecturally. */
14867 static bool
14869 {
14881 /* Callee's ISA flags should be a subset of the caller's. */
14886 /* Allow non-strict aligned functions inlining into strict
14887 aligned ones. */
14897 /* If the architectural features match up and the callee is always_inline
14898 then the other attributes don't matter. */
14910 /* Honour explicit requests to workaround errata. */
14923 /* If the user explicitly specified -momit-leaf-frame-pointer for the
14924 caller and calle and they don't match up, reject inlining. */
14931 /* If the callee has specific tuning overrides, respect them. */
14936 /* If the user specified tuning override strings for the
14937 caller and callee and they don't match up, reject inlining.
14938 We just do a string compare here, we don't analyze the meaning
14939 of the string, as it would be too costly for little gain. */
14947 }
14949 /* Return the ID of the TLDESC ABI, initializing the descriptor if hasn't
14950 been already. */
14952 unsigned int
14954 {
14957 {
14958 HARD_REG_SET full_reg_clobbers;
14965 }
14967 }
14969 /* Return true if SYMBOL_REF X binds locally. */
14971 static bool
14973 {
14977 }
14979 /* Return true if SYMBOL_REF X is thread local */
14980 static bool
14982 {
14990 }
14992 /* Classify a TLS symbol into one of the TLS kinds. */
14993 enum aarch64_symbol_type
14995 {
14999 {
15006 {
15012 }
15023 else
15032 }
15033 }
15035 /* Return the correct method for accessing X + OFFSET, where X is either
15036 a SYMBOL_REF or LABEL_REF. */
15038 enum aarch64_symbol_type
15040 {
15042 {
15044 {
15059 }
15060 }
15063 {
15068 {
15070 /* When we retrieve symbol + offset address, we have to make sure
15071 the offset does not cause overflow of the final address. But
15072 we have no way of knowing the address of symbol at compile time
15073 so we can't accurately say if the distance between the PC and
15074 symbol + offset is outside the addressible range of +/-1MB in the
15075 TINY code model. So we limit the maximum offset to +/-64KB and
15076 assume the offset to the symbol is not larger than +/-(1MB - 64KB).
15077 If offset_within_block_p is true we allow larger offsets.
15078 Furthermore force to memory if the symbol is a weak reference to
15079 something that doesn't resolve to a symbol in this module. */
15090 /* Same reasoning as the tiny code model, but the offset cap here is
15091 1MB, allowing +/-3.9GB for the offset to the symbol. */
15114 /* This is alright even in PIC code as the constant
15115 pool reference is always PC relative and within
15116 the same translation unit. */
15119 else
15124 }
15125 }
15127 /* By default push everything into the constant pool. */
15129 }
15131 bool
15133 {
15135 }
15137 bool
15139 {
15147 }
15149 /* Implement TARGET_LEGITIMATE_CONSTANT_P hook. Return true for constants
15150 that should be rematerialized rather than spilled. */
15152 static bool
15154 {
15155 /* Support CSE and rematerialization of common constants. */
15161 /* Do not allow vector struct mode constants for Advanced SIMD.
15162 We could support 0 and -1 easily, but they need support in
15163 aarch64-simd.md. */
15168 /* Only accept variable-length vector constants if they can be
15169 handled directly.
15171 ??? It would be possible to handle rematerialization of other
15172 constants via secondary reloads. */
15179 /* Accept polynomial constants that can be calculated by using the
15180 destination of a move as the sole temporary. Constants that
15181 require a second temporary cannot be rematerialized (they can't be
15182 forced to memory and also aren't legitimate constants). */
15183 poly_int64 offset;
15187 /* If an offset is being added to something else, we need to allow the
15188 base to be moved into the destination register, meaning that there
15189 are no free temporaries for the offset. */
15194 /* Do not allow const (plus (anchor_symbol, const_int)). */
15198 /* Treat symbols as constants. Avoid TLS symbols as they are complex,
15199 so spilling them is better than rematerialization. */
15203 /* Label references are always constant. */
15208 }
15210 rtx
15212 {
15218 /* Can return in any reg. */
15221 }
15223 /* On AAPCS systems, this is the "struct __va_list". */
15226 /* Implement TARGET_BUILD_BUILTIN_VA_LIST.
15227 Return the type to use as __builtin_va_list.
15229 AAPCS64 \S 7.1.4 requires that va_list be a typedef for a type defined as:
15231 struct __va_list
15232 {
15233 void *__stack;
15234 void *__gr_top;
15235 void *__vr_top;
15236 int __gr_offs;
15237 int __vr_offs;
15238 }; */
15240 static tree
15242 {
15243 tree va_list_name;
15246 /* Create the type. */
15248 /* Give it the required name. */
15250 TYPE_DECL,
15252 va_list_type);
15257 /* Create the fields. */
15260 ptr_type_node);
15263 ptr_type_node);
15266 ptr_type_node);
15269 integer_type_node);
15272 integer_type_node);
15274 /* Tell tree-stdarg pass about our internal offset fields.
15275 NOTE: va_list_gpr/fpr_counter_field are only used for tree comparision
15276 purpose to identify whether the code is updating va_list internal
15277 offset fields through irregular way. */
15299 /* Compute its layout. */
15303 }
15305 /* Implement TARGET_EXPAND_BUILTIN_VA_START. */
15306 static void
15308 {
15312 tree t;
15326 {
15329 }
15338 NULL_TREE);
15340 NULL_TREE);
15342 NULL_TREE);
15344 NULL_TREE);
15346 NULL_TREE);
15348 /* Emit code to initialize STACK, which points to the next varargs stack
15349 argument. CUM->AAPCS_STACK_SIZE gives the number of stack words used
15350 by named arguments. STACK is 8-byte aligned. */
15357 /* Emit code to initialize GRTOP, the top of the GR save area.
15358 virtual_incoming_args_rtx should have been 16 byte aligned. */
15363 /* Emit code to initialize VRTOP, the top of the VR save area.
15364 This address is gr_save_area_bytes below GRTOP, rounded
15365 down to the next 16-byte boundary. */
15375 /* Emit code to initialize GROFF, the offset from GRTOP of the
15376 next GPR argument. */
15381 /* Likewise emit code to initialize VROFF, the offset from FTOP
15382 of the next VR argument. */
15386 }
15388 /* Implement TARGET_GIMPLIFY_VA_ARG_EXPR. */
15390 static tree
15393 {
15394 tree addr;
15400 machine_mode mode;
15424 align
15430 type,
15434 {
15435 /* No frontends can create types with variable-sized modes, so we
15436 shouldn't be asked to pass or return them. */
15439 /* TYPE passed in fp/simd registers. */
15451 {
15454 }
15457 {
15459 }
15460 }
15461 else
15462 {
15463 /* TYPE passed in general registers. */
15472 {
15477 }
15481 {
15483 }
15484 }
15486 /* Get a local temporary for the field value. */
15489 /* Emit code to branch if off >= 0. */
15495 {
15496 /* Emit: offs = (offs + 15) & -16. */
15502 }
15503 else
15506 /* Update ap.__[g|v]r_offs */
15511 /* String up. */
15515 /* [cond2] if (ap.__[g|v]r_offs > 0) */
15520 /* String up: make sure the assignment happens before the use. */
15524 /* Prepare the trees handling the argument that is passed on the stack;
15525 the top level node will store in ON_STACK. */
15528 {
15529 /* if (alignof(type) > 8) (arg = arg + 15) & -16; */
15534 }
15535 else
15537 /* Advance ap.__stack */
15542 /* String up roundup and advance. */
15545 /* String up with arg */
15547 /* Big-endianness related address adjustment. */
15550 {
15554 }
15559 /* Adjustment to OFFSET in the case of BIG_ENDIAN. */
15569 {
15570 /* type ha; // treat as "struct {ftype field[n];}"
15571 ... [computing offs]
15572 for (i = 0; i <nregs; ++i, offs += 16)
15573 ha.field[i] = *((ftype *)(ap.__vr_top + offs));
15574 return ha; */
15578 /* Declare a local variable. */
15582 /* Establish the base type. */
15584 {
15607 {
15611 }
15615 }
15617 /* *(field_ptr_t)&ha = *((field_ptr_t)vr_saved_area */
15625 /* ha.field[i] = *((field_ptr_t)vr_saved_area + i) */
15627 {
15637 }
15641 }
15651 }
15653 /* Implement TARGET_SETUP_INCOMING_VARARGS. */
15655 static void
15659 {
15661 CUMULATIVE_ARGS local_cum;
15665 /* The caller has advanced CUM up to, but not beyond, the last named
15666 argument. Advance a local copy of CUM past the last "real" named
15667 argument, to find out how many registers are left over. */
15671 /* Found out how many registers we need to save.
15672 Honor tree-stdvar analysis results. */
15681 {
15684 }
15687 {
15689 {
15692 /* virtual_incoming_args_rtx should have been 16-byte aligned. */
15700 }
15702 {
15703 /* We can't use move_block_from_reg, because it will use
15704 the wrong mode, storing D regs only. */
15708 /* Set OFF to the offset from virtual_incoming_args_rtx of
15709 the first vector register. The VR save area lies below
15710 the GR one, and is aligned to 16 bytes. */
15717 {
15725 }
15726 }
15727 }
15729 /* We don't save the size into *PRETEND_SIZE because we want to avoid
15730 any complication of having crtl->args.pretend_args_size changed. */
15735 }
15737 static void
15739 {
15742 {
15744 {
15747 }
15748 }
15751 {
15754 }
15756 /* Only allow the FFR and FFRT to be accessed via special patterns. */
15760 /* When tracking speculation, we need a couple of call-clobbered registers
15761 to track the speculation state. It would be nice to just use
15762 IP0 and IP1, but currently there are numerous places that just
15763 assume these registers are free for other uses (eg pointer
15764 authentication). */
15766 {
15771 }
15772 }
15774 /* Walk down the type tree of TYPE counting consecutive base elements.
15775 If *MODEP is VOIDmode, then set it to the first valid floating point
15776 type. If a non-floating point type is found, or if a floating point
15777 type that doesn't match a non-VOIDmode *MODEP is found, then return -1,
15778 otherwise return the count in the sub-tree. */
15779 static int
15781 {
15782 machine_mode mode;
15783 HOST_WIDE_INT size;
15785 /* SVE types (and types containing SVE types) must be handled
15786 before calling this function. */
15790 {
15820 /* Use V2SImode and V4SImode as representatives of all 64-bit
15821 and 128-bit vector types. */
15824 {
15833 }
15838 /* Vector modes are considered to be opaque: two vectors are
15839 equivalent for the purposes of being homogeneous aggregates
15840 if they are the same size. */
15847 {
15851 /* Can't handle incomplete types nor sizes that are not
15852 fixed. */
15859 || !index
15870 /* There must be no padding. */
15876 }
15879 {
15882 tree field;
15884 /* Can't handle incomplete types nor sizes that are not
15885 fixed. */
15891 {
15899 }
15901 /* There must be no padding. */
15907 }
15911 {
15912 /* These aren't very interesting except in a degenerate case. */
15915 tree field;
15917 /* Can't handle incomplete types nor sizes that are not
15918 fixed. */
15924 {
15932 }
15934 /* There must be no padding. */
15940 }
15944 }
15947 }
15949 /* Return TRUE if the type, as described by TYPE and MODE, is a short vector
15950 type as described in AAPCS64 \S 4.1.2.
15952 See the comment above aarch64_composite_type_p for the notes on MODE. */
15954 static bool
15956 machine_mode mode)
15957 {
15970 }
15972 /* Return TRUE if the type, as described by TYPE and MODE, is a composite
15973 type as described in AAPCS64 \S 4.3. This includes aggregate, union and
15974 array types. The C99 floating-point complex types are also considered
15975 as composite types, according to AAPCS64 \S 7.1.1. The complex integer
15976 types, which are GCC extensions and out of the scope of AAPCS64, are
15977 treated as composite types here as well.
15979 Note that MODE itself is not sufficient in determining whether a type
15980 is such a composite type or not. This is because
15981 stor-layout.c:compute_record_mode may have already changed the MODE
15982 (BLKmode) of a RECORD_TYPE TYPE to some other mode. For example, a
15983 structure with only one field may have its MODE set to the mode of the
15984 field. Also an integer mode whose size matches the size of the
15985 RECORD_TYPE type may be used to substitute the original mode
15986 (i.e. BLKmode) in certain circumstances. In other words, MODE cannot be
15987 solely relied on. */
15989 static bool
15991 machine_mode mode)
15992 {
16005 }
16007 /* Return TRUE if an argument, whose type is described by TYPE and MODE,
16008 shall be passed or returned in simd/fp register(s) (providing these
16009 parameter passing registers are available).
16011 Upon successful return, *COUNT returns the number of needed registers,
16012 *BASE_MODE returns the mode of the individual register and when IS_HAF
16013 is not NULL, *IS_HA indicates whether or not the argument is a homogeneous
16014 floating-point aggregate or a homogeneous short-vector aggregate. */
16016 static bool
16018 const_tree type,
16022 {
16033 {
16036 }
16038 {
16042 }
16044 {
16048 {
16051 }
16052 else
16054 }
16055 else
16060 }
16062 /* Implement TARGET_STRUCT_VALUE_RTX. */
16064 static rtx
16067 {
16069 }
16071 /* Implements target hook vector_mode_supported_p. */
16072 static bool
16074 {
16077 }
16079 /* Return the full-width SVE vector mode for element mode MODE, if one
16080 exists. */
16081 opt_machine_mode
16083 {
16085 {
16102 }
16103 }
16105 /* Return the 128-bit Advanced SIMD vector mode for element mode MODE,
16106 if it exists. */
16107 opt_machine_mode
16109 {
16111 {
16130 }
16131 }
16133 /* Return appropriate SIMD container
16134 for MODE within a vector of WIDTH bits. */
16135 static machine_mode
16137 {
16145 {
16148 else
16150 {
16165 }
16166 }
16168 }
16170 /* Return 128-bit container as the preferred SIMD mode for MODE. */
16171 static machine_mode
16173 {
16176 }
16178 /* Return a list of possible vector sizes for the vectorizer
16179 to iterate over. */
16180 static unsigned int
16182 {
16184 /* Try using full vectors for all element types. */
16185 VNx16QImode,
16187 /* Try using 16-bit containers for 8-bit elements and full vectors
16188 for wider elements. */
16189 VNx8QImode,
16191 /* Try using 32-bit containers for 8-bit and 16-bit elements and
16192 full vectors for wider elements. */
16193 VNx4QImode,
16195 /* Try using 64-bit containers for all element types. */
16196 VNx2QImode
16197 };
16200 /* Try using 128-bit vectors for all element types. */
16201 V16QImode,
16203 /* Try using 64-bit vectors for 8-bit elements and 128-bit vectors
16204 for wider elements. */
16205 V8QImode,
16207 /* Try using 64-bit vectors for 16-bit elements and 128-bit vectors
16208 for wider elements.
16210 TODO: We could support a limited form of V4QImode too, so that
16211 we use 32-bit vectors for 8-bit elements. */
16212 V4HImode,
16214 /* Try using 64-bit vectors for 32-bit elements and 128-bit vectors
16215 for 64-bit elements.
16217 TODO: We could similarly support limited forms of V2QImode and V2HImode
16218 for this case. */
16219 V2SImode
16220 };
16222 /* Try using N-byte SVE modes only after trying N-byte Advanced SIMD mode.
16223 This is because:
16225 - If we can't use N-byte Advanced SIMD vectors then the placement
16226 doesn't matter; we'll just continue as though the Advanced SIMD
16227 entry didn't exist.
16229 - If an SVE main loop with N bytes ends up being cheaper than an
16230 Advanced SIMD main loop with N bytes then by default we'll replace
16231 the Advanced SIMD version with the SVE one.
16233 - If an Advanced SIMD main loop with N bytes ends up being cheaper
16234 than an SVE main loop with N bytes then by default we'll try to
16235 use the SVE loop to vectorize the epilogue instead. */
16239 {
16244 else
16246 }
16251 /* Consider enabling VECT_COMPARE_COSTS for SVE, both so that we
16252 can compare SVE against Advanced SIMD and so that we can compare
16253 multiple SVE vectorization approaches against each other. There's
16254 not really any point doing this for Advanced SIMD only, since the
16255 first mode that works should always be the best. */
16259 }
16261 /* Implement TARGET_MANGLE_TYPE. */
16265 {
16266 /* The AArch64 ABI documents say that "__va_list" has to be
16267 mangled as if it is in the "std" namespace. */
16271 /* Half-precision floating point types. */
16273 {
16276 else
16278 }
16280 /* Mangle AArch64-specific internal types. TYPE_NAME is non-NULL_TREE for
16281 builtin types. */
16283 {
16288 }
16290 /* Use the default mangling. */
16292 }
16294 /* Implement TARGET_VERIFY_TYPE_CONTEXT. */
16296 static bool
16299 {
16301 }
16303 /* Find the first rtx_insn before insn that will generate an assembly
16304 instruction. */
16308 {
16312 do
16313 {
16315 }
16319 }
16321 static bool
16323 {
16325 /* A number of these may be AArch32 only. */
16330 };
16333 {
16336 }
16339 }
16341 /* Check if there is a register dependency between a load and the insn
16342 for which we hold recog_data. */
16344 static bool
16346 {
16347 rtx load_reg;
16357 {
16363 }
16365 }
16368 /* When working around the Cortex-A53 erratum 835769,
16369 given rtx_insn INSN, return true if it is a 64-bit multiply-accumulate
16370 instruction and has a preceding memory instruction such that a NOP
16371 should be inserted between them. */
16373 bool
16375 {
16378 rtx body;
16391 /* aarch64_prev_real_insn can call recog_memoized on insns other than INSN.
16392 Restore recog state to INSN to avoid state corruption. */
16400 /* If the previous insn is a memory op and there is no dependency between
16401 it and the DImode madd, emit a NOP between them. If body is NULL then we
16402 have a complex memory operation, probably a load/store pair.
16403 Be conservative for now and emit a NOP. */
16410 }
16413 /* Implement FINAL_PRESCAN_INSN. */
16415 void
16417 {
16420 }
16423 /* Return true if BASE_OR_STEP is a valid immediate operand for an SVE INDEX
16424 instruction. */
16426 bool
16428 {
16431 }
16433 /* Return true if X is a valid immediate for the SVE ADD and SUB instructions
16434 when applied to mode MODE. Negate X first if NEGATE_P is true. */
16436 bool
16438 {
16451 }
16453 /* Return true if X is a valid immediate for the SVE SQADD and SQSUB
16454 instructions when applied to mode MODE. Negate X first if NEGATE_P
16455 is true. */
16457 bool
16459 {
16463 /* After the optional negation, the immediate must be nonnegative.
16464 E.g. a saturating add of -127 must be done via SQSUB Zn.B, Zn.B, #127
16465 instead of SQADD Zn.B, Zn.B, #129. */
16468 }
16470 /* Return true if X is a valid immediate operand for an SVE logical
16471 instruction such as AND. */
16473 bool
16475 {
16476 rtx elt;
16482 }
16484 /* Return true if X is a valid immediate for the SVE DUP and CPY
16485 instructions. */
16487 bool
16489 {
16498 }
16500 /* Return true if X is a valid immediate operand for an SVE CMP instruction.
16501 SIGNED_P says whether the operand is signed rather than unsigned. */
16503 bool
16505 {
16508 && (signed_p
16511 }
16513 /* Return true if X is a valid immediate operand for an SVE FADD or FSUB
16514 instruction. Negate X first if NEGATE_P is true. */
16516 bool
16518 {
16519 rtx elt;
16520 REAL_VALUE_TYPE r;
16536 }
16538 /* Return true if X is a valid immediate operand for an SVE FMUL
16539 instruction. */
16541 bool
16543 {
16544 rtx elt;
16550 }
16552 /* Return true if replicating VAL32 is a valid 2-byte or 4-byte immediate
16553 for the Advanced SIMD operation described by WHICH and INSN. If INFO
16554 is nonnull, use it to describe valid immediates. */
16555 static bool
16560 {
16561 /* Try a 4-byte immediate with LSL. */
16564 {
16569 }
16571 /* Try a 2-byte immediate with LSL. */
16576 {
16581 }
16583 /* Try a 4-byte immediate with MSL, except for cases that MVN
16584 can handle. */
16587 {
16590 {
16595 }
16596 }
16599 }
16601 /* Return true if replicating VAL64 is a valid immediate for the
16602 Advanced SIMD operation described by WHICH. If INFO is nonnull,
16603 use it to describe valid immediates. */
16604 static bool
16608 {
16614 {
16625 /* Try using a replicated byte. */
16629 {
16633 }
16634 }
16636 /* Try using a bit-to-bytemask. */
16638 {
16641 {
16645 }
16647 {
16651 }
16652 }
16654 }
16656 /* Return true if replicating VAL64 gives a valid immediate for an SVE MOV
16657 instruction. If INFO is nonnull, use it to describe valid immediates. */
16659 static bool
16662 {
16666 {
16670 {
16675 }
16676 }
16679 {
16680 /* DUP with no shift. */
16684 }
16686 {
16687 /* DUP with LSL #8. */
16691 }
16693 {
16694 /* DUPM. */
16698 }
16700 }
16702 /* Return true if X is an UNSPEC_PTRUE constant of the form:
16704 (const (unspec [PATTERN ZERO] UNSPEC_PTRUE))
16706 where PATTERN is the svpattern as a CONST_INT and where ZERO
16707 is a zero constant of the required PTRUE mode (which can have
16708 fewer elements than X's mode, if zero bits are significant).
16710 If so, and if INFO is nonnull, describe the immediate in INFO. */
16711 bool
16713 {
16722 {
16723 aarch64_svpattern pattern
16728 }
16730 }
16732 /* Return true if X is a valid SVE predicate. If INFO is nonnull, use
16733 it to describe valid immediates. */
16735 static bool
16737 {
16742 {
16746 }
16748 /* Analyze the value as a VNx16BImode. This should be relatively
16749 efficient, since rtx_vector_builder has enough built-in capacity
16750 to store all VLA predicate constants without needing the heap. */
16751 rtx_vector_builder builder;
16757 {
16761 {
16763 {
16766 }
16768 }
16769 }
16771 }
16773 /* Return true if OP is a valid SIMD immediate for the operation
16774 described by WHICH. If INFO is nonnull, use it to describe valid
16775 immediates. */
16776 bool
16779 {
16796 {
16803 {
16804 /* Get the corresponding container mode. E.g. an INDEX on V2SI
16805 should yield two integer values per 128-bit block, meaning
16806 that we need to treat it in the same way as V2DI and then
16807 ignore the upper 32 bits of each element. */
16810 }
16812 }
16816 else
16819 scalar_float_mode elt_float_mode;
16822 {
16826 {
16830 }
16831 }
16833 /* If all elements in an SVE vector have the same value, we have a free
16834 choice between using the element mode and using the container mode.
16835 Using the element mode means that unused parts of the vector are
16836 duplicates of the used elements, while using the container mode means
16837 that the unused parts are an extension of the used elements. Using the
16838 element mode is better for (say) VNx4HI 0x101, since 0x01010101 is valid
16839 for its container mode VNx4SI while 0x00000101 isn't.
16841 If not all elements in an SVE vector have the same value, we need the
16842 transition from one element to the next to occur at container boundaries.
16843 E.g. a fixed-length VNx4HI containing { 1, 2, 3, 4 } should be treated
16844 in the same way as a VNx4SI containing { 1, 2, 3, 4 }. */
16845 scalar_int_mode elt_int_mode;
16848 else
16855 /* Expand the vector constant out into a byte vector, with the least
16856 significant byte of the register first. */
16860 {
16861 /* The vector is provided in gcc endian-neutral fashion.
16862 For aarch64_be Advanced SIMD, it must be laid out in the vector
16863 register in reverse order. */
16875 {
16878 }
16879 }
16881 /* The immediate must repeat every eight bytes. */
16887 /* Get the repeating 8-byte value as an integer. No endian correction
16888 is needed here because bytes is already in lsb-first order. */
16896 else
16898 }
16900 /* Check whether X is a VEC_SERIES-like constant that starts at 0 and
16901 has a step in the range of INDEX. Return the index expression if so,
16902 otherwise return null. */
16903 rtx
16905 {
16912 }
16914 /* Check of immediate shift constants are within range. */
16915 bool
16917 {
16924 else
16926 }
16928 /* Return the bitmask CONST_INT to select the bits required by a zero extract
16929 operation of width WIDTH at bit position POS. */
16931 rtx
16933 {
16937 unsigned HOST_WIDE_INT mask
16940 }
16942 bool
16944 {
16953 {
16954 /* Require predicate constants to be VNx16BI before RA, so that we
16955 force everything to have a canonical form. */
16957 && !reload_completed
16963 }
16973 }
16975 /* Return a const_int vector of VAL. */
16976 rtx
16978 {
16981 }
16983 /* Check OP is a legal scalar immediate for the MOVI instruction. */
16985 bool
16987 {
16988 machine_mode vmode;
16993 }
16995 /* Construct and return a PARALLEL RTX vector with elements numbering the
16996 lanes of either the high (HIGH == TRUE) or low (HIGH == FALSE) half of
16997 the vector - from the perspective of the architecture. This does not
16998 line up with GCC's perspective on lane numbers, so we end up with
16999 different masks depending on our target endian-ness. The diagram
17000 below may help. We must draw the distinction when building masks
17001 which select one half of the vector. An instruction selecting
17002 architectural low-lanes for a big-endian target, must be described using
17003 a mask selecting GCC high-lanes.
17005 Big-Endian Little-Endian
17007 GCC 0 1 2 3 3 2 1 0
17008 | x | x | x | x | | x | x | x | x |
17009 Architecture 3 2 1 0 3 2 1 0
17011 Low Mask: { 2, 3 } { 0, 1 }
17012 High Mask: { 0, 1 } { 2, 3 }
17014 MODE Is the mode of the vector and NUNITS is the number of units in it. */
17016 rtx
17018 {
17023 rtx t1;
17028 else
17036 }
17038 /* Check OP for validity as a PARALLEL RTX vector with elements
17039 numbering the lanes of either the high (HIGH == TRUE) or low lanes,
17040 from the perspective of the architecture. See the diagram above
17041 aarch64_simd_vect_par_cnst_half for more details. */
17043 bool
17046 {
17060 {
17067 }
17069 }
17071 /* Return a PARALLEL containing NELTS elements, with element I equal
17072 to BASE + I * STEP. */
17074 rtx
17076 {
17081 }
17083 /* Return true if OP is a PARALLEL of CONST_INTs that form a linear
17084 series with step STEP. */
17086 bool
17088 {
17099 }
17101 /* Bounds-check lanes. Ensure OPERAND lies between LOW (inclusive) and
17102 HIGH (exclusive). */
17103 void
17105 const_tree exp)
17106 {
17107 HOST_WIDE_INT lane;
17112 {
17115 else
17117 }
17118 }
17120 /* Peform endian correction on lane number N, which indexes a vector
17121 of mode MODE, and return the result as an SImode rtx. */
17123 rtx
17125 {
17127 }
17129 /* Return TRUE if OP is a valid vector addressing mode. */
17131 bool
17133 {
17136 }
17138 /* Return true if OP is a valid MEM operand for an SVE LD1R instruction. */
17140 bool
17142 {
17144 scalar_mode mode;
17151 }
17153 /* Return true if OP is a valid MEM operand for an SVE LD1R{Q,O} instruction
17154 where the size of the read data is specified by `mode` and the size of the
17155 vector elements are specified by `elem_mode`. */
17156 bool
17158 scalar_mode elem_mode)
17159 {
17172 }
17174 /* Return true if OP is a valid MEM operand for an SVE LD1RQ instruction. */
17175 bool
17177 {
17180 }
17182 /* Return true if OP is a valid MEM operand for an SVE LD1RO instruction for
17183 accessing a vector where the element size is specified by `elem_mode`. */
17184 bool
17186 {
17188 }
17190 /* Return true if OP is a valid MEM operand for an SVE LDFF1 instruction. */
17191 bool
17193 {
17205 }
17207 /* Return true if OP is a valid MEM operand for an SVE LDNF1 instruction. */
17208 bool
17210 {
17217 }
17219 /* Return true if OP is a valid MEM operand for an SVE LDR instruction.
17220 The conditions for STR are the same. */
17221 bool
17223 {
17230 }
17232 /* Return true if OP is a valid address for an SVE PRF[BHWD] instruction,
17233 addressing memory of mode MODE. */
17234 bool
17236 {
17245 }
17247 /* Return true if OP is a valid MEM operand for an SVE_STRUCT mode.
17248 We need to be able to access the individual pieces, so the range
17249 is different from LD[234] and ST[234]. */
17250 bool
17252 {
17259 ADDR_QUERY_ANY)
17267 }
17269 /* Emit a register copy from operand to operand, taking care not to
17270 early-clobber source registers in the process.
17272 COUNT is the number of components into which the copy needs to be
17273 decomposed. */
17274 void
17277 {
17287 else
17291 }
17293 /* Compute and return the length of aarch64_simd_reglist<mode>, where <mode> is
17294 one of VSTRUCT modes: OI, CI, or XI. */
17295 int
17297 {
17298 /* This is only used (and only meaningful) for Advanced SIMD, not SVE. */
17300 }
17302 /* Implement target hook TARGET_VECTOR_ALIGNMENT. The AAPCS64 sets the maximum
17303 alignment of a vector to 128 bits. SVE predicates have an alignment of
17304 16 bits. */
17305 static HOST_WIDE_INT
17307 {
17308 /* ??? Checking the mode isn't ideal, but VECTOR_BOOLEAN_TYPE_P can
17309 be set for non-predicate vectors of booleans. Modes are the most
17310 direct way we have of identifying real SVE predicate types. */
17313 widest_int min_size
17316 }
17318 /* Implement target hook TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT. */
17319 static poly_uint64
17321 {
17323 {
17324 /* If the length of the vector is fixed, try to align to that length,
17325 otherwise don't try to align at all. */
17326 HOST_WIDE_INT result;
17330 }
17332 }
17334 /* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE. */
17335 static bool
17337 {
17341 /* For fixed-length vectors, check that the vectorizer will aim for
17342 full-vector alignment. This isn't true for generic GCC vectors
17343 that are wider than the ABI maximum of 128 bits. */
17344 poly_uint64 preferred_alignment =
17348 preferred_alignment))
17351 /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned. */
17353 }
17355 /* Return true if the vector misalignment factor is supported by the
17356 target. */
17357 static bool
17361 {
17363 {
17364 /* Return if movmisalign pattern is not supported for this mode. */
17368 /* Misalignment factor is unknown at compile time. */
17371 }
17373 is_packed);
17374 }
17376 /* If VALS is a vector constant that can be loaded into a register
17377 using DUP, generate instructions to do so and return an RTX to
17378 assign to the register. Otherwise return NULL_RTX. */
17379 static rtx
17381 {
17384 rtx x;
17389 /* We can load this constant by using DUP and a constant in a
17390 single ARM register. This will be cheaper than a vector
17391 load. */
17394 }
17397 /* Generate code to load VALS, which is a PARALLEL containing only
17398 constants (for vec_init) or CONST_VECTOR, efficiently into a
17399 register. Returns an RTX to copy into the register, or NULL_RTX
17400 for a PARALLEL that cannot be converted into a CONST_VECTOR. */
17401 static rtx
17403 {
17405 rtx const_dup;
17413 {
17414 /* A CONST_VECTOR must contain only CONST_INTs and
17415 CONST_DOUBLEs, but CONSTANT_P allows more (e.g. SYMBOL_REF).
17416 Only store valid constants in a CONST_VECTOR. */
17419 {
17422 n_const++;
17423 }
17426 }
17427 else
17432 /* Load using MOVI/MVNI. */
17435 /* Loaded using DUP. */
17438 /* Load from constant pool. We cannot take advantage of single-cycle
17439 LD1 because we need a PC-relative addressing mode. */
17441 else
17442 /* A PARALLEL containing something not valid inside CONST_VECTOR.
17443 We cannot construct an initializer. */
17445 }
17447 /* Expand a vector initialisation sequence, such that TARGET is
17448 initialised to contain VALS. */
17450 void
17452 {
17455 /* The number of vector elements. */
17457 /* The number of vector elements which are not constant. */
17460 /* The first element of vals. */
17464 /* This is a special vec_init<M><N> where N is not an element mode but a
17465 vector mode with half the elements of M. We expect to find two entries
17466 of mode N in VALS and we must put their concatentation into TARGET. */
17468 {
17477 /* When we want to concatenate a half-width vector with zeroes we can
17478 use the aarch64_combinez[_be] patterns. Just make sure that the
17479 zeroes are in the right half. */
17488 else
17489 {
17490 /* Else create the two half-width registers and combine them. */
17499 }
17501 }
17503 /* Count the number of variable elements to initialise. */
17505 {
17509 else
17513 }
17515 /* No variable elements, hand off to aarch64_simd_make_constant which knows
17516 how best to handle this. */
17518 {
17521 {
17524 }
17525 }
17527 /* Splat a single non-constant element if we can. */
17529 {
17533 }
17538 /* If there are only variable elements, try to optimize
17539 the insertion using dup for the most common element
17540 followed by insertions. */
17542 /* The algorithm will fill matches[*][0] with the earliest matching element,
17543 and matches[X][1] with the count of duplicate elements (if X is the
17544 earliest element which has duplicates). */
17547 {
17550 {
17552 {
17554 {
17558 }
17559 }
17560 }
17565 {
17568 }
17570 /* Create a duplicate of the most common element, unless all elements
17571 are equally useless to us, in which case just immediately set the
17572 vector register using the first element. */
17575 {
17576 /* For vectors of two 64-bit elements, we can do even better. */
17581 {
17584 /* Combine can pick up this case, but handling it directly
17585 here leaves clearer RTL.
17587 This is load_pair_lanes<mode>, and also gives us a clean-up
17588 for store_pair_lanes<mode>. */
17591 && !STRICT_ALIGNMENT
17596 {
17597 rtx t;
17600 else
17604 }
17605 }
17606 /* The subreg-move sequence below will move into lane zero of the
17607 vector register. For big-endian we want that position to hold
17608 the last element of VALS. */
17612 }
17613 else
17614 {
17617 }
17619 /* Insert the rest. */
17621 {
17627 }
17629 }
17631 /* Initialise a vector which is part-variable. We want to first try
17632 to build those lanes which are constant in the most efficient way we
17633 can. */
17635 {
17638 /* Load constant part of vector. We really don't care what goes into the
17639 parts we will overwrite, but we're more likely to be able to load the
17640 constant efficiently if it has fewer, larger, repeating parts
17641 (see aarch64_simd_valid_immediate). */
17643 {
17649 {
17650 /* Look in the copied vector, as more elements are const. */
17653 {
17656 }
17657 }
17659 }
17661 }
17663 /* Insert the variable lanes directly. */
17665 {
17671 }
17672 }
17674 /* Emit RTL corresponding to:
17675 insr TARGET, ELEM. */
17677 static void
17679 {
17687 }
17689 /* Subroutine of aarch64_sve_expand_vector_init for handling
17690 trailing constants.
17691 This function works as follows:
17692 (a) Create a new vector consisting of trailing constants.
17693 (b) Initialize TARGET with the constant vector using emit_move_insn.
17694 (c) Insert remaining elements in TARGET using insr.
17695 NELTS is the total number of elements in original vector while
17696 while NELTS_REQD is the number of elements that are actually
17697 significant.
17699 ??? The heuristic used is to do above only if number of constants
17700 is at least half the total number of elements. May need fine tuning. */
17702 static bool
17703 aarch64_sve_expand_vector_init_handle_trailing_constants
17705 {
17712 i--)
17713 n_trailing_constants++;
17716 {
17727 }
17730 }
17732 /* Subroutine of aarch64_sve_expand_vector_init.
17733 Works as follows:
17734 (a) Initialize TARGET by broadcasting element NELTS_REQD - 1 of BUILDER.
17735 (b) Skip trailing elements from BUILDER, which are the same as
17736 element NELTS_REQD - 1.
17737 (c) Insert earlier elements in reverse order in TARGET using insr. */
17739 static void
17743 {
17758 }
17760 /* Subroutine of aarch64_sve_expand_vector_init to handle case
17761 when all trailing elements of builder are same.
17762 This works as follows:
17763 (a) Use expand_insn interface to broadcast last vector element in TARGET.
17764 (b) Insert remaining elements in TARGET using insr.
17766 ??? The heuristic used is to do above if number of same trailing elements
17767 is at least 3/4 of total number of elements, loosely based on
17768 heuristic from mostly_zeros_p. May need fine-tuning. */
17770 static bool
17771 aarch64_sve_expand_vector_init_handle_trailing_same_elem
17773 {
17776 {
17780 }
17783 }
17785 /* Initialize register TARGET from BUILDER. NELTS is the constant number
17786 of elements in BUILDER.
17788 The function tries to initialize TARGET from BUILDER if it fits one
17789 of the special cases outlined below.
17791 Failing that, the function divides BUILDER into two sub-vectors:
17792 v_even = even elements of BUILDER;
17793 v_odd = odd elements of BUILDER;
17795 and recursively calls itself with v_even and v_odd.
17797 if (recursive call succeeded for v_even or v_odd)
17798 TARGET = zip (v_even, v_odd)
17800 The function returns true if it managed to build TARGET from BUILDER
17801 with one of the special cases, false otherwise.
17803 Example: {a, 1, b, 2, c, 3, d, 4}
17805 The vector gets divided into:
17806 v_even = {a, b, c, d}
17807 v_odd = {1, 2, 3, 4}
17809 aarch64_sve_expand_vector_init(v_odd) hits case 1 and
17810 initialize tmp2 from constant vector v_odd using emit_move_insn.
17812 aarch64_sve_expand_vector_init(v_even) fails since v_even contains
17813 4 elements, so we construct tmp1 from v_even using insr:
17814 tmp1 = dup(d)
17815 insr tmp1, c
17816 insr tmp1, b
17817 insr tmp1, a
17819 And finally:
17820 TARGET = zip (tmp1, tmp2)
17821 which sets TARGET to {a, 1, b, 2, c, 3, d, 4}. */
17823 static bool
17826 {
17829 /* Case 1: Vector contains trailing constants. */
17835 /* Case 2: Vector contains leading constants. */
17844 {
17847 }
17849 /* Case 3: Vector contains trailing same element. */
17855 /* Case 4: Vector contains leading same element. */
17859 {
17862 }
17864 /* Avoid recursing below 4-elements.
17865 ??? The threshold 4 may need fine-tuning. */
17874 {
17877 }
17893 /* Initialize v_even and v_odd using INSR if it didn't match any of the
17894 special cases and zip v_even, v_odd. */
17905 }
17907 /* Initialize register TARGET from the elements in PARALLEL rtx VALS. */
17909 void
17911 {
17920 /* If neither sub-vectors of v could be initialized specially,
17921 then use INSR to insert all elements from v into TARGET.
17922 ??? This might not be optimal for vectors with large
17923 initializers like 16-element or above.
17924 For nelts < 4, it probably isn't useful to handle specially. */
17929 }
17931 /* Check whether VALUE is a vector constant in which every element
17932 is either a power of 2 or a negated power of 2. If so, return
17933 a constant vector of log2s, and flip CODE between PLUS and MINUS
17934 if VALUE contains negated powers of 2. Return NULL_RTX otherwise. */
17936 static rtx
17938 {
17942 rtx_vector_builder builder;
17947 /* 1 if the result of the multiplication must be negated,
17948 0 if it mustn't, or -1 if we don't yet care. */
17952 {
17959 {
17960 /* It matters whether we negate or not. Make that choice,
17961 and make sure that it's consistent with previous elements. */
17967 }
17968 /* POW2 is now the value that we want to be a power of 2. */
17973 }
17975 /* PLUS and MINUS are equivalent; canonicalize on PLUS. */
17980 }
17982 /* Prepare for an integer SVE multiply-add or multiply-subtract pattern;
17983 CODE is PLUS for the former and MINUS for the latter. OPERANDS is the
17984 operands array, in the same order as for fma_optab. Return true if
17985 the function emitted all the necessary instructions, false if the caller
17986 should generate the pattern normally with the new OPERANDS array. */
17988 bool
17990 {
17993 {
17998 OPTAB_DIRECT);
18000 }
18003 }
18005 /* Likewise, but for a conditional pattern. */
18007 bool
18009 {
18012 {
18018 }
18021 }
18023 static unsigned HOST_WIDE_INT
18025 {
18029 }
18031 /* Select a format to encode pointers in exception handling data. */
18032 int
18034 {
18037 {
18043 /* text+got+data < 4Gb. 4-byte signed relocs are sufficient
18044 for everything. */
18048 /* No assumptions here. 8-byte relocs required. */
18051 }
18053 }
18055 /* Output .variant_pcs for aarch64_vector_pcs function symbols. */
18057 static void
18059 {
18061 {
18064 {
18068 }
18069 }
18070 }
18072 /* The last .arch and .tune assembly strings that we printed. */
18076 /* Implement ASM_DECLARE_FUNCTION_NAME. Output the ISA features used
18077 by the function fndecl. */
18079 void
18081 tree fndecl)
18082 {
18088 else
18099 /* Only update the assembler .arch string if it is distinct from the last
18100 such string we printed. */
18103 {
18106 }
18108 /* Print the cpu name we're tuning for in the comments, might be
18109 useful to readers of the generated asm. Do it only when it changes
18110 from function to function and verbose assembly is requested. */
18115 {
18119 }
18123 /* Don't forget the type directive for ELF. */
18126 }
18128 /* Implement ASM_OUTPUT_DEF_FROM_DECLS. Output .variant_pcs for aliases. */
18130 void
18132 {
18137 }
18139 /* Implement ASM_OUTPUT_EXTERNAL. Output .variant_pcs for undefined
18140 function symbol references. */
18142 void
18144 {
18147 }
18149 /* Triggered after a .cfi_startproc directive is emitted into the assembly file.
18150 Used to output the .cfi_b_key_frame directive when signing the current
18151 function with the B key. */
18153 void
18155 {
18159 }
18161 /* Implements TARGET_ASM_FILE_START. Output the assembly header. */
18163 static void
18165 {
18182 }
18184 /* Emit load exclusive. */
18186 static void
18189 {
18194 else
18196 }
18198 /* Emit store exclusive. */
18200 static void
18203 {
18208 else
18210 }
18212 /* Mark the previous jump instruction as unlikely. */
18214 static void
18216 {
18219 }
18221 /* We store the names of the various atomic helpers in a 5x4 array.
18222 Return the libcall function given MODE, MODEL and NAMES. */
18224 rtx
18227 {
18232 {
18250 }
18253 {
18270 }
18273 VISIBILITY_HIDDEN);
18274 }
18276 #define DEF0(B, N) \
18282 #define DEF4(B) DEF0(B, 1), DEF0(B, 2), DEF0(B, 4), DEF0(B, 8), \
18283 { NULL, NULL, NULL, NULL }
18284 #define DEF5(B) DEF0(B, 1), DEF0(B, 2), DEF0(B, 4), DEF0(B, 8), DEF0(B, 16)
18293 #undef DEF0
18294 #undef DEF4
18295 #undef DEF5
18297 /* Expand a compare and swap pattern. */
18299 void
18301 {
18315 /* Normally the succ memory model must be stronger than fail, but in the
18316 unlikely event of fail being ACQUIRE and succ being RELEASE we need to
18317 promote succ to ACQ_REL so that we don't lose the acquire semantics. */
18324 {
18327 }
18330 {
18331 /* The CAS insn requires oldval and rval overlap, but we need to
18332 have a copy of oldval saved across the operation to tell if
18333 the operation is successful. */
18336 else
18342 }
18344 {
18345 /* Oldval must satisfy compare afterward. */
18353 }
18354 else
18355 {
18356 /* The oldval predicate varies by mode. Test it and force to reg. */
18364 }
18372 }
18374 /* Emit a barrier, that is appropriate for memory model MODEL, at the end of a
18375 sequence implementing an atomic operation. */
18377 static void
18379 {
18386 {
18388 }
18389 }
18391 /* Split a compare and swap pattern. */
18393 void
18395 {
18396 /* Split after prolog/epilog to avoid interactions with shrinkwrapping. */
18400 machine_mode mode;
18415 /* When OLDVAL is zero and we want the strong version we can emit a tighter
18416 loop:
18417 .label1:
18418 LD[A]XR rval, [mem]
18419 CBNZ rval, .label2
18420 ST[L]XR scratch, newval, [mem]
18421 CBNZ scratch, .label1
18422 .label2:
18423 CMP rval, 0. */
18429 {
18432 }
18435 /* The initial load can be relaxed for a __sync operation since a final
18436 barrier will be emitted to stop code hoisting. */
18439 else
18444 else
18445 {
18448 }
18456 {
18458 {
18459 /* Emit an explicit compare instruction, so that we can correctly
18460 track the condition codes. */
18463 }
18464 else
18470 }
18471 else
18476 /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL
18477 to set the condition flags. If this is not used it will be removed by
18478 later passes. */
18482 /* Emit any final barrier needed for a __sync operation. */
18485 }
18487 /* Split an atomic operation. */
18489 void
18492 {
18493 /* Split after prolog/epilog to avoid interactions with shrinkwrapping. */
18501 rtx x;
18503 /* Split the atomic operation into a sequence. */
18511 else
18515 /* The initial load can be relaxed for a __sync operation since a final
18516 barrier will be emitted to stop code hoisting. */
18520 else
18524 {
18538 {
18541 }
18542 /* Fall through. */
18548 }
18554 {
18555 /* Emit an explicit compare instruction, so that we can correctly
18556 track the condition codes. */
18559 }
18560 else
18567 /* Emit any final barrier needed for a __sync operation. */
18570 }
18572 static void
18574 {
18575 /* Half-precision float operations. The compiler handles all operations
18576 with NULL libfuncs by converting to SFmode. */
18578 /* Conversions. */
18582 /* Arithmetic. */
18589 /* Comparisons. */
18597 }
18599 /* Target hook for c_mode_for_suffix. */
18600 static machine_mode
18602 {
18607 }
18609 /* We can only represent floating point constants which will fit in
18610 "quarter-precision" values. These values are characterised by
18611 a sign bit, a 4-bit mantissa and a 3-bit exponent. And are given
18612 by:
18614 (-1)^s * (n/16) * 2^r
18616 Where:
18617 's' is the sign bit.
18618 'n' is an integer in the range 16 <= n <= 31.
18619 'r' is an integer in the range -3 <= r <= 4. */
18621 /* Return true iff X can be represented by a quarter-precision
18622 floating point immediate operand X. Note, we cannot represent 0.0. */
18623 bool
18625 {
18626 /* This represents our current view of how many bits
18627 make up the mantissa. */
18644 /* We cannot represent infinities, NaNs or +/-zero. We won't
18645 know if we have +zero until we analyse the mantissa, but we
18646 can reject the other invalid values. */
18651 /* Extract exponent. */
18655 /* For the mantissa, we expand into two HOST_WIDE_INTS, apart from the
18656 highest (sign) bit, with a fixed binary point at bit point_pos.
18657 m1 holds the low part of the mantissa, m2 the high part.
18658 WARNING: If we ever have a representation using more than 2 * H_W_I - 1
18659 bits for the mantissa, this can fail (low bits will be lost). */
18663 /* If the low part of the mantissa has bits set we cannot represent
18664 the value. */
18667 /* We have rejected the lower HOST_WIDE_INT, so update our
18668 understanding of how many bits lie in the mantissa and
18669 look only at the high HOST_WIDE_INT. */
18673 /* We can only represent values with a mantissa of the form 1.xxxx. */
18678 /* Having filtered unrepresentable values, we may now remove all
18679 but the highest 5 bits. */
18682 /* We cannot represent the value 0.0, so reject it. This is handled
18683 elsewhere. */
18687 /* Then, as bit 4 is always set, we can mask it off, leaving
18688 the mantissa in the range [0, 15]. */
18692 /* GCC internally does not use IEEE754-like encoding (where normalized
18693 significands are in the range [1, 2). GCC uses [0.5, 1) (see real.c).
18694 Our mantissa values are shifted 4 places to the left relative to
18695 normalized IEEE754 so we must modify the exponent returned by REAL_EXP
18696 by 5 places to correct for GCC's representation. */
18700 }
18702 /* Returns the string with the instruction for AdvSIMD MOVI, MVNI, ORR or BIC
18703 immediate with a CONST_VECTOR of MODE and WIDTH. WHICH selects whether to
18704 output MOVI/MVNI, ORR or BIC immediate. */
18708 {
18718 /* This will return true to show const_vector is legal for use as either
18719 a AdvSIMD MOVI instruction (or, implicitly, MVNI), ORR or BIC immediate.
18720 It will also update INFO to show how the immediate should be generated.
18721 WHICH selects whether to check for MOVI/MVNI, ORR or BIC. */
18729 {
18732 /* For FP zero change it to a CONST_INT 0 and use the integer SIMD
18733 move immediate path. */
18736 else
18737 {
18746 else
18750 }
18751 }
18756 {
18768 else
18772 }
18773 else
18774 {
18775 /* For AARCH64_CHECK_BIC and AARCH64_CHECK_ORR. */
18782 else
18786 }
18788 }
18792 {
18794 /* If a floating point number was passed and we desire to use it in an
18795 integer mode do the conversion to integer. */
18797 {
18802 }
18804 machine_mode vmode;
18805 /* use a 64 bit mode for everything except for DI/DF mode, where we use
18806 a 128 bit vector mode. */
18812 }
18814 /* Return the output string to use for moving immediate CONST_VECTOR
18815 into an SVE register. */
18819 {
18831 {
18834 {
18837 }
18838 else
18839 {
18846 else
18849 }
18851 }
18854 {
18860 }
18863 {
18866 else
18867 {
18877 }
18878 }
18883 }
18885 /* Return the asm template for a PTRUES. CONST_UNSPEC is the
18886 aarch64_sve_ptrue_svpattern_immediate that describes the predicate
18887 pattern. */
18891 {
18902 }
18904 /* Split operands into moves from op[1] + op[2] into op[0]. */
18906 void
18908 {
18919 {
18920 /* No-op move. Can't split to nothing; emit something. */
18923 }
18925 /* Preserve register attributes for variable tracking. */
18930 /* Special case of reversed high/low parts. */
18933 {
18937 }
18939 {
18940 /* Try to avoid unnecessary moves if part of the result
18941 is in the right place already. */
18946 }
18947 else
18948 {
18953 }
18954 }
18956 /* vec_perm support. */
18958 struct expand_vec_perm_d
18959 {
18961 vec_perm_indices perm;
18962 machine_mode vmode;
18966 };
18968 /* Generate a variable permutation. */
18970 static void
18972 {
18983 {
18985 {
18986 /* Expand the argument to a V16QI mode by duplicating it. */
18990 }
18991 else
18992 {
18994 }
18995 }
18996 else
18997 {
18998 rtx pair;
19001 {
19005 }
19006 else
19007 {
19011 }
19012 }
19013 }
19015 /* Expand a vec_perm with the operands given by TARGET, OP0, OP1 and SEL.
19016 NELT is the number of elements in the vector. */
19018 void
19021 {
19024 rtx mask;
19026 /* The TBL instruction does not use a modulo index, so we must take care
19027 of that ourselves. */
19032 /* For big-endian, we also need to reverse the index within the vector
19033 (but not which vector). */
19035 {
19036 /* If one_vector_p, mask is a vector of (nelt - 1)'s already. */
19041 }
19043 }
19045 /* Generate (set TARGET (unspec [OP0 OP1] CODE)). */
19047 static void
19049 {
19053 }
19055 /* Expand an SVE vec_perm with the given operands. */
19057 void
19059 {
19062 /* Enforced by the pattern condition. */
19065 /* Note: vec_perm indices are supposed to wrap when they go beyond the
19066 size of the two value vectors, i.e. the upper bits of the indices
19067 are effectively ignored. SVE TBL instead produces 0 for any
19068 out-of-range indices, so we need to modulo all the vec_perm indices
19069 to ensure they are all in range. */
19072 /* Check if the sel only references the first values vector. */
19075 {
19078 }
19080 /* Check if the two values vectors are the same. */
19082 {
19088 }
19090 /* Run TBL on for each value vector and combine the results. */
19097 {
19102 }
19109 else
19111 }
19113 /* Recognize patterns suitable for the TRN instructions. */
19114 static bool
19116 {
19117 HOST_WIDE_INT odd;
19125 /* Note that these are little-endian tests.
19126 We correct for big-endian later. */
19133 /* Success! */
19139 /* We don't need a big-endian lane correction for SVE; see the comment
19140 at the head of aarch64-sve.md for details. */
19142 {
19145 }
19151 }
19153 /* Recognize patterns suitable for the UZP instructions. */
19154 static bool
19156 {
19157 HOST_WIDE_INT odd;
19164 /* Note that these are little-endian tests.
19165 We correct for big-endian later. */
19171 /* Success! */
19177 /* We don't need a big-endian lane correction for SVE; see the comment
19178 at the head of aarch64-sve.md for details. */
19180 {
19183 }
19189 }
19191 /* Recognize patterns suitable for the ZIP instructions. */
19192 static bool
19194 {
19203 /* Note that these are little-endian tests.
19204 We correct for big-endian later. */
19212 /* Success! */
19218 /* We don't need a big-endian lane correction for SVE; see the comment
19219 at the head of aarch64-sve.md for details. */
19221 {
19224 }
19230 }
19232 /* Recognize patterns for the EXT insn. */
19234 static bool
19236 {
19237 HOST_WIDE_INT location;
19238 rtx offset;
19240 /* The first element always refers to the first vector.
19241 Check if the extracted indices are increasing by one. */
19247 /* Success! */
19251 /* The case where (location == 0) is a no-op for both big- and little-endian,
19252 and is removed by the mid-end at optimization levels -O1 and higher.
19254 We don't need a big-endian lane correction for SVE; see the comment
19255 at the head of aarch64-sve.md for details. */
19257 {
19258 /* After setup, we want the high elements of the first vector (stored
19259 at the LSB end of the register), and the low elements of the second
19260 vector (stored at the MSB end of the register). So swap. */
19262 /* location != 0 (above), so safe to assume (nelt - location) < nelt.
19263 to_constant () is safe since this is restricted to Advanced SIMD
19264 vectors. */
19266 }
19272 UNSPEC_EXT));
19274 }
19276 /* Recognize patterns for the REV{64,32,16} insns, which reverse elements
19277 within each 64-bit, 32-bit or 16-bit granule. */
19279 static bool
19281 {
19282 HOST_WIDE_INT diff;
19284 machine_mode pred_mode;
19293 {
19296 }
19298 {
19301 }
19303 {
19306 }
19307 else
19315 /* Success! */
19320 {
19324 /* The act of taking a subreg between INT_MODE and d->vmode
19325 is itself a reversing operation on big-endian targets;
19326 see the comment at the head of aarch64-sve.md for details.
19327 First reinterpret OP0 as INT_MODE without using a subreg
19328 and without changing the contents. */
19330 else
19331 {
19332 /* For SVE we use REV[BHW] unspecs derived from the element size
19333 of v->mode and vector modes whose elements have SIZE bytes.
19334 This ensures that the vector modes match the predicate modes. */
19339 }
19342 }
19346 }
19348 /* Recognize patterns for the REV insn, which reverses elements within
19349 a full vector. */
19351 static bool
19353 {
19362 /* Success! */
19369 }
19371 static bool
19373 {
19375 rtx in0;
19376 HOST_WIDE_INT elt;
19378 rtx lane;
19388 /* Success! */
19392 /* The generic preparation in aarch64_expand_vec_perm_const_1
19393 swaps the operand order and the permute indices if it finds
19394 d->perm[0] to be in the second operand. Thus, we can always
19395 use d->op0 and need not do any extra arithmetic to get the
19396 correct lane number. */
19404 }
19406 static bool
19408 {
19412 /* Make sure that the indices are constant. */
19421 /* Generic code will try constant permutation twice. Once with the
19422 original mode and again with the elements lowered to QImode.
19423 So wait and don't do the selector expansion ourselves. */
19427 /* to_constant is safe since this routine is specific to Advanced SIMD
19428 vectors. */
19431 /* If big-endian and two vectors we end up with a weird mixed-endian
19432 mode on NEON. Reverse the index within each word but not the word
19433 itself. to_constant is safe because we checked is_constant above. */
19443 }
19445 /* Try to implement D using an SVE TBL instruction. */
19447 static bool
19449 {
19452 /* Permuting two variable-length vectors could overflow the
19453 index range. */
19464 else
19467 }
19469 /* Try to implement D using SVE SEL instruction. */
19471 static bool
19473 {
19499 /* Build a predicate that is true when op0 elements should be used. */
19502 {
19506 }
19510 /* TARGET = PRED ? OP0 : OP1. */
19513 }
19515 static bool
19517 {
19518 /* The pattern matching functions above are written to look for a small
19519 number to begin the sequence (0, 1, N/2). If we begin with an index
19520 from the second operand, we can swap the operands. */
19523 {
19526 }
19532 {
19553 }
19555 }
19557 /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
19559 static bool
19562 {
19565 /* Check whether the mask can be applied to a single vector. */
19570 {
19573 }
19575 {
19578 }
19579 else
19599 }
19601 /* Generate a byte permute mask for a register of mode MODE,
19602 which has NUNITS units. */
19604 rtx
19606 {
19607 /* We have to reverse each vector because we dont have
19608 a permuted load that can reverse-load according to ABI rules. */
19609 rtx mask;
19622 }
19624 /* Expand an SVE integer comparison using the SVE equivalent of:
19626 (set TARGET (CODE OP0 OP1)). */
19628 void
19630 {
19637 }
19639 /* Return the UNSPEC_COND_* code for comparison CODE. */
19641 static unsigned int
19643 {
19645 {
19662 }
19663 }
19665 /* Emit:
19667 (set TARGET (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X>))
19669 where <X> is the operation associated with comparison CODE.
19670 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
19672 static void
19675 {
19681 }
19683 /* Emit the SVE equivalent of:
19685 (set TMP1 (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X1>))
19686 (set TMP2 (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X2>))
19687 (set TARGET (ior:PRED_MODE TMP1 TMP2))
19689 where <Xi> is the operation associated with comparison CODEi.
19690 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
19692 static void
19695 {
19702 }
19704 /* Emit the SVE equivalent of:
19706 (set TMP (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X>))
19707 (set TARGET (not TMP))
19709 where <X> is the operation associated with comparison CODE.
19710 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
19712 static void
19715 {
19720 }
19722 /* Expand an SVE floating-point comparison using the SVE equivalent of:
19724 (set TARGET (CODE OP0 OP1))
19726 If CAN_INVERT_P is true, the caller can also handle inverted results;
19727 return true if the result is in fact inverted. */
19729 bool
19732 {
19738 {
19740 /* UNORDERED has no immediate form. */
19742 /* fall through */
19749 {
19750 /* There is native support for the comparison. */
19753 }
19756 /* This is a trapping operation (LT or GT). */
19762 {
19763 /* This would trap for signaling NaNs. */
19768 }
19769 /* fall through */
19775 {
19776 /* Work out which elements are ordered. */
19782 /* Test the opposite condition for the ordered elements,
19783 then invert the result. */
19786 else
19789 {
19793 }
19797 }
19801 /* ORDERED has no immediate form. */
19807 }
19809 /* There is native support for the inverse comparison. */
19812 {
19815 }
19818 }
19820 /* Expand an SVE vcond pattern with operands OPS. DATA_MODE is the mode
19821 of the data being selected and CMP_MODE is the mode of the values being
19822 compared. */
19824 void
19827 {
19831 {
19835 }
19836 else
19841 /* The "false" value can only be zero if the "true" value is a constant. */
19848 }
19850 /* Implement TARGET_MODES_TIEABLE_P. In principle we should always return
19851 true. However due to issues with register allocation it is preferable
19852 to avoid tieing integer scalar and FP scalar modes. Executing integer
19853 operations in general registers is better than treating them as scalar
19854 vector operations. This reduces latency and avoids redundant int<->FP
19855 moves. So tie modes if they are either the same class, or vector modes
19856 with other vector modes, vector structs or any scalar mode. */
19858 static bool
19860 {
19864 /* We specifically want to allow elements of "structure" modes to
19865 be tieable to the structure. This more general condition allows
19866 other rarer situations too. The reason we don't extend this to
19867 predicate modes is that there are no predicate structure modes
19868 nor any specific instructions for extracting part of a predicate
19869 register. */
19874 /* Also allow any scalar modes with vectors. */
19880 }
19882 /* Return a new RTX holding the result of moving POINTER forward by
19883 AMOUNT bytes. */
19885 static rtx
19887 {
19892 }
19894 /* Return a new RTX holding the result of moving POINTER forward by the
19895 size of the mode it points to. */
19897 static rtx
19899 {
19901 }
19903 /* Copy one MODE sized block from SRC to DST, then progress SRC and DST by
19904 MODE bytes. */
19906 static void
19908 machine_mode mode)
19909 {
19912 /* "Cast" the pointers to the correct mode. */
19915 /* Emit the memcpy. */
19918 /* Move the pointers forward. */
19921 }
19923 /* Expand cpymem, as if from a __builtin_memcpy. Return true if
19924 we succeed, otherwise return false. */
19926 bool
19928 {
19932 rtx base;
19936 /* When optimizing for size, give a better estimate of the length of a
19937 memcpy call, but use the default otherwise. Moves larger than 8 bytes
19938 will always require an even number of instructions to do now. And each
19939 operation requires both a load+store, so devide the max number by 2. */
19942 /* We can't do anything smart if the amount to copy is not constant. */
19948 /* Try to keep the number of instructions low. For all cases we will do at
19949 most two moves for the residual amount, since we'll always overlap the
19950 remainder. */
19960 /* Convert n to bits to make the rest of the code simpler. */
19963 /* Maximum amount to copy in one go. The AArch64 back-end has integer modes
19964 larger than TImode, but we should not use them for loads/stores here. */
19968 {
19969 /* Find the largest mode in which to do the copy in without over reading
19970 or writing. */
19971 opt_scalar_int_mode mode_iter;
19983 /* Do certain trailing copies as overlapping if it's going to be
19984 cheaper. i.e. less instructions to do so. For instance doing a 15
19985 byte copy it's more efficient to do two overlapping 8 byte copies than
19986 8 + 6 + 1. */
19988 {
19994 }
19995 }
19998 }
20000 /* Split a DImode store of a CONST_INT SRC to MEM DST as two
20001 SImode stores. Handle the case when the constant has identical
20002 bottom and top halves. This is beneficial when the two stores can be
20003 merged into an STP and we avoid synthesising potentially expensive
20004 immediates twice. Return true if such a split is possible. */
20006 bool
20008 {
20017 unsigned int orig_cost
20019 unsigned int lo_cost
20022 /* We want to transform:
20023 MOV x1, 49370
20024 MOVK x1, 0x140, lsl 16
20025 MOVK x1, 0xc0da, lsl 32
20026 MOVK x1, 0x140, lsl 48
20027 STR x1, [x0]
20028 into:
20029 MOV w1, 49370
20030 MOVK w1, 0x140, lsl 16
20031 STP w1, w1, [x0]
20032 So we want to perform this only when we save two instructions
20033 or more. When optimizing for size, however, accept any code size
20034 savings we can. */
20049 /* Don't emit an explicit store pair as this may not be always profitable.
20050 Let the sched-fusion logic decide whether to merge them. */
20055 }
20057 /* Generate RTL for a conditional branch with rtx comparison CODE in
20058 mode CC_MODE. The destination of the unlikely conditional branch
20059 is LABEL_REF. */
20061 void
20063 rtx label_ref)
20064 {
20065 rtx x;
20068 const0_rtx);
20072 pc_rtx);
20074 }
20076 /* Generate DImode scratch registers for 128-bit (TImode) addition.
20078 OP1 represents the TImode destination operand 1
20079 OP2 represents the TImode destination operand 2
20080 LOW_DEST represents the low half (DImode) of TImode operand 0
20081 LOW_IN1 represents the low half (DImode) of TImode operand 1
20082 LOW_IN2 represents the low half (DImode) of TImode operand 2
20083 HIGH_DEST represents the high half (DImode) of TImode operand 0
20084 HIGH_IN1 represents the high half (DImode) of TImode operand 1
20085 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
20087 void
20092 {
20101 }
20103 /* Generate DImode scratch registers for 128-bit (TImode) subtraction.
20105 This function differs from 'arch64_addti_scratch_regs' in that
20106 OP1 can be an immediate constant (zero). We must call
20107 subreg_highpart_offset with DImode and TImode arguments, otherwise
20108 VOIDmode will be used for the const_int which generates an internal
20109 error from subreg_size_highpart_offset which does not expect a size of zero.
20111 OP1 represents the TImode destination operand 1
20112 OP2 represents the TImode destination operand 2
20113 LOW_DEST represents the low half (DImode) of TImode operand 0
20114 LOW_IN1 represents the low half (DImode) of TImode operand 1
20115 LOW_IN2 represents the low half (DImode) of TImode operand 2
20116 HIGH_DEST represents the high half (DImode) of TImode operand 0
20117 HIGH_IN1 represents the high half (DImode) of TImode operand 1
20118 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
20121 void
20126 {
20139 }
20141 /* Generate RTL for 128-bit (TImode) subtraction with overflow.
20143 OP0 represents the TImode destination operand 0
20144 LOW_DEST represents the low half (DImode) of TImode operand 0
20145 LOW_IN1 represents the low half (DImode) of TImode operand 1
20146 LOW_IN2 represents the low half (DImode) of TImode operand 2
20147 HIGH_DEST represents the high half (DImode) of TImode operand 0
20148 HIGH_IN1 represents the high half (DImode) of TImode operand 1
20149 HIGH_IN2 represents the high half (DImode) of TImode operand 2
20150 UNSIGNED_P is true if the operation is being performed on unsigned
20151 values. */
20152 void
20156 {
20158 {
20163 else
20165 }
20166 else
20167 {
20169 {
20173 }
20174 else
20179 else
20181 }
20186 }
20188 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
20190 static unsigned HOST_WIDE_INT
20192 {
20195 else
20197 }
20199 static rtx
20202 {
20206 insn_code icode;
20217 {
20245 }
20250 {
20253 }
20262 {
20265 }
20271 }
20273 static rtx
20276 {
20280 insn_code icode;
20292 {
20316 }
20323 {
20326 }
20334 {
20335 /* Treat the ccmp patterns as canonical and use them where possible,
20336 but fall back to ccmp_rev patterns if there's no other option. */
20345 else
20346 {
20349 }
20351 }
20362 {
20365 }
20371 }
20373 #undef TARGET_GEN_CCMP_FIRST
20374 #define TARGET_GEN_CCMP_FIRST aarch64_gen_ccmp_first
20376 #undef TARGET_GEN_CCMP_NEXT
20377 #define TARGET_GEN_CCMP_NEXT aarch64_gen_ccmp_next
20379 /* Implement TARGET_SCHED_MACRO_FUSION_P. Return true if target supports
20380 instruction fusion of some sort. */
20382 static bool
20384 {
20386 }
20389 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P. Return true if PREV and CURR
20390 should be kept together during scheduling. */
20392 static bool
20394 {
20395 rtx set_dest;
20398 /* prev and curr are simple SET insns i.e. no flag setting or branching. */
20405 {
20406 /* We are trying to match:
20407 prev (mov) == (set (reg r0) (const_int imm16))
20408 curr (movk) == (set (zero_extract (reg r0)
20409 (const_int 16)
20410 (const_int 16))
20411 (const_int imm16_1)) */
20423 {
20425 }
20426 }
20429 {
20431 /* We're trying to match:
20432 prev (adrp) == (set (reg r1)
20433 (high (symbol_ref ("SYM"))))
20434 curr (add) == (set (reg r0)
20435 (lo_sum (reg r1)
20436 (symbol_ref ("SYM"))))
20437 Note that r0 need not necessarily be the same as r1, especially
20438 during pre-regalloc scheduling. */
20442 {
20450 }
20451 }
20454 {
20456 /* We're trying to match:
20457 prev (movk) == (set (zero_extract (reg r0)
20458 (const_int 16)
20459 (const_int 32))
20460 (const_int imm16_1))
20461 curr (movk) == (set (zero_extract (reg r0)
20462 (const_int 16)
20463 (const_int 48))
20464 (const_int imm16_2)) */
20480 }
20482 {
20483 /* We're trying to match:
20484 prev (adrp) == (set (reg r0)
20485 (high (symbol_ref ("SYM"))))
20486 curr (ldr) == (set (reg r1)
20487 (mem (lo_sum (reg r0)
20488 (symbol_ref ("SYM")))))
20489 or
20490 curr (ldr) == (set (reg r1)
20491 (zero_extend (mem
20492 (lo_sum (reg r0)
20493 (symbol_ref ("SYM")))))) */
20496 {
20509 }
20510 }
20512 /* Fuse compare (CMP/CMN/TST/BICS) and conditional branch. */
20520 /* Fuse flag-setting ALU instructions and conditional branch. */
20523 {
20525 rtx cc_reg_1;
20530 && prev
20532 {
20535 /* FIXME: this misses some which is considered simple arthematic
20536 instructions for ThunderX. Simple shifts are missed here. */
20542 }
20543 }
20545 /* Fuse ALU instructions and CBZ/CBNZ. */
20547 && curr_set
20550 {
20551 /* We're trying to match:
20552 prev (alu_insn) == (set (r0) plus ((r0) (r1/imm)))
20553 curr (cbz) == (set (pc) (if_then_else (eq/ne) (r0)
20554 (const_int 0))
20555 (label_ref ("SYM"))
20556 (pc)) */
20563 {
20564 /* Fuse ALU operations followed by conditional branch instruction. */
20566 {
20587 }
20588 }
20589 }
20592 }
20594 /* Return true iff the instruction fusion described by OP is enabled. */
20596 bool
20598 {
20600 }
20602 /* If MEM is in the form of [base+offset], extract the two parts
20603 of address and set to BASE and OFFSET, otherwise return false
20604 after clearing BASE and OFFSET. */
20606 bool
20608 {
20609 rtx addr;
20616 {
20620 }
20624 {
20628 }
20634 }
20636 /* Types for scheduling fusion. */
20637 enum sched_fusion_type
20638 {
20640 SCHED_FUSION_LD_SIGN_EXTEND,
20641 SCHED_FUSION_LD_ZERO_EXTEND,
20642 SCHED_FUSION_LD,
20643 SCHED_FUSION_ST,
20644 SCHED_FUSION_NUM
20645 };
20647 /* If INSN is a load or store of address in the form of [base+offset],
20648 extract the two parts and set to BASE and OFFSET. Return scheduling
20649 fusion type this INSN is. */
20651 static enum sched_fusion_type
20653 {
20671 {
20676 }
20678 {
20683 }
20688 {
20691 }
20692 else
20699 }
20701 /* Implement the TARGET_SCHED_FUSION_PRIORITY hook.
20703 Currently we only support to fuse ldr or str instructions, so FUSION_PRI
20704 and PRI are only calculated for these instructions. For other instruction,
20705 FUSION_PRI and PRI are simply set to MAX_PRI - 1. In the future, other
20706 type instruction fusion can be added by returning different priorities.
20708 It's important that irrelevant instructions get the largest FUSION_PRI. */
20710 static void
20713 {
20723 {
20727 }
20729 /* Set FUSION_PRI according to fusion type and base register. */
20732 /* Calculate PRI. */
20735 /* INSN with smaller offset goes first. */
20739 else
20744 }
20746 /* Implement the TARGET_SCHED_ADJUST_PRIORITY hook.
20747 Adjust priority of sha1h instructions so they are scheduled before
20748 other SHA1 instructions. */
20750 static int
20752 {
20756 {
20761 }
20764 }
20766 /* Given OPERANDS of consecutive load/store, check if we can merge
20767 them into ldp/stp. LOAD is true if they are load instructions.
20768 MODE is the mode of memory operands. */
20770 bool
20772 machine_mode mode)
20773 {
20779 {
20787 }
20788 else
20789 {
20794 }
20796 /* The mems cannot be volatile. */
20800 /* If we have SImode and slow unaligned ldp,
20801 check the alignment to be at least 8 byte. */
20805 && !optimize_size
20809 /* Check if the addresses are in the form of [base+offset]. */
20817 /* Check if the bases are same. */
20821 /* The operands must be of the same size. */
20827 /* We should only be trying this for fixed-sized modes. There is no
20828 SVE LDP/STP instruction. */
20830 /* Check if the offsets are consecutive. */
20834 /* Check if the addresses are clobbered by load. */
20836 {
20840 /* In increasing order, the last load can clobber the address. */
20843 }
20845 /* One of the memory accesses must be a mempair operand.
20846 If it is not the first one, they need to be swapped by the
20847 peephole. */
20854 else
20859 else
20862 /* Check if the registers are of same class. */
20867 }
20869 /* Given OPERANDS of consecutive load/store that can be merged,
20870 swap them if they are not in ascending order. */
20871 void
20873 {
20878 {
20881 }
20882 else
20883 {
20886 }
20895 {
20896 /* Irrespective of whether this is a load or a store,
20897 we do the same swap. */
20900 }
20901 }
20903 /* Taking X and Y to be HOST_WIDE_INT pointers, return the result of a
20904 comparison between the two. */
20905 int
20907 {
20910 }
20912 /* Taking X and Y to be pairs of RTX, one pointing to a MEM rtx and the
20913 other pointing to a REG rtx containing an offset, compare the offsets
20914 of the two pairs.
20916 Return:
20918 1 iff offset (X) > offset (Y)
20919 0 iff offset (X) == offset (Y)
20920 -1 iff offset (X) < offset (Y) */
20921 int
20923 {
20930 else
20935 else
20938 /* Extract the offsets. */
20945 }
20947 /* Given OPERANDS of consecutive load/store, check if we can merge
20948 them into ldp/stp by adjusting the offset. LOAD is true if they
20949 are load instructions. MODE is the mode of memory operands.
20951 Given below consecutive stores:
20953 str w1, [xb, 0x100]
20954 str w1, [xb, 0x104]
20955 str w1, [xb, 0x108]
20956 str w1, [xb, 0x10c]
20958 Though the offsets are out of the range supported by stp, we can
20959 still pair them after adjusting the offset, like:
20961 add scratch, xb, 0x100
20962 stp w1, w1, [scratch]
20963 stp w1, w1, [scratch, 0x8]
20965 The peephole patterns detecting this opportunity should guarantee
20966 the scratch register is avaliable. */
20968 bool
20970 scalar_mode mode)
20971 {
20978 {
20980 {
20985 }
20987 /* Do not attempt to merge the loads if the loads clobber each other. */
20992 }
20993 else
20995 {
20998 }
21000 /* Skip if memory operand is by itself valid for ldp/stp. */
21005 {
21006 /* The mems cannot be volatile. */
21010 /* Check if the addresses are in the form of [base+offset]. */
21014 }
21016 /* Check if the registers are of same class. */
21022 {
21025 }
21026 else
21027 {
21030 }
21032 /* Only the last register in the order in which they occur
21033 may be clobbered by the load. */
21039 /* Check if the bases are same. */
21049 /* Check if the offsets can be put in the right order to do a ldp/stp. */
21051 aarch64_host_wide_int_compare);
21057 /* Check that offsets are within range of each other. The ldp/stp
21058 instructions have 7 bit immediate offsets, so use 0x80. */
21062 /* The offsets must be aligned with respect to each other. */
21066 /* If we have SImode and slow unaligned ldp,
21067 check the alignment to be at least 8 byte. */
21071 && !optimize_size
21076 }
21078 /* Given OPERANDS of consecutive load/store, this function pairs them
21079 into LDP/STP after adjusting the offset. It depends on the fact
21080 that the operands can be sorted so the offsets are correct for STP.
21081 MODE is the mode of memory operands. CODE is the rtl operator
21082 which should be applied to all memory operands, it's SIGN_EXTEND,
21083 ZERO_EXTEND or UNKNOWN. */
21085 bool
21088 {
21095 /* We make changes on a copy as we may still bail out. */
21099 /* Sort the operands. */
21102 /* Copy the memory operands so that if we have to bail for some
21103 reason the original addresses are unchanged. */
21105 {
21110 }
21111 else
21112 {
21118 }
21125 /* Adjust offset so it can fit in LDP/STP instruction. */
21133 /* The base offset is optimally half way between the two STP/LDP offsets. */
21136 else
21137 /* However, due to issues with negative LDP/STP offset generation for
21138 larger modes, for DF, DI and vector modes. we must not use negative
21139 addresses smaller than 9 signed unadjusted bits can store. This
21140 provides the most range in this case. */
21143 /* Adjust the base so that it is aligned with the addresses but still
21144 optimal. */
21146 /* Fix the offset, bearing in mind we want to make it bigger not
21147 smaller. */
21150 /* The negative range of LDP/STP is one larger than the positive range. */
21153 /* Check if base offset is too big or too small. We can attempt to resolve
21154 this issue by setting it to the maximum value and seeing if the offsets
21155 still fit. */
21157 {
21159 /* We must still make sure that the base offset is aligned with respect
21160 to the address. But it may may not be made any bigger. */
21162 }
21164 /* Likewise for the case where the base is too small. */
21166 {
21169 }
21171 /* Offset of the first STP/LDP. */
21174 /* Offset of the second STP/LDP. */
21177 /* The offsets must be within the range of the LDP/STP instructions. */
21196 {
21201 }
21203 {
21208 }
21211 {
21220 }
21221 else
21222 {
21231 }
21233 /* Emit adjusting instruction. */
21235 /* Emit ldp/stp instructions. */
21243 }
21245 /* Implement TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE. Assume for now that
21246 it isn't worth branching around empty masked ops (including masked
21247 stores). */
21249 static bool
21251 {
21253 }
21255 /* Return 1 if pseudo register should be created and used to hold
21256 GOT address for PIC code. */
21258 bool
21260 {
21262 }
21264 /* Implement TARGET_UNSPEC_MAY_TRAP_P. */
21266 static int
21268 {
21270 {
21277 }
21280 }
21283 /* If X is a positive CONST_DOUBLE with a value that is a power of 2
21284 return the log2 of that value. Otherwise return -1. */
21286 int
21288 {
21303 }
21305 /* If X is a positive CONST_DOUBLE with a value that is the reciprocal of a
21306 power of 2 (i.e 1/2^n) return the number of float bits. e.g. for x==(1/2^n)
21307 return n. Otherwise return -1. */
21309 int
21311 {
21312 REAL_VALUE_TYPE r0;
21320 {
21324 }
21326 }
21328 /* If X is a vector of equal CONST_DOUBLE values and that value is
21329 Y, return the aarch64_fpconst_pow_of_2 of Y. Otherwise return -1. */
21331 int
21333 {
21351 }
21353 /* Implement TARGET_PROMOTED_TYPE to promote 16-bit floating point types
21354 to float.
21356 __fp16 always promotes through this hook.
21357 _Float16 may promote if TARGET_FLT_EVAL_METHOD is 16, but we do that
21358 through the generic excess precision logic rather than here. */
21360 static tree
21362 {
21368 }
21370 /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
21372 static bool
21374 optimization_type opt_type)
21375 {
21377 {
21383 }
21384 }
21386 /* Implement the TARGET_DWARF_POLY_INDETERMINATE_VALUE hook. */
21388 static unsigned int
21391 {
21392 /* Polynomial invariant 1 == (VG / 2) - 1. */
21397 }
21399 /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
21400 if MODE is HFmode, and punt to the generic implementation otherwise. */
21402 static bool
21404 {
21408 }
21410 /* Implement TARGET_SCALAR_MODE_SUPPORTED_P - return TRUE
21411 if MODE is HFmode, and punt to the generic implementation otherwise. */
21413 static bool
21415 {
21419 }
21421 /* Set the value of FLT_EVAL_METHOD.
21422 ISO/IEC TS 18661-3 defines two values that we'd like to make use of:
21424 0: evaluate all operations and constants, whose semantic type has at
21425 most the range and precision of type float, to the range and
21426 precision of float; evaluate all other operations and constants to
21427 the range and precision of the semantic type;
21429 N, where _FloatN is a supported interchange floating type
21430 evaluate all operations and constants, whose semantic type has at
21431 most the range and precision of _FloatN type, to the range and
21432 precision of the _FloatN type; evaluate all other operations and
21433 constants to the range and precision of the semantic type;
21435 If we have the ARMv8.2-A extensions then we support _Float16 in native
21436 precision, so we should set this to 16. Otherwise, we support the type,
21437 but want to evaluate expressions in float precision, so set this to
21438 0. */
21440 static enum flt_eval_method
21442 {
21444 {
21447 /* We can calculate either in 16-bit range and precision or
21448 32-bit range and precision. Make that decision based on whether
21449 we have native support for the ARMv8.2-A 16-bit floating-point
21450 instructions or not. */
21452 ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
21458 }
21460 }
21462 /* Implement TARGET_SCHED_CAN_SPECULATE_INSN. Return true if INSN can be
21463 scheduled for speculative execution. Reject the long-running division
21464 and square-root instructions. */
21466 static bool
21468 {
21470 {
21488 }
21489 }
21491 /* Implement TARGET_COMPUTE_PRESSURE_CLASSES. */
21493 static int
21495 {
21499 /* PR_REGS isn't a useful pressure class because many predicate pseudo
21500 registers need to go in PR_LO_REGS at some point during their
21501 lifetime. Splitting it into two halves has the effect of making
21502 all predicates count against PR_LO_REGS, so that we try whenever
21503 possible to restrict the number of live predicates to 8. This
21504 greatly reduces the amount of spilling in certain loops. */
21508 }
21510 /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
21512 static bool
21515 {
21525 /* Don't allow changes between partial SVE modes and other modes.
21526 The contents of partial SVE modes are distributed evenly across
21527 the register, whereas GCC expects them to be clustered together. */
21531 /* Similarly reject changes between partial SVE modes that have
21532 different patterns of significant and insignificant bits. */
21539 {
21540 /* Don't allow changes between SVE data modes and non-SVE modes.
21541 See the comment at the head of aarch64-sve.md for details. */
21545 /* Don't allow changes in element size: lane 0 of the new vector
21546 would not then be lane 0 of the old vector. See the comment
21547 above aarch64_maybe_expand_sve_subreg_move for a more detailed
21548 description.
21550 In the worst case, this forces a register to be spilled in
21551 one mode and reloaded in the other, which handles the
21552 endianness correctly. */
21555 }
21557 }
21559 /* Implement TARGET_EARLY_REMAT_MODES. */
21561 static void
21563 {
21564 /* SVE values are not normally live across a call, so it should be
21565 worth doing early rematerialization even in VL-specific mode. */
21569 }
21571 /* Override the default target speculation_safe_value. */
21572 static rtx
21575 {
21576 /* Maybe we should warn if falling back to hard barriers. They are
21577 likely to be noticably more expensive than the alternative below. */
21589 }
21591 /* Implement TARGET_ESTIMATED_POLY_VALUE.
21592 Look into the tuning structure for an estimate.
21593 VAL.coeffs[1] is multiplied by the number of VQ chunks over the initial
21594 Advanced SIMD 128 bits. */
21596 static HOST_WIDE_INT
21598 {
21599 enum aarch64_sve_vector_bits_enum width_source
21602 /* If we still don't have an estimate, use the default. */
21608 }
21611 /* Return true for types that could be supported as SIMD return or
21612 argument types. */
21614 static bool
21616 {
21618 {
21621 }
21623 }
21625 /* Return true for types that currently are supported as SIMD return
21626 or argument types. */
21628 static bool
21630 {
21638 }
21640 /* Implement TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN. */
21642 static int
21646 {
21657 {
21661 }
21666 {
21669 "GCC does not currently support mixed size types "
21673 "GCC does not currently support return type %qT "
21675 else
21678 ret_type);
21680 }
21683 {
21687 {
21690 "GCC does not currently support mixed size types "
21692 else
21694 "GCC does not currently support argument type %qT "
21697 }
21698 }
21704 {
21708 }
21709 else
21710 {
21714 {
21719 }
21720 }
21724 }
21726 /* Implement TARGET_SIMD_CLONE_ADJUST. */
21728 static void
21730 {
21731 /* Add aarch64_vector_pcs target attribute to SIMD clones so they
21732 use the correct ABI. */
21737 }
21739 /* Implement TARGET_SIMD_CLONE_USABLE. */
21741 static int
21743 {
21745 {
21752 }
21753 }
21755 /* Implement TARGET_COMP_TYPE_ATTRIBUTES */
21757 static int
21759 {
21764 }
21766 /* Implement TARGET_GET_MULTILIB_ABI_NAME */
21770 {
21774 }
21776 /* Implement TARGET_STACK_PROTECT_GUARD. In case of a
21777 global variable based guard use the default else
21778 return a null tree. */
21779 static tree
21781 {
21786 }
21788 /* Return the diagnostic message string if conversion from FROMTYPE to
21789 TOTYPE is not allowed, NULL otherwise. */
21793 {
21795 {
21796 /* Do no allow conversions to/from BFmode scalar types. */
21801 }
21803 /* Conversion allowed. */
21805 }
21807 /* Return the diagnostic message string if the unary operation OP is
21808 not permitted on TYPE, NULL otherwise. */
21812 {
21813 /* Reject all single-operand operations on BFmode except for &. */
21817 /* Operation allowed. */
21819 }
21821 /* Return the diagnostic message string if the binary operation OP is
21822 not permitted on TYPE1 and TYPE2, NULL otherwise. */
21826 const_tree type2)
21827 {
21828 /* Reject all 2-operand operations on BFmode. */
21833 /* Operation allowed. */
21835 }
21837 /* Implement TARGET_ASM_FILE_END for AArch64. This adds the AArch64 GNU NOTE
21838 section at the end if needed. */
21839 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
21840 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
21841 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1)
21842 void
21844 {
21855 {
21856 /* Generate .note.gnu.property section. */
21860 /* PT_NOTE header: namesz, descsz, type.
21861 namesz = 4 ("GNU\0")
21862 descsz = 16 (Size of the program property array)
21863 [(12 + padding) * Number of array elements]
21864 type = 5 (NT_GNU_PROPERTY_TYPE_0). */
21870 /* PT_NOTE name. */
21873 /* PT_NOTE contents for NT_GNU_PROPERTY_TYPE_0:
21874 type = GNU_PROPERTY_AARCH64_FEATURE_1_AND
21875 datasz = 4
21876 data = feature_1_and. */
21881 /* Pad the size of the note to the required alignment. */
21883 }
21884 }
21885 #undef GNU_PROPERTY_AARCH64_FEATURE_1_PAC
21886 #undef GNU_PROPERTY_AARCH64_FEATURE_1_BTI
21887 #undef GNU_PROPERTY_AARCH64_FEATURE_1_AND
21889 /* Target-specific selftests. */
21891 #if CHECKING_P
21895 /* Selftest for the RTL loader.
21896 Verify that the RTL loader copes with a dump from
21897 print_rtx_function. This is essentially just a test that class
21898 function_reader can handle a real dump, but it also verifies
21899 that lookup_reg_by_dump_name correctly handles hard regs.
21900 The presence of hard reg names in the dump means that the test is
21901 target-specific, hence it is in this file. */
21903 static void
21905 {
21917 /* Verify crtl->return_rtx. */
21921 }
21923 /* Run all target-specific selftests. */
21925 static void
21927 {
21929 }
21935 #undef TARGET_STACK_PROTECT_GUARD
21936 #define TARGET_STACK_PROTECT_GUARD aarch64_stack_protect_guard
21938 #undef TARGET_ADDRESS_COST
21939 #define TARGET_ADDRESS_COST aarch64_address_cost
21941 /* This hook will determines whether unnamed bitfields affect the alignment
21942 of the containing structure. The hook returns true if the structure
21943 should inherit the alignment requirements of an unnamed bitfield's
21944 type. */
21945 #undef TARGET_ALIGN_ANON_BITFIELD
21946 #define TARGET_ALIGN_ANON_BITFIELD hook_bool_void_true
21948 #undef TARGET_ASM_ALIGNED_DI_OP
21951 #undef TARGET_ASM_ALIGNED_HI_OP
21954 #undef TARGET_ASM_ALIGNED_SI_OP
21957 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
21958 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK \
21959 hook_bool_const_tree_hwi_hwi_const_tree_true
21961 #undef TARGET_ASM_FILE_START
21962 #define TARGET_ASM_FILE_START aarch64_start_file
21964 #undef TARGET_ASM_OUTPUT_MI_THUNK
21965 #define TARGET_ASM_OUTPUT_MI_THUNK aarch64_output_mi_thunk
21967 #undef TARGET_ASM_SELECT_RTX_SECTION
21968 #define TARGET_ASM_SELECT_RTX_SECTION aarch64_select_rtx_section
21970 #undef TARGET_ASM_TRAMPOLINE_TEMPLATE
21971 #define TARGET_ASM_TRAMPOLINE_TEMPLATE aarch64_asm_trampoline_template
21973 #undef TARGET_BUILD_BUILTIN_VA_LIST
21974 #define TARGET_BUILD_BUILTIN_VA_LIST aarch64_build_builtin_va_list
21976 #undef TARGET_CALLEE_COPIES
21977 #define TARGET_CALLEE_COPIES hook_bool_CUMULATIVE_ARGS_arg_info_false
21979 #undef TARGET_CAN_ELIMINATE
21980 #define TARGET_CAN_ELIMINATE aarch64_can_eliminate
21982 #undef TARGET_CAN_INLINE_P
21983 #define TARGET_CAN_INLINE_P aarch64_can_inline_p
21985 #undef TARGET_CANNOT_FORCE_CONST_MEM
21986 #define TARGET_CANNOT_FORCE_CONST_MEM aarch64_cannot_force_const_mem
21988 #undef TARGET_CASE_VALUES_THRESHOLD
21989 #define TARGET_CASE_VALUES_THRESHOLD aarch64_case_values_threshold
21991 #undef TARGET_CONDITIONAL_REGISTER_USAGE
21992 #define TARGET_CONDITIONAL_REGISTER_USAGE aarch64_conditional_register_usage
21994 /* Only the least significant bit is used for initialization guard
21995 variables. */
21996 #undef TARGET_CXX_GUARD_MASK_BIT
21997 #define TARGET_CXX_GUARD_MASK_BIT hook_bool_void_true
21999 #undef TARGET_C_MODE_FOR_SUFFIX
22000 #define TARGET_C_MODE_FOR_SUFFIX aarch64_c_mode_for_suffix
22002 #ifdef TARGET_BIG_ENDIAN_DEFAULT
22003 #undef TARGET_DEFAULT_TARGET_FLAGS
22004 #define TARGET_DEFAULT_TARGET_FLAGS (MASK_BIG_END)
22005 #endif
22007 #undef TARGET_CLASS_MAX_NREGS
22008 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
22010 #undef TARGET_BUILTIN_DECL
22011 #define TARGET_BUILTIN_DECL aarch64_builtin_decl
22013 #undef TARGET_BUILTIN_RECIPROCAL
22014 #define TARGET_BUILTIN_RECIPROCAL aarch64_builtin_reciprocal
22016 #undef TARGET_C_EXCESS_PRECISION
22017 #define TARGET_C_EXCESS_PRECISION aarch64_excess_precision
22019 #undef TARGET_EXPAND_BUILTIN
22020 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
22022 #undef TARGET_EXPAND_BUILTIN_VA_START
22023 #define TARGET_EXPAND_BUILTIN_VA_START aarch64_expand_builtin_va_start
22025 #undef TARGET_FOLD_BUILTIN
22026 #define TARGET_FOLD_BUILTIN aarch64_fold_builtin
22028 #undef TARGET_FUNCTION_ARG
22029 #define TARGET_FUNCTION_ARG aarch64_function_arg
22031 #undef TARGET_FUNCTION_ARG_ADVANCE
22032 #define TARGET_FUNCTION_ARG_ADVANCE aarch64_function_arg_advance
22034 #undef TARGET_FUNCTION_ARG_BOUNDARY
22035 #define TARGET_FUNCTION_ARG_BOUNDARY aarch64_function_arg_boundary
22037 #undef TARGET_FUNCTION_ARG_PADDING
22038 #define TARGET_FUNCTION_ARG_PADDING aarch64_function_arg_padding
22040 #undef TARGET_GET_RAW_RESULT_MODE
22041 #define TARGET_GET_RAW_RESULT_MODE aarch64_get_reg_raw_mode
22042 #undef TARGET_GET_RAW_ARG_MODE
22043 #define TARGET_GET_RAW_ARG_MODE aarch64_get_reg_raw_mode
22045 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
22046 #define TARGET_FUNCTION_OK_FOR_SIBCALL aarch64_function_ok_for_sibcall
22048 #undef TARGET_FUNCTION_VALUE
22049 #define TARGET_FUNCTION_VALUE aarch64_function_value
22051 #undef TARGET_FUNCTION_VALUE_REGNO_P
22052 #define TARGET_FUNCTION_VALUE_REGNO_P aarch64_function_value_regno_p
22054 #undef TARGET_GIMPLE_FOLD_BUILTIN
22055 #define TARGET_GIMPLE_FOLD_BUILTIN aarch64_gimple_fold_builtin
22057 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
22058 #define TARGET_GIMPLIFY_VA_ARG_EXPR aarch64_gimplify_va_arg_expr
22060 #undef TARGET_INIT_BUILTINS
22061 #define TARGET_INIT_BUILTINS aarch64_init_builtins
22063 #undef TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS
22064 #define TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS \
22065 aarch64_ira_change_pseudo_allocno_class
22067 #undef TARGET_LEGITIMATE_ADDRESS_P
22068 #define TARGET_LEGITIMATE_ADDRESS_P aarch64_legitimate_address_hook_p
22070 #undef TARGET_LEGITIMATE_CONSTANT_P
22071 #define TARGET_LEGITIMATE_CONSTANT_P aarch64_legitimate_constant_p
22073 #undef TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT
22074 #define TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT \
22075 aarch64_legitimize_address_displacement
22077 #undef TARGET_LIBGCC_CMP_RETURN_MODE
22078 #define TARGET_LIBGCC_CMP_RETURN_MODE aarch64_libgcc_cmp_return_mode
22080 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
22081 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
22082 aarch64_libgcc_floating_mode_supported_p
22084 #undef TARGET_MANGLE_TYPE
22085 #define TARGET_MANGLE_TYPE aarch64_mangle_type
22087 #undef TARGET_INVALID_CONVERSION
22088 #define TARGET_INVALID_CONVERSION aarch64_invalid_conversion
22090 #undef TARGET_INVALID_UNARY_OP
22091 #define TARGET_INVALID_UNARY_OP aarch64_invalid_unary_op
22093 #undef TARGET_INVALID_BINARY_OP
22094 #define TARGET_INVALID_BINARY_OP aarch64_invalid_binary_op
22096 #undef TARGET_VERIFY_TYPE_CONTEXT
22097 #define TARGET_VERIFY_TYPE_CONTEXT aarch64_verify_type_context
22099 #undef TARGET_MEMORY_MOVE_COST
22100 #define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
22102 #undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
22103 #define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
22105 #undef TARGET_MUST_PASS_IN_STACK
22106 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
22108 /* This target hook should return true if accesses to volatile bitfields
22109 should use the narrowest mode possible. It should return false if these
22110 accesses should use the bitfield container type. */
22111 #undef TARGET_NARROW_VOLATILE_BITFIELD
22112 #define TARGET_NARROW_VOLATILE_BITFIELD hook_bool_void_false
22114 #undef TARGET_OPTION_OVERRIDE
22115 #define TARGET_OPTION_OVERRIDE aarch64_override_options
22117 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
22118 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE \
22119 aarch64_override_options_after_change
22121 #undef TARGET_OPTION_SAVE
22122 #define TARGET_OPTION_SAVE aarch64_option_save
22124 #undef TARGET_OPTION_RESTORE
22125 #define TARGET_OPTION_RESTORE aarch64_option_restore
22127 #undef TARGET_OPTION_PRINT
22128 #define TARGET_OPTION_PRINT aarch64_option_print
22130 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
22131 #define TARGET_OPTION_VALID_ATTRIBUTE_P aarch64_option_valid_attribute_p
22133 #undef TARGET_SET_CURRENT_FUNCTION
22134 #define TARGET_SET_CURRENT_FUNCTION aarch64_set_current_function
22136 #undef TARGET_PASS_BY_REFERENCE
22137 #define TARGET_PASS_BY_REFERENCE aarch64_pass_by_reference
22139 #undef TARGET_PREFERRED_RELOAD_CLASS
22140 #define TARGET_PREFERRED_RELOAD_CLASS aarch64_preferred_reload_class
22142 #undef TARGET_SCHED_REASSOCIATION_WIDTH
22143 #define TARGET_SCHED_REASSOCIATION_WIDTH aarch64_reassociation_width
22145 #undef TARGET_PROMOTED_TYPE
22146 #define TARGET_PROMOTED_TYPE aarch64_promoted_type
22148 #undef TARGET_SECONDARY_RELOAD
22149 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
22151 #undef TARGET_SHIFT_TRUNCATION_MASK
22152 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
22154 #undef TARGET_SETUP_INCOMING_VARARGS
22155 #define TARGET_SETUP_INCOMING_VARARGS aarch64_setup_incoming_varargs
22157 #undef TARGET_STRUCT_VALUE_RTX
22158 #define TARGET_STRUCT_VALUE_RTX aarch64_struct_value_rtx
22160 #undef TARGET_REGISTER_MOVE_COST
22161 #define TARGET_REGISTER_MOVE_COST aarch64_register_move_cost
22163 #undef TARGET_RETURN_IN_MEMORY
22164 #define TARGET_RETURN_IN_MEMORY aarch64_return_in_memory
22166 #undef TARGET_RETURN_IN_MSB
22167 #define TARGET_RETURN_IN_MSB aarch64_return_in_msb
22169 #undef TARGET_RTX_COSTS
22170 #define TARGET_RTX_COSTS aarch64_rtx_costs_wrapper
22172 #undef TARGET_SCALAR_MODE_SUPPORTED_P
22173 #define TARGET_SCALAR_MODE_SUPPORTED_P aarch64_scalar_mode_supported_p
22175 #undef TARGET_SCHED_ISSUE_RATE
22176 #define TARGET_SCHED_ISSUE_RATE aarch64_sched_issue_rate
22178 #undef TARGET_SCHED_VARIABLE_ISSUE
22179 #define TARGET_SCHED_VARIABLE_ISSUE aarch64_sched_variable_issue
22181 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
22182 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
22183 aarch64_sched_first_cycle_multipass_dfa_lookahead
22185 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
22186 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD \
22187 aarch64_first_cycle_multipass_dfa_lookahead_guard
22189 #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
22190 #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS \
22191 aarch64_get_separate_components
22193 #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
22194 #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB \
22195 aarch64_components_for_bb
22197 #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
22198 #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS \
22199 aarch64_disqualify_components
22201 #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
22202 #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
22203 aarch64_emit_prologue_components
22205 #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
22206 #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
22207 aarch64_emit_epilogue_components
22209 #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
22210 #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS \
22211 aarch64_set_handled_components
22213 #undef TARGET_TRAMPOLINE_INIT
22214 #define TARGET_TRAMPOLINE_INIT aarch64_trampoline_init
22216 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
22217 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P aarch64_use_blocks_for_constant_p
22219 #undef TARGET_VECTOR_MODE_SUPPORTED_P
22220 #define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
22222 #undef TARGET_COMPATIBLE_VECTOR_TYPES_P
22223 #define TARGET_COMPATIBLE_VECTOR_TYPES_P aarch64_compatible_vector_types_p
22225 #undef TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT
22226 #define TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT \
22227 aarch64_builtin_support_vector_misalignment
22229 #undef TARGET_ARRAY_MODE
22230 #define TARGET_ARRAY_MODE aarch64_array_mode
22232 #undef TARGET_ARRAY_MODE_SUPPORTED_P
22233 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
22235 #undef TARGET_VECTORIZE_ADD_STMT_COST
22236 #define TARGET_VECTORIZE_ADD_STMT_COST aarch64_add_stmt_cost
22238 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
22239 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
22240 aarch64_builtin_vectorization_cost
22242 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
22243 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
22245 #undef TARGET_VECTORIZE_BUILTINS
22246 #define TARGET_VECTORIZE_BUILTINS
22248 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
22249 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
22250 aarch64_builtin_vectorized_function
22252 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_MODES
22253 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_MODES \
22254 aarch64_autovectorize_vector_modes
22256 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
22257 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV \
22258 aarch64_atomic_assign_expand_fenv
22260 /* Section anchor support. */
22262 #undef TARGET_MIN_ANCHOR_OFFSET
22263 #define TARGET_MIN_ANCHOR_OFFSET -256
22265 /* Limit the maximum anchor offset to 4k-1, since that's the limit for a
22266 byte offset; we can do much more for larger data types, but have no way
22267 to determine the size of the access. We assume accesses are aligned. */
22268 #undef TARGET_MAX_ANCHOR_OFFSET
22269 #define TARGET_MAX_ANCHOR_OFFSET 4095
22271 #undef TARGET_VECTOR_ALIGNMENT
22272 #define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
22274 #undef TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
22275 #define TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT \
22276 aarch64_vectorize_preferred_vector_alignment
22277 #undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
22278 #define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
22279 aarch64_simd_vector_alignment_reachable
22281 /* vec_perm support. */
22283 #undef TARGET_VECTORIZE_VEC_PERM_CONST
22284 #define TARGET_VECTORIZE_VEC_PERM_CONST \
22285 aarch64_vectorize_vec_perm_const
22287 #undef TARGET_VECTORIZE_RELATED_MODE
22288 #define TARGET_VECTORIZE_RELATED_MODE aarch64_vectorize_related_mode
22289 #undef TARGET_VECTORIZE_GET_MASK_MODE
22290 #define TARGET_VECTORIZE_GET_MASK_MODE aarch64_get_mask_mode
22291 #undef TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE
22292 #define TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE \
22293 aarch64_empty_mask_is_expensive
22294 #undef TARGET_PREFERRED_ELSE_VALUE
22295 #define TARGET_PREFERRED_ELSE_VALUE \
22296 aarch64_preferred_else_value
22298 #undef TARGET_INIT_LIBFUNCS
22299 #define TARGET_INIT_LIBFUNCS aarch64_init_libfuncs
22301 #undef TARGET_FIXED_CONDITION_CODE_REGS
22302 #define TARGET_FIXED_CONDITION_CODE_REGS aarch64_fixed_condition_code_regs
22304 #undef TARGET_FLAGS_REGNUM
22305 #define TARGET_FLAGS_REGNUM CC_REGNUM
22307 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
22308 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
22310 #undef TARGET_ASAN_SHADOW_OFFSET
22311 #define TARGET_ASAN_SHADOW_OFFSET aarch64_asan_shadow_offset
22313 #undef TARGET_LEGITIMIZE_ADDRESS
22314 #define TARGET_LEGITIMIZE_ADDRESS aarch64_legitimize_address
22316 #undef TARGET_SCHED_CAN_SPECULATE_INSN
22317 #define TARGET_SCHED_CAN_SPECULATE_INSN aarch64_sched_can_speculate_insn
22319 #undef TARGET_CAN_USE_DOLOOP_P
22320 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
22322 #undef TARGET_SCHED_ADJUST_PRIORITY
22323 #define TARGET_SCHED_ADJUST_PRIORITY aarch64_sched_adjust_priority
22325 #undef TARGET_SCHED_MACRO_FUSION_P
22326 #define TARGET_SCHED_MACRO_FUSION_P aarch64_macro_fusion_p
22328 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
22329 #define TARGET_SCHED_MACRO_FUSION_PAIR_P aarch_macro_fusion_pair_p
22331 #undef TARGET_SCHED_FUSION_PRIORITY
22332 #define TARGET_SCHED_FUSION_PRIORITY aarch64_sched_fusion_priority
22334 #undef TARGET_UNSPEC_MAY_TRAP_P
22335 #define TARGET_UNSPEC_MAY_TRAP_P aarch64_unspec_may_trap_p
22337 #undef TARGET_USE_PSEUDO_PIC_REG
22338 #define TARGET_USE_PSEUDO_PIC_REG aarch64_use_pseudo_pic_reg
22340 #undef TARGET_PRINT_OPERAND
22341 #define TARGET_PRINT_OPERAND aarch64_print_operand
22343 #undef TARGET_PRINT_OPERAND_ADDRESS
22344 #define TARGET_PRINT_OPERAND_ADDRESS aarch64_print_operand_address
22346 #undef TARGET_OPTAB_SUPPORTED_P
22347 #define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p
22349 #undef TARGET_OMIT_STRUCT_RETURN_REG
22350 #define TARGET_OMIT_STRUCT_RETURN_REG true
22352 #undef TARGET_DWARF_POLY_INDETERMINATE_VALUE
22353 #define TARGET_DWARF_POLY_INDETERMINATE_VALUE \
22354 aarch64_dwarf_poly_indeterminate_value
22356 /* The architecture reserves bits 0 and 1 so use bit 2 for descriptors. */
22357 #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
22358 #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 4
22360 #undef TARGET_HARD_REGNO_NREGS
22361 #define TARGET_HARD_REGNO_NREGS aarch64_hard_regno_nregs
22362 #undef TARGET_HARD_REGNO_MODE_OK
22363 #define TARGET_HARD_REGNO_MODE_OK aarch64_hard_regno_mode_ok
22365 #undef TARGET_MODES_TIEABLE_P
22366 #define TARGET_MODES_TIEABLE_P aarch64_modes_tieable_p
22368 #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
22369 #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
22370 aarch64_hard_regno_call_part_clobbered
22372 #undef TARGET_INSN_CALLEE_ABI
22373 #define TARGET_INSN_CALLEE_ABI aarch64_insn_callee_abi
22375 #undef TARGET_CONSTANT_ALIGNMENT
22376 #define TARGET_CONSTANT_ALIGNMENT aarch64_constant_alignment
22378 #undef TARGET_STACK_CLASH_PROTECTION_ALLOCA_PROBE_RANGE
22379 #define TARGET_STACK_CLASH_PROTECTION_ALLOCA_PROBE_RANGE \
22380 aarch64_stack_clash_protection_alloca_probe_range
22382 #undef TARGET_COMPUTE_PRESSURE_CLASSES
22383 #define TARGET_COMPUTE_PRESSURE_CLASSES aarch64_compute_pressure_classes
22385 #undef TARGET_CAN_CHANGE_MODE_CLASS
22386 #define TARGET_CAN_CHANGE_MODE_CLASS aarch64_can_change_mode_class
22388 #undef TARGET_SELECT_EARLY_REMAT_MODES
22389 #define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
22391 #undef TARGET_SPECULATION_SAFE_VALUE
22392 #define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value
22394 #undef TARGET_ESTIMATED_POLY_VALUE
22395 #define TARGET_ESTIMATED_POLY_VALUE aarch64_estimated_poly_value
22397 #undef TARGET_ATTRIBUTE_TABLE
22398 #define TARGET_ATTRIBUTE_TABLE aarch64_attribute_table
22400 #undef TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
22401 #define TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN \
22402 aarch64_simd_clone_compute_vecsize_and_simdlen
22404 #undef TARGET_SIMD_CLONE_ADJUST
22405 #define TARGET_SIMD_CLONE_ADJUST aarch64_simd_clone_adjust
22407 #undef TARGET_SIMD_CLONE_USABLE
22408 #define TARGET_SIMD_CLONE_USABLE aarch64_simd_clone_usable
22410 #undef TARGET_COMP_TYPE_ATTRIBUTES
22411 #define TARGET_COMP_TYPE_ATTRIBUTES aarch64_comp_type_attributes
22413 #undef TARGET_GET_MULTILIB_ABI_NAME
22414 #define TARGET_GET_MULTILIB_ABI_NAME aarch64_get_multilib_abi_name
22416 #undef TARGET_FNTYPE_ABI
22417 #define TARGET_FNTYPE_ABI aarch64_fntype_abi
22419 #if CHECKING_P
22420 #undef TARGET_RUN_TARGET_SELFTESTS
22421 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests
22424 #undef TARGET_ASM_POST_CFI_STARTPROC
22425 #define TARGET_ASM_POST_CFI_STARTPROC aarch64_post_cfi_startproc
22427 #undef TARGET_STRICT_ARGUMENT_NAMING
22428 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
22430 #undef TARGET_MD_ASM_ADJUST
22431 #define TARGET_MD_ASM_ADJUST arm_md_asm_adjust