| blob | 27a9a5fd28dc0a87fa44699da4a5aeed878d9a0f |
1 /* Machine description for AArch64 architecture.
2 Copyright (C) 2009-2019 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
24 #define INCLUDE_STRING
77 /* This file should be included last. */
80 /* Defined for convenience. */
81 #define POINTER_BYTES (POINTER_SIZE / BITS_PER_UNIT)
83 /* Information about a legitimate vector immediate operand. */
84 struct simd_immediate_info
85 {
97 /* The mode of the elements. */
98 scalar_mode elt_mode;
100 /* The instruction to use to move the immediate into a vector. */
101 insn_type insn;
103 union
104 {
105 /* For MOV and MVN. */
106 struct
107 {
108 /* The value of each element. */
109 rtx value;
111 /* The kind of shift modifier to use, and the number of bits to shift.
112 This is (LSL, 0) if no shift is needed. */
113 modifier_type modifier;
117 /* For INDEX. */
118 struct
119 {
120 /* The value of the first element and the step to be added for each
121 subsequent element. */
125 /* For PTRUE. */
126 aarch64_svpattern pattern;
128 };
130 /* Construct a floating-point immediate in which each element has mode
131 ELT_MODE_IN and value VALUE_IN. */
132 inline simd_immediate_info
135 {
139 }
141 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
142 and value VALUE_IN. The other parameters are as for the structure
143 fields. */
144 inline simd_immediate_info
150 {
154 }
156 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
157 and where element I is equal to BASE_IN + I * STEP_IN. */
158 inline simd_immediate_info
161 {
164 }
166 /* Construct a predicate that controls elements of mode ELT_MODE_IN
167 and has PTRUE pattern PATTERN_IN. */
168 inline simd_immediate_info
170 aarch64_svpattern pattern_in)
172 {
174 }
176 /* The current code model. */
179 /* The number of 64-bit elements in an SVE vector. */
180 poly_uint16 aarch64_sve_vg;
182 #ifdef HAVE_AS_TLS
183 #undef TARGET_HAVE_TLS
184 #define TARGET_HAVE_TLS 1
185 #endif
189 const_tree,
198 const_tree type,
203 aarch64_addr_query_type);
206 /* Major revision number of the ARM Architecture implemented by the target. */
209 /* The processor for which instructions should be scheduled. */
212 /* Mask to specify which instruction scheduling options should be used. */
215 /* Global flag for PC relative loads. */
218 /* Global flag for whether frame pointer is enabled. */
221 #define BRANCH_PROTECT_STR_MAX 255
224 static enum aarch64_parse_opt_result
227 /* Support for command line parsing of boolean flags in the tuning
228 structures. */
229 struct aarch64_flag_desc
230 {
233 };
235 #define AARCH64_FUSION_PAIR(name, internal_name) \
236 { name, AARCH64_FUSE_##internal_name },
238 {
243 };
245 #define AARCH64_EXTRA_TUNING_OPTION(name, internal_name) \
246 { name, AARCH64_EXTRA_TUNE_##internal_name },
248 {
253 };
255 /* Tuning parameters. */
258 {
259 {
264 },
271 };
274 {
275 {
280 },
287 };
290 {
291 {
296 },
303 };
306 {
307 {
312 },
319 };
322 {
323 {
328 },
335 };
338 {
339 {
344 },
351 };
354 {
356 /* Avoid the use of slow int<->fp moves for spilling by setting
357 their cost higher than memmov_cost. */
361 };
364 {
366 /* Avoid the use of slow int<->fp moves for spilling by setting
367 their cost higher than memmov_cost. */
371 };
374 {
376 /* Avoid the use of slow int<->fp moves for spilling by setting
377 their cost higher than memmov_cost. */
381 };
384 {
386 /* Avoid the use of slow int<->fp moves for spilling by setting
387 their cost higher than memmov_cost (actual, 4 and 9). */
391 };
394 {
399 };
402 {
404 /* Avoid the use of slow int<->fp moves for spilling by setting
405 their cost higher than memmov_cost. */
409 };
412 {
414 /* Avoid the use of int<->fp moves for spilling. */
418 };
421 {
423 /* Avoid the use of int<->fp moves for spilling. */
427 };
430 {
432 /* Avoid the use of slow int<->fp moves for spilling by setting
433 their cost higher than memmov_cost. */
437 };
439 /* Generic costs for vector insn classes. */
441 {
457 };
459 /* QDF24XX costs for vector insn classes. */
461 {
477 };
479 /* ThunderX costs for vector insn classes. */
481 {
497 };
500 {
516 };
518 /* Generic costs for vector insn classes. */
520 {
536 };
539 {
555 };
557 /* Generic costs for vector insn classes. */
559 {
575 };
577 /* Costs for vector insn classes for Vulcan. */
579 {
595 };
597 /* Generic costs for branch instructions. */
599 {
602 };
604 /* Generic approximation modes. */
606 {
609 AARCH64_APPROX_NONE /* recip_sqrt */
610 };
612 /* Approximation modes for Exynos M1. */
614 {
617 AARCH64_APPROX_ALL /* recip_sqrt */
618 };
620 /* Approximation modes for X-Gene 1. */
622 {
625 AARCH64_APPROX_ALL /* recip_sqrt */
626 };
628 /* Generic prefetch settings (which disable prefetch). */
630 {
638 };
641 {
649 };
652 {
660 };
663 {
671 };
674 {
682 };
685 {
693 };
696 {
704 };
707 {
715 };
718 {
740 &generic_prefetch_tune
741 };
744 {
767 &generic_prefetch_tune
768 };
771 {
794 &generic_prefetch_tune
795 };
798 {
821 &generic_prefetch_tune
822 };
825 {
848 &generic_prefetch_tune
849 };
852 {
875 &generic_prefetch_tune
876 };
881 {
903 &exynosm1_prefetch_tune
904 };
907 {
929 &thunderxt88_prefetch_tune
930 };
933 {
954 (AARCH64_EXTRA_TUNE_SLOW_UNALIGNED_LDPW
956 &thunderx_prefetch_tune
957 };
960 {
983 &tsv110_prefetch_tune
984 };
987 {
1009 &xgene1_prefetch_tune
1010 };
1013 {
1020 SVE_NOT_IMPLEMENTED,
1035 &xgene1_prefetch_tune
1036 };
1039 {
1062 &qdf24xx_prefetch_tune
1063 };
1065 /* Tuning structure for the Qualcomm Saphira core. Default to falkor values
1066 for now. */
1068 {
1091 &generic_prefetch_tune
1092 };
1095 {
1118 &thunderx2t99_prefetch_tune
1119 };
1122 {
1144 &generic_prefetch_tune
1145 };
1147 /* Support for fine-grained override of the tuning structures. */
1148 struct aarch64_tuning_override_function
1149 {
1152 };
1158 static const struct aarch64_tuning_override_function
1159 aarch64_tuning_override_functions[] =
1160 {
1165 };
1167 /* A processor implementing AArch64. */
1168 struct processor
1169 {
1177 };
1179 /* Architectures implementing AArch64. */
1181 {
1182 #define AARCH64_ARCH(NAME, CORE, ARCH_IDENT, ARCH_REV, FLAGS) \
1183 {NAME, CORE, CORE, AARCH64_ARCH_##ARCH_IDENT, ARCH_REV, FLAGS, NULL},
1186 };
1188 /* Processor cores implementing AArch64. */
1190 {
1191 #define AARCH64_CORE(NAME, IDENT, SCHED, ARCH, FLAGS, COSTS, IMP, PART, VARIANT) \
1192 {NAME, IDENT, SCHED, AARCH64_ARCH_##ARCH, \
1193 all_architectures[AARCH64_ARCH_##ARCH].architecture_version, \
1194 FLAGS, &COSTS##_tunings},
1199 };
1202 /* Target specification. These are populated by the -march, -mtune, -mcpu
1203 handling code or by target attributes. */
1210 /* The current tuning set. */
1213 /* Table of machine attributes. */
1215 {
1216 /* { name, min_len, max_len, decl_req, type_req, fn_type_req,
1217 affects_type_identity, handler, exclude } */
1220 };
1222 #define AARCH64_CPU_DEFAULT_FLAGS ((selected_cpu) ? selected_cpu->flags : 0)
1224 /* An ISA extension in the co-processor and main instruction set space. */
1225 struct aarch64_option_extension
1226 {
1230 };
1233 {
1237 }
1238 aarch64_cc;
1240 #define AARCH64_INVERSE_CONDITION_CODE(X) ((aarch64_cc) (((int) X) ^ 1))
1242 struct aarch64_branch_protect_type
1243 {
1244 /* The type's name that the user passes to the branch-protection option
1245 string. */
1247 /* Function to handle the protection type and set global variables.
1248 First argument is the string token corresponding with this type and the
1249 second argument is the next token in the option string.
1250 Return values:
1251 * AARCH64_PARSE_OK: Handling was sucessful.
1252 * AARCH64_INVALID_ARG: The type is invalid in this context and the caller
1253 should print an error.
1254 * AARCH64_INVALID_FEATURE: The type is invalid and the handler prints its
1255 own error. */
1257 /* A list of types that can follow this type in the option string. */
1260 };
1262 static enum aarch64_parse_opt_result
1264 {
1268 {
1271 }
1273 }
1275 static enum aarch64_parse_opt_result
1277 {
1282 {
1285 }
1287 }
1289 static enum aarch64_parse_opt_result
1292 {
1296 }
1298 static enum aarch64_parse_opt_result
1301 {
1304 }
1306 static enum aarch64_parse_opt_result
1309 {
1312 }
1314 static enum aarch64_parse_opt_result
1317 {
1320 }
1326 };
1335 };
1337 /* The condition codes of the processor, and the inverse function. */
1339 {
1342 };
1344 /* The preferred condition codes for SVE conditions. */
1346 {
1349 };
1351 /* Return the assembly token for svpattern value VALUE. */
1355 {
1357 {
1358 #define CASE(UPPER, LOWER, VALUE) case AARCH64_SV_##UPPER: return #LOWER;
1360 #undef CASE
1363 }
1365 }
1367 /* Generate code to enable conditional branches in functions over 1 MiB. */
1371 {
1388 }
1390 void
1392 {
1397 else
1400 else
1404 else
1407 }
1409 /* Implement TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS.
1410 The register allocator chooses POINTER_AND_FP_REGS if FP_REGS and
1411 GENERAL_REGS have the same cost - even if POINTER_AND_FP_REGS has a much
1412 higher cost. POINTER_AND_FP_REGS is also used if the cost of both FP_REGS
1413 and GENERAL_REGS is lower than the memory cost (in this case the best class
1414 is the lowest cost one). Using POINTER_AND_FP_REGS irrespectively of its
1415 cost results in bad allocations with many redundant int<->FP moves which
1416 are expensive on various cores.
1417 To avoid this we don't allow POINTER_AND_FP_REGS as the allocno class, but
1418 force a decision between FP_REGS and GENERAL_REGS. We use the allocno class
1419 if it isn't POINTER_AND_FP_REGS. Similarly, use the best class if it isn't
1420 POINTER_AND_FP_REGS. Otherwise set the allocno class depending on the mode.
1421 The result of this is that it is no longer inefficient to have a higher
1422 memory move cost than the register move cost.
1423 */
1425 static reg_class_t
1427 reg_class_t best_class)
1428 {
1429 machine_mode mode;
1441 }
1443 static unsigned int
1445 {
1449 }
1451 /* Return the reassociation width of treeop OPC with mode MODE. */
1452 static int
1454 {
1459 /* Avoid reassociating floating point addition so we emit more FMAs. */
1463 }
1465 /* Provide a mapping from gcc register numbers to dwarf register numbers. */
1466 unsigned
1468 {
1480 /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
1481 equivalent DWARF register. */
1483 }
1485 /* Return true if MODE is any of the Advanced SIMD structure modes. */
1486 static bool
1488 {
1491 }
1493 /* Return true if MODE is an SVE predicate mode. */
1494 static bool
1496 {
1502 }
1504 /* Three mutually-exclusive flags describing a vector or predicate type. */
1508 /* Can be used in combination with VEC_ADVSIMD or VEC_SVE_DATA to indicate
1509 a structure of 2, 3 or 4 vectors. */
1511 /* Useful combinations of the above. */
1515 /* Return a set of flags describing the vector properties of mode MODE.
1516 Ignore modes that are not supported by the current target. */
1517 static unsigned int
1519 {
1526 /* Make the decision based on the mode's enum value rather than its
1527 properties, so that we keep the correct classification regardless
1528 of -msve-vector-bits. */
1530 {
1531 /* Single SVE vectors. */
1541 /* x2 SVE vectors. */
1549 /* x3 SVE vectors. */
1557 /* x4 SVE vectors. */
1567 /* 64-bit Advanced SIMD vectors. */
1571 /* ...E_V1DImode doesn't exist. */
1575 /* 128-bit Advanced SIMD vectors. */
1587 }
1588 }
1590 /* Return true if MODE is any of the data vector modes, including
1591 structure modes. */
1592 static bool
1594 {
1596 }
1598 /* Return true if MODE is an SVE data vector mode; either a single vector
1599 or a structure of vectors. */
1600 static bool
1602 {
1604 }
1606 /* Implement target hook TARGET_ARRAY_MODE. */
1607 static opt_machine_mode
1609 {
1616 }
1618 /* Implement target hook TARGET_ARRAY_MODE_SUPPORTED_P. */
1619 static bool
1622 {
1630 }
1632 /* Return the SVE predicate mode to use for elements that have
1633 ELEM_NBYTES bytes, if such a mode exists. */
1635 opt_machine_mode
1637 {
1639 {
1648 }
1650 }
1652 /* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
1654 static opt_machine_mode
1656 {
1658 {
1660 machine_mode pred_mode;
1663 }
1666 }
1668 /* Return the integer element mode associated with SVE mode MODE. */
1670 static scalar_int_mode
1672 {
1676 }
1678 /* Implement TARGET_PREFERRED_ELSE_VALUE. For binary operations,
1679 prefer to use the first arithmetic operand as the else value if
1680 the else value doesn't matter, since that exactly matches the SVE
1681 destructive merging form. For ternary operations we could either
1682 pick the first operand and use FMAD-like instructions or the last
1683 operand and use FMLA-like instructions; the latter seems more
1684 natural. */
1686 static tree
1688 {
1690 }
1692 /* Implement TARGET_HARD_REGNO_NREGS. */
1694 static unsigned int
1696 {
1697 /* ??? Logically we should only need to provide a value when
1698 HARD_REGNO_MODE_OK says that the combination is valid,
1699 but at the moment we need to handle all modes. Just ignore
1700 any runtime parts for registers that can't store them. */
1703 {
1717 }
1719 }
1721 /* Implement TARGET_HARD_REGNO_MODE_OK. */
1723 static bool
1725 {
1730 /* This must have the same size as _Unwind_Word. */
1741 /* The purpose of comparing with ptr_mode is to support the
1742 global register variable associated with the stack pointer
1743 register via the syntax of asm ("wsp") in ILP32. */
1750 {
1755 }
1757 {
1760 else
1762 }
1765 }
1767 /* Return true if this is a definition of a vectorized simd function. */
1769 static bool
1771 {
1772 tree fntype;
1780 /* Functions with the aarch64_vector_pcs attribute use the simd ABI. */
1785 }
1787 /* Return the mode a register save/restore should use. DImode for integer
1788 registers, DFmode for FP registers in non-SIMD functions (they only save
1789 the bottom half of a 128 bit register), or TFmode for FP registers in
1790 SIMD functions. */
1792 static machine_mode
1794 {
1796 ? E_DImode
1798 }
1800 /* Return true if the instruction is a call to a SIMD function, false
1801 if it is not a SIMD function or if we do not know anything about
1802 the function. */
1804 static bool
1806 {
1807 rtx symbol;
1808 rtx call;
1809 tree fndecl;
1821 }
1823 /* Implement TARGET_REMOVE_EXTRA_CALL_PRESERVED_REGS. If INSN calls
1824 a function that uses the SIMD ABI, take advantage of the extra
1825 call-preserved registers that the ABI provides. */
1827 void
1830 {
1832 {
1836 }
1837 }
1839 /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The callee only saves
1840 the lower 64 bits of a 128-bit register. Tell the compiler the callee
1841 clobbers the top 64 bits when restoring the bottom 64 bits. */
1843 static bool
1845 machine_mode mode)
1846 {
1850 }
1852 /* Implement TARGET_RETURN_CALL_WITH_MAX_CLOBBERS. */
1854 rtx_insn *
1856 {
1861 else
1863 }
1865 /* Implement REGMODE_NATURAL_SIZE. */
1866 poly_uint64
1868 {
1869 /* The natural size for SVE data modes is one SVE data vector,
1870 and similarly for predicates. We can't independently modify
1871 anything smaller than that. */
1872 /* ??? For now, only do this for variable-width SVE registers.
1873 Doing it for constant-sized registers breaks lower-subreg.c. */
1874 /* ??? And once that's fixed, we should probably have similar
1875 code for Advanced SIMD. */
1877 {
1883 }
1885 }
1887 /* Implement HARD_REGNO_CALLER_SAVE_MODE. */
1888 machine_mode
1890 machine_mode mode)
1891 {
1892 /* The predicate mode determines which bits are significant and
1893 which are "don't care". Decreasing the number of lanes would
1894 lose data while increasing the number of lanes would make bits
1895 unnecessarily significant. */
1900 else
1902 }
1904 /* Return true if I's bits are consecutive ones from the MSB. */
1905 bool
1907 {
1909 }
1911 /* Implement TARGET_CONSTANT_ALIGNMENT. Make strings word-aligned so
1912 that strcpy from constants will be faster. */
1914 static HOST_WIDE_INT
1916 {
1920 }
1922 /* Return true if calls to DECL should be treated as
1923 long-calls (ie called via a register). */
1924 static bool
1926 {
1928 }
1930 /* Return true if calls to symbol-ref SYM should be treated as
1931 long-calls (ie called via a register). */
1932 bool
1934 {
1936 }
1938 /* Return true if calls to symbol-ref SYM should not go through
1939 plt stubs. */
1941 bool
1943 {
1947 && decl
1948 && (!flag_plt
1954 }
1956 /* Return true if the offsets to a zero/sign-extract operation
1957 represent an expression that matches an extend operation. The
1958 operands represent the paramters from
1960 (extract:MODE (mult (reg) (MULT_IMM)) (EXTRACT_IMM) (const_int 0)). */
1961 bool
1963 rtx extract_imm)
1964 {
1981 }
1983 /* Emit an insn that's a simple single-set. Both the operands must be
1984 known to be valid. */
1987 {
1989 }
1991 /* X and Y are two things to compare using CODE. Emit the compare insn and
1992 return the rtx for register 0 in the proper mode. */
1993 rtx
1995 {
2001 }
2003 /* Similarly, but maybe zero-extend Y if Y_MODE < SImode. */
2005 static rtx
2007 machine_mode y_mode)
2008 {
2010 {
2013 else
2014 {
2016 machine_mode cc_mode;
2024 }
2025 }
2028 }
2030 /* Build the SYMBOL_REF for __tls_get_addr. */
2034 rtx
2036 {
2040 }
2042 /* Return the TLS model to use for ADDR. */
2044 static enum tls_model
2046 {
2049 {
2050 poly_int64 addend;
2054 }
2059 }
2061 /* We'll allow lo_sum's in addresses in our legitimate addresses
2062 so that combine would take care of combining addresses where
2063 necessary, but for generation purposes, we'll generate the address
2064 as :
2065 RTL Absolute
2066 tmp = hi (symbol_ref); adrp x1, foo
2067 dest = lo_sum (tmp, symbol_ref); add dest, x1, :lo_12:foo
2068 nop
2070 PIC TLS
2071 adrp x1, :got:foo adrp tmp, :tlsgd:foo
2072 ldr x1, [:got_lo12:foo] add dest, tmp, :tlsgd_lo12:foo
2073 bl __tls_get_addr
2074 nop
2076 Load TLS symbol, depending on TLS mechanism and TLS access model.
2078 Global Dynamic - Traditional TLS:
2079 adrp tmp, :tlsgd:imm
2080 add dest, tmp, #:tlsgd_lo12:imm
2081 bl __tls_get_addr
2083 Global Dynamic - TLS Descriptors:
2084 adrp dest, :tlsdesc:imm
2085 ldr tmp, [dest, #:tlsdesc_lo12:imm]
2086 add dest, dest, #:tlsdesc_lo12:imm
2087 blr tmp
2088 mrs tp, tpidr_el0
2089 add dest, dest, tp
2091 Initial Exec:
2092 mrs tp, tpidr_el0
2093 adrp tmp, :gottprel:imm
2094 ldr dest, [tmp, #:gottprel_lo12:imm]
2095 add dest, dest, tp
2097 Local Exec:
2098 mrs tp, tpidr_el0
2099 add t0, tp, #:tprel_hi12:imm, lsl #12
2100 add t0, t0, #:tprel_lo12_nc:imm
2101 */
2103 static void
2106 {
2108 {
2110 {
2111 /* In ILP32, the mode of dest can be either SImode or DImode. */
2123 }
2130 {
2133 rtx insn;
2134 rtx mem;
2136 /* NOTE: pic_offset_table_rtx can be NULL_RTX, because we can reach
2137 here before rtl expand. Tree IVOPT will generate rtl pattern to
2138 decide rtx costs, in which case pic_offset_table_rtx is not
2139 initialized. For that case no need to generate the first adrp
2140 instruction as the final cost for global variable access is
2141 one instruction. */
2143 {
2144 /* -fpic for -mcmodel=small allow 32K GOT table size (but we are
2145 using the page base as GOT base, the first page may be wasted,
2146 in the worst scenario, there is only 28K space for GOT).
2148 The generate instruction sequence for accessing global variable
2149 is:
2151 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym]
2153 Only one instruction needed. But we must initialize
2154 pic_offset_table_rtx properly. We generate initialize insn for
2155 every global access, and allow CSE to remove all redundant.
2157 The final instruction sequences will look like the following
2158 for multiply global variables access.
2160 adrp pic_offset_table_rtx, _GLOBAL_OFFSET_TABLE_
2162 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym1]
2163 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym2]
2164 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym3]
2165 ... */
2174 }
2177 {
2180 else
2184 }
2185 else
2186 {
2191 }
2193 /* The operand is expected to be MEM. Whenever the related insn
2194 pattern changed, above code which calculate mem should be
2195 updated. */
2201 }
2204 {
2205 /* In ILP32, the mode of dest can be either SImode or DImode,
2206 while the got entry is always of SImode size. The mode of
2207 dest depends on how dest is used: if dest is assigned to a
2208 pointer (e.g. in the memory), it has SImode; it may have
2209 DImode if dest is dereferenced to access the memeory.
2210 This is why we have to handle three different ldr_got_small
2211 patterns here (two patterns for ILP32). */
2213 rtx insn;
2214 rtx mem;
2223 {
2226 else
2230 }
2231 else
2232 {
2237 }
2244 }
2247 {
2255 else
2263 }
2266 {
2269 rtx tp;
2273 /* In ILP32, the got entry is always of SImode size. Unlike
2274 small GOT, the dest is fixed at reg 0. */
2277 else
2288 }
2291 {
2292 /* In ILP32, the mode of dest can be either SImode or DImode,
2293 while the got entry is always of SImode size. The mode of
2294 dest depends on how dest is used: if dest is assigned to a
2295 pointer (e.g. in the memory), it has SImode; it may have
2296 DImode if dest is dereferenced to access the memeory.
2297 This is why we have to handle three different tlsie_small
2298 patterns here (two patterns for ILP32). */
2304 {
2307 else
2308 {
2311 }
2312 }
2313 else
2314 {
2317 }
2323 }
2329 {
2337 {
2360 }
2365 }
2372 {
2377 {
2380 else
2381 {
2384 }
2385 }
2386 else
2387 {
2390 }
2395 }
2399 }
2400 }
2402 /* Emit a move from SRC to DEST. Assume that the move expanders can
2403 handle all moves if !can_create_pseudo_p (). The distinction is
2404 important because, unlike emit_move_insn, the move expanders know
2405 how to force Pmode objects into the constant pool even when the
2406 constant pool address is not itself legitimate. */
2407 static rtx
2409 {
2413 }
2415 /* Apply UNOPTAB to OP and store the result in DEST. */
2417 static void
2419 {
2423 }
2425 /* Apply BINOPTAB to OP0 and OP1 and store the result in DEST. */
2427 static void
2429 {
2431 OPTAB_DIRECT);
2434 }
2436 /* Split a 128-bit move operation into two 64-bit move operations,
2437 taking care to handle partial overlap of register to register
2438 copies. Special cases are needed when moving between GP regs and
2439 FP regs. SRC can be a register, constant or memory; DST a register
2440 or memory. If either operand is memory it must not have any side
2441 effects. */
2442 void
2444 {
2455 {
2459 /* Handle FP <-> GP regs. */
2461 {
2468 }
2470 {
2477 }
2478 }
2485 /* At most one pairing may overlap. */
2487 {
2490 }
2491 else
2492 {
2495 }
2496 }
2498 bool
2500 {
2503 }
2505 /* Split a complex SIMD combine. */
2507 void
2509 {
2520 }
2522 /* Split a complex SIMD move. */
2524 void
2526 {
2533 {
2536 }
2537 }
2539 bool
2542 {
2546 }
2549 /* Return TARGET if it is nonnull and a register of mode MODE.
2550 Otherwise, return a fresh register of mode MODE if we can,
2551 or TARGET reinterpreted as MODE if we can't. */
2553 static rtx
2555 {
2559 {
2562 }
2564 }
2566 /* Return a register that contains the constant in BUILDER, given that
2567 the constant is a legitimate move operand. Use TARGET as the register
2568 if it is nonnull and convenient. */
2570 static rtx
2572 {
2577 }
2579 static rtx
2581 {
2584 else
2585 {
2589 }
2590 }
2592 /* Return true if predicate value X is a constant in which every element
2593 is a CONST_INT. When returning true, describe X in BUILDER as a VNx16BI
2594 value, i.e. as a predicate in which all bits are significant. */
2596 static bool
2598 {
2610 {
2618 }
2621 }
2623 /* BUILDER contains a predicate constant of mode VNx16BI. Return the
2624 widest predicate element size it can have (that is, the largest size
2625 for which each element would still be 0 or 1). */
2627 unsigned int
2629 {
2630 /* Start with the most optimistic assumption: that we only need
2631 one bit per pattern. This is what we will use if only the first
2632 bit in each pattern is ever set. */
2636 /* Look for set bits. */
2640 {
2644 }
2646 }
2648 /* BUILDER is a predicate constant of mode VNx16BI. Consider the value
2649 that the constant would have with predicate element size ELT_SIZE
2650 (ignoring the upper bits in each element) and return:
2652 * -1 if all bits are set
2653 * N if the predicate has N leading set bits followed by all clear bits
2654 * 0 if the predicate does not have any of these forms. */
2656 int
2659 {
2660 /* If nelts_per_pattern is 3, we have set bits followed by clear bits
2661 followed by set bits. */
2665 /* Skip over leading set bits. */
2673 /* Check for the all-true case. */
2677 /* If nelts_per_pattern is 1, then either VL is zero, or we have a
2678 repeating pattern of set bits followed by clear bits. */
2682 /* We have a "foreground" value and a duplicated "background" value.
2683 If the background might repeat and the last set bit belongs to it,
2684 we might have set bits followed by clear bits followed by set bits. */
2688 /* Make sure that the rest are all clear. */
2694 }
2696 /* See if there is an svpattern that encodes an SVE predicate of mode
2697 PRED_MODE in which the first VL bits are set and the rest are clear.
2698 Return the pattern if so, otherwise return AARCH64_NUM_SVPATTERNS.
2699 A VL of -1 indicates an all-true vector. */
2701 aarch64_svpattern
2703 {
2718 {
2721 /* These would only trigger for non-power-of-2 lengths. */
2728 }
2730 }
2732 /* Return a VNx16BImode constant in which every sequence of ELT_SIZE
2733 bits has the lowest bit set and the upper bits clear. This is the
2734 VNx16BImode equivalent of a PTRUE for controlling elements of
2735 ELT_SIZE bytes. However, because the constant is VNx16BImode,
2736 all bits are significant, even the upper zeros. */
2738 rtx
2740 {
2746 }
2748 /* Return an all-true predicate register of mode MODE. */
2750 rtx
2752 {
2756 }
2758 /* Return an all-false predicate register of mode MODE. */
2760 rtx
2762 {
2766 }
2768 /* Return true if predicate PRED1[0] is true whenever predicate PRED2 is
2769 true, or alternatively if we know that the operation predicated by
2770 PRED1[0] is safe to perform whenever PRED2 is true. PRED1[1] is a
2771 aarch64_sve_gp_strictness operand that describes the operation
2772 predicated by PRED1[0]. */
2774 bool
2776 {
2784 }
2786 /* PRED1[0] is a PTEST predicate and PRED1[1] is an aarch64_sve_ptrue_flag
2787 for it. PRED2[0] is the predicate for the instruction whose result
2788 is tested by the PTEST and PRED2[1] is again an aarch64_sve_ptrue_flag
2789 for it. Return true if we can prove that the two predicates are
2790 equivalent for PTEST purposes; that is, if we can replace PRED2[0]
2791 with PRED1[0] without changing behavior. */
2793 bool
2795 {
2807 }
2809 /* Emit a comparison CMP between OP0 and OP1, both of which have mode
2810 DATA_MODE, and return the result in a predicate of mode PRED_MODE.
2811 Use TARGET as the target register if nonnull and convenient. */
2813 static rtx
2816 {
2826 }
2828 /* Use a comparison to convert integer vector SRC into MODE, which is
2829 the corresponding SVE predicate mode. Use TARGET for the result
2830 if it's nonnull and convenient. */
2832 static rtx
2834 {
2838 }
2840 /* Return true if we can move VALUE into a register using a single
2841 CNT[BHWD] instruction. */
2843 static bool
2845 {
2847 /* The coefficient must be [1, 16] * {2, 4, 8, 16}. */
2852 }
2854 /* Likewise for rtx X. */
2856 bool
2858 {
2859 poly_int64 value;
2861 }
2863 /* Return the asm string for an instruction with a CNT-like vector size
2864 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2865 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2866 first part of the operands template (the part that comes before the
2867 vector size itself). FACTOR is the number of quadwords.
2868 NELTS_PER_VQ, if nonzero, is the number of elements in each quadword.
2869 If it is zero, we can use any element size. */
2875 {
2879 /* There is some overlap in the ranges of the four CNT instructions.
2880 Here we always use the smallest possible element size, so that the
2881 multiplier is 1 whereever possible. */
2892 else
2897 }
2899 /* Return the asm string for an instruction with a CNT-like vector size
2900 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2901 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2902 first part of the operands template (the part that comes before the
2903 vector size itself). X is the value of the vector size operand,
2904 as a polynomial integer rtx. */
2908 rtx x)
2909 {
2914 }
2916 /* Return true if we can add VALUE to a register using a single ADDVL
2917 or ADDPL instruction. */
2919 static bool
2921 {
2925 /* FACTOR counts VG / 2, so a value of 2 is one predicate width
2926 and a value of 16 is one vector width. */
2929 }
2931 /* Likewise for rtx X. */
2933 bool
2935 {
2936 poly_int64 value;
2939 }
2941 /* Return the asm string for adding ADDVL or ADDPL immediate X to operand 1
2942 and storing the result in operand 0. */
2946 {
2951 /* Use INC or DEC if possible. */
2953 {
2960 }
2965 else
2968 }
2970 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2971 instruction. If it is, store the number of elements in each vector
2972 quadword in *NELTS_PER_VQ_OUT (if nonnull) and store the multiplication
2973 factor in *FACTOR_OUT (if nonnull). */
2975 bool
2978 {
2979 rtx elt;
2980 poly_int64 value;
2988 /* There's no vector INCB. */
2995 /* The coefficient must be [1, 16] * NELTS_PER_VQ. */
3005 }
3007 /* Return true if X is a valid immediate for an SVE vector INC or DEC
3008 instruction. */
3010 bool
3012 {
3014 }
3016 /* Return the asm template for an SVE vector INC or DEC instruction.
3017 OPERANDS gives the operands before the vector count and X is the
3018 value of the vector count operand itself. */
3022 {
3029 nelts_per_vq);
3030 else
3032 nelts_per_vq);
3033 }
3035 static int
3037 scalar_int_mode mode)
3038 {
3047 {
3051 }
3053 /* Check to see if the low 32 bits are either 0xffffXXXX or 0xXXXXffff
3054 (with XXXX non-zero). In that case check to see if the move can be done in
3055 a smaller mode. */
3060 {
3064 /* Check if we have to emit a second instruction by checking to see
3065 if any of the upper 32 bits of the original DI mode value is set. */
3076 }
3079 {
3081 {
3086 else
3089 }
3091 }
3093 /* Remaining cases are all for DImode. */
3102 {
3103 /* Try emitting a bitmask immediate with a movk replacing 16 bits.
3104 For a 64-bit bitmask try whether changing 16 bits to all ones or
3105 zeroes creates a valid bitmask. To check any repeated bitmask,
3106 try using 16 bits from the other 32-bit half of val. */
3109 {
3120 }
3122 {
3124 {
3128 }
3130 }
3131 }
3133 /* Generate 2-4 instructions, skipping 16 bits of all zeroes or ones which
3134 are emitted by the initial mov. If one_match > zero_match, skip set bits,
3135 otherwise skip zero bits. */
3147 {
3153 num_insns ++;
3154 }
3157 }
3159 /* Return whether imm is a 128-bit immediate which is simple enough to
3160 expand inline. */
3161 bool
3163 {
3174 }
3177 /* Return the number of temporary registers that aarch64_add_offset_1
3178 would need to add OFFSET to a register. */
3180 static unsigned int
3182 {
3184 }
3186 /* A subroutine of aarch64_add_offset. Set DEST to SRC + OFFSET for
3187 a non-polynomial OFFSET. MODE is the mode of the addition.
3188 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
3189 be set and CFA adjustments added to the generated instructions.
3191 TEMP1, if nonnull, is a register of mode MODE that can be used as a
3192 temporary if register allocation is already complete. This temporary
3193 register may overlap DEST but must not overlap SRC. If TEMP1 is known
3194 to hold abs (OFFSET), EMIT_MOVE_IMM can be set to false to avoid emitting
3195 the immediate again.
3197 Since this function may be used to adjust the stack pointer, we must
3198 ensure that it cannot cause transient stack deallocation (for example
3199 by first incrementing SP and then decrementing when adjusting by a
3200 large immediate). */
3202 static void
3206 {
3214 {
3216 {
3219 }
3221 }
3223 /* Single instruction adjustment. */
3225 {
3229 }
3231 /* Emit 2 additions/subtractions if the adjustment is less than 24 bits
3232 and either:
3234 a) the offset cannot be loaded by a 16-bit move or
3235 b) there is no spare register into which we can move it. */
3239 {
3248 }
3250 /* Emit a move immediate if required and an addition/subtraction. */
3252 {
3255 }
3260 {
3264 }
3265 }
3267 /* Return the number of temporary registers that aarch64_add_offset
3268 would need to move OFFSET into a register or add OFFSET to a register;
3269 ADD_P is true if we want the latter rather than the former. */
3271 static unsigned int
3273 {
3274 /* This follows the same structure as aarch64_add_offset. */
3283 /* Need one register for the ADDVL/ADDPL result. */
3286 {
3289 /* Need one register for the CNT result and one for the multiplication
3290 factor. If necessary, the second temporary can be reused for the
3291 constant part of the offset. */
3293 /* Need one register for the CNT result (which might then
3294 be shifted). */
3296 }
3298 }
3300 /* If X can be represented as a poly_int64, return the number
3301 of temporaries that are required to add it to a register.
3302 Return -1 otherwise. */
3304 int
3306 {
3307 poly_int64 offset;
3311 }
3313 /* Set DEST to SRC + OFFSET. MODE is the mode of the addition.
3314 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
3315 be set and CFA adjustments added to the generated instructions.
3317 TEMP1, if nonnull, is a register of mode MODE that can be used as a
3318 temporary if register allocation is already complete. This temporary
3319 register may overlap DEST if !FRAME_RELATED_P but must not overlap SRC.
3320 If TEMP1 is known to hold abs (OFFSET), EMIT_MOVE_IMM can be set to
3321 false to avoid emitting the immediate again.
3323 TEMP2, if nonnull, is a second temporary register that doesn't
3324 overlap either DEST or REG.
3326 Since this function may be used to adjust the stack pointer, we must
3327 ensure that it cannot cause transient stack deallocation (for example
3328 by first incrementing SP and then decrementing when adjusting by a
3329 large immediate). */
3331 static void
3335 {
3339 || !frame_related_p
3343 /* Try using ADDVL or ADDPL to add the whole value. */
3345 {
3350 }
3352 /* Coefficient 1 is multiplied by the number of 128-bit blocks in an
3353 SVE vector register, over and above the minimum size of 128 bits.
3354 This is equivalent to half the value returned by CNTD with a
3355 vector shape of ALL. */
3359 /* Try using ADDVL or ADDPL to add the VG-based part. */
3363 {
3366 {
3370 }
3371 else
3372 {
3377 }
3378 }
3379 /* Otherwise use a CNT-based sequence. */
3381 {
3382 /* Use a subtraction if we have a negative factor. */
3385 {
3388 }
3390 /* Calculate CNTD * FACTOR / 2. First try to fold the division
3391 into the multiplication. */
3392 rtx val;
3395 /* Use a right shift by 1. */
3397 else
3401 {
3403 {
3404 /* "CNTB Xn, ALL, MUL #FACTOR" is out of range, so calculate
3405 the value with the minimum multiplier and shift it into
3406 position. */
3410 }
3412 }
3413 else
3414 {
3415 /* Use CNTD, then multiply it by FACTOR. */
3419 /* Go back to using a negative multiplication factor if we have
3420 no register from which to subtract. */
3422 {
3425 }
3429 }
3432 {
3433 /* Multiply by 1 << SHIFT. */
3436 }
3438 {
3439 /* Divide by 2. */
3442 }
3444 /* Calculate SRC +/- CNTD * FACTOR / 2. */
3446 {
3449 }
3451 {
3454 }
3457 {
3460 {
3464 poly_offset)));
3465 }
3469 }
3470 else
3471 {
3475 }
3478 }
3482 }
3484 /* Like aarch64_add_offset, but the offset is given as an rtx rather
3485 than a poly_int64. */
3487 void
3490 {
3493 }
3495 /* Add DELTA to the stack pointer, marking the instructions frame-related.
3496 TEMP1 is available as a temporary if nonnull. EMIT_MOVE_IMM is false
3497 if TEMP1 already contains abs (DELTA). */
3501 {
3504 }
3506 /* Subtract DELTA from the stack pointer, marking the instructions
3507 frame-related if FRAME_RELATED_P. TEMP1 is available as a temporary
3508 if nonnull. */
3513 {
3516 }
3518 /* Set DEST to (vec_series BASE STEP). */
3520 static void
3522 {
3526 /* Each operand can be a register or an immediate in the range [-16, 15]. */
3533 }
3535 /* Duplicate 128-bit Advanced SIMD vector SRC so that it fills an SVE
3536 register of mode MODE. Use TARGET for the result if it's nonnull
3537 and convenient.
3539 The two vector modes must have the same element mode. The behavior
3540 is to duplicate architectural lane N of SRC into architectural lanes
3541 N + I * STEP of the result. On big-endian targets, architectural
3542 lane 0 of an Advanced SIMD vector is the last element of the vector
3543 in memory layout, so for big-endian targets this operation has the
3544 effect of reversing SRC before duplicating it. Callers need to
3545 account for this. */
3547 rtx
3549 {
3552 insn_code icode = (BYTES_BIG_ENDIAN
3561 {
3562 /* Create a PARALLEL describing the reversal of SRC. */
3567 }
3570 }
3572 /* Try to force 128-bit vector value SRC into memory and use LD1RQ to fetch
3573 the memory image into DEST. Return true on success. */
3575 static bool
3577 {
3582 /* Make sure that the address is legitimate. */
3584 {
3587 }
3595 }
3597 /* Return a register containing CONST_VECTOR SRC, given that SRC has an
3598 SVE data mode and isn't a legitimate constant. Use TARGET for the
3599 result if convenient.
3601 The returned register can have whatever mode seems most natural
3602 given the contents of SRC. */
3604 static rtx
3606 {
3615 {
3616 /* The constant is a duplicated quadword but can't be narrowed
3617 beyond a quadword. Get the memory image of the first quadword
3618 as a 128-bit vector and try using LD1RQ to load it from memory.
3620 The effect for both endiannesses is to load memory lane N into
3621 architectural lanes N + I * STEP of the result. On big-endian
3622 targets, the layout of the 128-bit vector in an Advanced SIMD
3623 register would be different from its layout in an SVE register,
3624 but this 128-bit vector is a memory value only. */
3629 }
3632 {
3633 /* The vector is a repeating sequence of 64 bits or fewer.
3634 See if we can load them using an Advanced SIMD move and then
3635 duplicate it to fill a vector. This is better than using a GPR
3636 move because it keeps everything in the same register file. */
3640 {
3641 /* We want memory lane N to go into architectural lane N,
3642 so reverse for big-endian targets. The DUP .Q pattern
3643 has a compensating reverse built-in. */
3646 }
3649 {
3652 }
3654 /* Get an integer representation of the repeating part of Advanced
3655 SIMD vector VQ_SRC. This preserves the endianness of VQ_SRC,
3656 which for big-endian targets is lane-swapped wrt a normal
3657 Advanced SIMD vector. This means that for both endiannesses,
3658 memory lane N of SVE vector SRC corresponds to architectural
3659 lane N of a register holding VQ_SRC. This in turn means that
3660 memory lane 0 of SVE vector SRC is in the lsb of VQ_SRC (viewed
3661 as a single 128-bit value) and thus that memory lane 0 of SRC is
3662 in the lsb of the integer. Duplicating the integer therefore
3663 ensures that memory lane N of SRC goes into architectural lane
3664 N + I * INDEX of the SVE register. */
3668 {
3669 /* Pretend that we had a vector of INT_MODE to start with. */
3673 /* If the integer can be moved into a general register by a
3674 single instruction, do that and duplicate the result. */
3677 {
3680 }
3681 }
3683 /* We're duplicating a single value, but can't do better than
3684 force it to memory and load from there. This handles things
3685 like symbolic constants. */
3689 {
3690 /* Load the element from memory if we can, otherwise move it into
3691 a register and use a DUP. */
3696 }
3697 }
3699 /* Try using INDEX. */
3702 {
3705 }
3707 /* From here on, it's better to force the whole constant to memory
3708 if we can. */
3712 /* Expand each pattern individually. */
3714 rtx_vector_builder builder;
3717 {
3722 }
3724 /* Use permutes to interleave the separate vectors. */
3726 {
3729 {
3734 }
3735 }
3738 }
3740 /* Use WHILE to set a predicate register of mode MODE in which the first
3741 VL bits are set and the rest are clear. Use TARGET for the register
3742 if it's nonnull and convenient. */
3744 static rtx
3747 {
3752 }
3754 static rtx
3757 /* BUILDER is a constant predicate in which the index of every set bit
3758 is a multiple of ELT_SIZE (which is <= 8). Try to load the constant
3759 by inverting every element at a multiple of ELT_SIZE and EORing the
3760 result with an ELT_SIZE PTRUE.
3762 Return a register that contains the constant on success, otherwise
3763 return null. Use TARGET as the register if it is nonnull and
3764 convenient. */
3766 static rtx
3769 {
3770 /* Invert every element at a multiple of ELT_SIZE, keeping the
3771 other bits zero. */
3777 else
3781 /* See if we can load the constant cheaply. */
3786 /* EOR the result with an ELT_SIZE PTRUE. */
3792 }
3794 /* BUILDER is a constant predicate in which the index of every set bit
3795 is a multiple of ELT_SIZE (which is <= 8). Try to load the constant
3796 using a TRN1 of size PERMUTE_SIZE, which is >= ELT_SIZE. Return the
3797 register on success, otherwise return null. Use TARGET as the register
3798 if nonnull and convenient. */
3800 static rtx
3804 {
3805 /* We're going to split the constant into two new constants A and B,
3806 with element I of BUILDER going into A if (I & PERMUTE_SIZE) == 0
3807 and into B otherwise. E.g. for PERMUTE_SIZE == 4 && ELT_SIZE == 1:
3809 A: { 0, 1, 2, 3, _, _, _, _, 8, 9, 10, 11, _, _, _, _ }
3810 B: { 4, 5, 6, 7, _, _, _, _, 12, 13, 14, 15, _, _, _, _ }
3812 where _ indicates elements that will be discarded by the permute.
3814 First calculate the ELT_SIZEs for A and B. */
3819 {
3822 else
3824 }
3828 /* Now construct the vectors themselves. */
3836 {
3839 }
3841 {
3842 /* The A and B elements are significant. */
3845 }
3846 else
3847 {
3848 /* The A and B elements are going to be discarded, so pick whatever
3849 is likely to give a nice constant. We are targeting element
3850 sizes A_ELT_SIZE and B_ELT_SIZE for A and B respectively,
3851 with the aim of each being a sequence of ones followed by
3852 a sequence of zeros. So:
3854 * if X_ELT_SIZE <= PERMUTE_SIZE, the best approach is to
3855 duplicate the last X_ELT_SIZE element, to extend the
3856 current sequence of ones or zeros.
3858 * if X_ELT_SIZE > PERMUTE_SIZE, the best approach is to add a
3859 zero, so that the constant really does have X_ELT_SIZE and
3860 not a smaller size. */
3863 else
3867 else
3869 }
3873 /* Try loading A into a register. */
3879 /* Try loading B into a register. */
3882 {
3885 {
3888 }
3889 }
3891 /* Emit the TRN1 itself. */
3898 }
3900 /* Subroutine of aarch64_expand_sve_const_pred. Try to load the VNx16BI
3901 constant in BUILDER into an SVE predicate register. Return the register
3902 on success, otherwise return null. Use TARGET for the register if
3903 nonnull and convenient.
3905 ALLOW_RECURSE_P is true if we can use methods that would call this
3906 function recursively. */
3908 static rtx
3911 {
3913 /* A PFALSE or a PTRUE .B ALL. */
3918 {
3919 /* If we can load the constant using PTRUE, use it as-is. */
3924 /* Otherwise use WHILE to set the first VL bits. */
3926 }
3931 /* Try inverting the vector in element size ELT_SIZE and then EORing
3932 the result with an ELT_SIZE PTRUE. */
3935 elt_size))
3938 /* Try using TRN1 to permute two simpler constants. */
3945 }
3947 /* Return an SVE predicate register that contains the VNx16BImode
3948 constant in BUILDER, without going through the move expanders.
3950 The returned register can have whatever mode seems most natural
3951 given the contents of BUILDER. Use TARGET for the result if
3952 convenient. */
3954 static rtx
3956 {
3957 /* Try loading the constant using pure predicate operations. */
3961 /* Try forcing the constant to memory. */
3964 {
3968 }
3970 /* The last resort is to load the constant as an integer and then
3971 compare it against zero. Use -1 for set bits in order to increase
3972 the changes of using SVE DUPM or an Advanced SIMD byte mask. */
3980 }
3982 /* Set DEST to immediate IMM. */
3984 void
3986 {
3989 /* Check on what type of symbol it is. */
3990 scalar_int_mode int_mode;
3996 {
3997 rtx mem;
3998 poly_int64 offset;
3999 HOST_WIDE_INT const_offset;
4002 /* If we have (const (plus symbol offset)), separate out the offset
4003 before we start classifying the symbol. */
4006 /* We must always add an offset involving VL separately, rather than
4007 folding it into the relocation. */
4009 {
4012 else
4013 {
4014 /* Do arithmetic on 32-bit values if the result is smaller
4015 than that. */
4017 {
4018 /* It is invalid to do symbol calculations in modes
4019 narrower than SImode. */
4023 }
4025 {
4029 }
4030 else
4033 }
4035 }
4039 {
4043 {
4049 }
4054 /* If we aren't generating PC relative literals, then
4055 we need to expand the literal pool access carefully.
4056 This is something that needs to be done in a number
4057 of places, so could well live as a separate function. */
4059 {
4066 }
4083 {
4089 }
4090 /* FALLTHRU */
4103 }
4104 }
4107 {
4109 {
4110 /* Only the low bit of each .H, .S and .D element is defined,
4111 so we can set the upper bits to whatever we like. If the
4112 predicate is all-true in MODE, prefer to set all the undefined
4113 bits as well, so that we can share a single .B predicate for
4114 all modes. */
4118 /* All methods for constructing predicate modes wider than VNx16BI
4119 will set the upper bits of each element to zero. Expose this
4120 by moving such constants as a VNx16BI, so that all bits are
4121 significant and so that constants for different modes can be
4122 shared. The wider constant will still be available as a
4123 REG_EQUAL note. */
4124 rtx_vector_builder builder;
4126 {
4131 }
4132 }
4136 {
4139 }
4143 {
4147 }
4153 }
4157 }
4159 /* Emit an SVE predicated move from SRC to DEST. PRED is a predicate
4160 that is known to contain PTRUE. */
4162 void
4164 {
4172 }
4174 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one
4175 operand is in memory. In this case we need to use the predicated LD1
4176 and ST1 instead of LDR and STR, both for correctness on big-endian
4177 targets and because LD1 and ST1 support a wider range of addressing modes.
4178 PRED_MODE is the mode of the predicate.
4180 See the comment at the head of aarch64-sve.md for details about the
4181 big-endian handling. */
4183 void
4185 {
4190 {
4194 else
4197 }
4199 }
4201 /* Called only on big-endian targets. See whether an SVE vector move
4202 from SRC to DEST is effectively a REV[BHW] instruction, because at
4203 least one operand is a subreg of an SVE vector that has wider or
4204 narrower elements. Return true and emit the instruction if so.
4206 For example:
4208 (set (reg:VNx8HI R1) (subreg:VNx8HI (reg:VNx16QI R2) 0))
4210 represents a VIEW_CONVERT between the following vectors, viewed
4211 in memory order:
4213 R2: { [0].high, [0].low, [1].high, [1].low, ... }
4214 R1: { [0], [1], [2], [3], ... }
4216 The high part of lane X in R2 should therefore correspond to lane X*2
4217 of R1, but the register representations are:
4219 msb lsb
4220 R2: ...... [1].high [1].low [0].high [0].low
4221 R1: ...... [3] [2] [1] [0]
4223 where the low part of lane X in R2 corresponds to lane X*2 in R1.
4224 We therefore need a reverse operation to swap the high and low values
4225 around.
4227 This is purely an optimization. Without it we would spill the
4228 subreg operand to the stack in one mode and reload it in the
4229 other mode, which has the same effect as the REV. */
4231 bool
4233 {
4240 /* The optimization handles two single SVE REGs with different element
4241 sizes. */
4250 /* Generate *aarch64_sve_mov<mode>_subreg_be. */
4253 UNSPEC_REV_SUBREG);
4256 }
4258 /* Return a copy of X with mode MODE, without changing its other
4259 attributes. Unlike gen_lowpart, this doesn't care whether the
4260 mode change is valid. */
4262 static rtx
4264 {
4271 }
4273 /* Split a *aarch64_sve_mov<mode>_subreg_be pattern with the given
4274 operands. */
4276 void
4278 {
4279 /* Decide which REV operation we need. The mode with narrower elements
4280 determines the mode of the operands and the mode with the wider
4281 elements determines the reverse width. */
4296 else
4300 /* Emit:
4302 (set DEST (unspec [PTRUE (unspec [SRC] UNSPEC_REV<nn>)] UNSPEC_PRED_X))
4304 with the appropriate modes. */
4310 UNSPEC_PRED_X);
4312 }
4314 static bool
4316 tree exp ATTRIBUTE_UNUSED)
4317 {
4322 }
4324 /* Implement TARGET_PASS_BY_REFERENCE. */
4326 static bool
4328 machine_mode mode,
4329 const_tree type,
4331 {
4332 HOST_WIDE_INT size;
4333 machine_mode dummymode;
4336 /* GET_MODE_SIZE (BLKmode) is useless since it is 0. */
4339 else
4340 /* No frontends can create types with variable-sized modes, so we
4341 shouldn't be asked to pass or return them. */
4344 /* Aggregates are passed by reference based on their size. */
4346 {
4348 }
4350 /* Variable sized arguments are always returned by reference. */
4354 /* Can this be a candidate to be passed in fp/simd register(s)? */
4357 NULL))
4360 /* Arguments which are variable sized or larger than 2 registers are
4361 passed by reference unless they are a homogenous floating point
4362 aggregate. */
4364 }
4366 /* Return TRUE if VALTYPE is padded to its least significant bits. */
4367 static bool
4369 {
4370 machine_mode dummy_mode;
4373 /* Never happens in little-endian mode. */
4377 /* Only composite types smaller than or equal to 16 bytes can
4378 be potentially returned in registers. */
4384 /* But not a composite that is an HFA (Homogeneous Floating-point Aggregate)
4385 or an HVA (Homogeneous Short-Vector Aggregate); such a special composite
4386 is always passed/returned in the least significant bits of fp/simd
4387 register(s). */
4393 }
4395 /* Implement TARGET_FUNCTION_VALUE.
4396 Define how to find the value returned by a function. */
4398 static rtx
4401 {
4402 machine_mode mode;
4405 machine_mode ag_mode;
4412 {
4416 {
4419 }
4420 }
4424 {
4426 {
4429 }
4430 else
4431 {
4433 rtx par;
4437 {
4442 }
4444 }
4445 }
4446 else
4448 }
4450 /* Implements TARGET_FUNCTION_VALUE_REGNO_P.
4451 Return true if REGNO is the number of a hard register in which the values
4452 of called function may come back. */
4454 static bool
4456 {
4457 /* Maximum of 16 bytes can be returned in the general registers. Examples
4458 of 16-byte return values are: 128-bit integers and 16-byte small
4459 structures (excluding homogeneous floating-point aggregates). */
4463 /* Up to four fp/simd registers can return a function value, e.g. a
4464 homogeneous floating-point aggregate having four members. */
4469 }
4471 /* Implement TARGET_RETURN_IN_MEMORY.
4473 If the type T of the result of a function is such that
4474 void func (T arg)
4475 would require that arg be passed as a value in a register (or set of
4476 registers) according to the parameter passing rules, then the result
4477 is returned in the same registers as would be used for such an
4478 argument. */
4480 static bool
4482 {
4483 HOST_WIDE_INT size;
4484 machine_mode ag_mode;
4490 /* Simple scalar types always returned in registers. */
4494 type,
4497 NULL))
4500 /* Types larger than 2 registers returned in memory. */
4503 }
4505 static bool
4508 {
4511 type,
4513 nregs,
4514 NULL);
4515 }
4517 /* Given MODE and TYPE of a function argument, return the alignment in
4518 bits. The idea is to suppress any stronger alignment requested by
4519 the user and opt for the natural alignment (specified in AAPCS64 \S
4520 4.1). ABI_BREAK is set to true if the alignment was incorrectly
4521 calculated in versions of GCC prior to GCC-9. This is a helper
4522 function for local use only. */
4524 static unsigned int
4527 {
4547 {
4550 bitfield_alignment
4553 }
4556 {
4559 }
4562 }
4564 /* Layout a function argument according to the AAPCS64 rules. The rule
4565 numbers refer to the rule numbers in the AAPCS64. */
4567 static void
4569 const_tree type,
4571 {
4575 HOST_WIDE_INT size;
4578 /* We need to do this once per argument. */
4584 /* Size in bytes, rounded to the nearest multiple of 8 bytes. */
4587 else
4588 /* No frontends can create types with variable-sized modes, so we
4589 shouldn't be asked to pass or return them. */
4595 mode,
4596 type,
4599 /* allocate_ncrn may be false-positive, but allocate_nvrn is quite reliable.
4600 The following code thus handles passing by SIMD/FP registers first. */
4604 /* C1 - C5 for floating point, homogenous floating point aggregates (HFA)
4605 and homogenous short-vector aggregates (HVA). */
4607 {
4612 {
4615 {
4618 }
4619 else
4620 {
4621 rtx par;
4625 {
4628 rtx offset = gen_int_mode
4632 }
4634 }
4636 }
4637 else
4638 {
4639 /* C.3 NSRN is set to 8. */
4642 }
4643 }
4648 /* C6 - C9. though the sign and zero extension semantics are
4649 handled elsewhere. This is the case where the argument fits
4650 entirely general registers. */
4652 {
4655 /* C.8 if the argument has an alignment of 16 then the NGRN is
4656 rounded up to the next even number. */
4659 /* The == 16 * BITS_PER_UNIT instead of >= 16 * BITS_PER_UNIT
4660 comparison is there because for > 16 * BITS_PER_UNIT
4661 alignment nregs should be > 2 and therefore it should be
4662 passed by reference rather than value. */
4665 {
4671 }
4673 /* NREGS can be 0 when e.g. an empty structure is to be passed.
4674 A reg is still generated for it, but the caller should be smart
4675 enough not to use it. */
4678 else
4679 {
4680 rtx par;
4685 {
4690 }
4692 }
4696 }
4698 /* C.11 */
4701 /* The argument is passed on stack; record the needed number of words for
4702 this argument and align the total size if necessary. */
4703 on_stack:
4708 {
4711 {
4716 }
4717 }
4719 }
4721 /* Implement TARGET_FUNCTION_ARG. */
4723 static rtx
4726 {
4735 }
4737 void
4739 const_tree fntype ATTRIBUTE_UNUSED,
4740 rtx libname ATTRIBUTE_UNUSED,
4741 const_tree fndecl ATTRIBUTE_UNUSED,
4743 {
4757 {
4764 }
4766 }
4768 static void
4770 machine_mode mode,
4771 const_tree type,
4773 {
4776 {
4786 }
4787 }
4789 bool
4791 {
4794 }
4796 /* Implement FUNCTION_ARG_BOUNDARY. Every parameter gets at least
4797 PARM_BOUNDARY bits of alignment, but will be given anything up
4798 to STACK_BOUNDARY bits if the type requires it. This makes sure
4799 that both before and after the layout of each argument, the Next
4800 Stacked Argument Address (NSAA) will have a minimum alignment of
4801 8 bytes. */
4803 static unsigned int
4805 {
4814 }
4816 /* Implement TARGET_GET_RAW_RESULT_MODE and TARGET_GET_RAW_ARG_MODE. */
4818 static fixed_size_mode
4820 {
4822 /* Don't use the SVE part of the register for __builtin_apply and
4823 __builtin_return. The SVE registers aren't used by the normal PCS,
4824 so using them there would be a waste of time. The PCS extensions
4825 for SVE types are fundamentally incompatible with the
4826 __builtin_return/__builtin_apply interface. */
4829 }
4831 /* Implement TARGET_FUNCTION_ARG_PADDING.
4833 Small aggregate types are placed in the lowest memory address.
4835 The related parameter passing rules are B.4, C.3, C.5 and C.14. */
4837 static pad_direction
4839 {
4840 /* On little-endian targets, the least significant byte of every stack
4841 argument is passed at the lowest byte address of the stack slot. */
4845 /* Otherwise, integral, floating-point and pointer types are padded downward:
4846 the least significant byte of a stack argument is passed at the highest
4847 byte address of the stack slot. */
4854 /* Everything else padded upward, i.e. data in first byte of stack slot. */
4856 }
4858 /* Similarly, for use by BLOCK_REG_PADDING (MODE, TYPE, FIRST).
4860 It specifies padding for the last (may also be the only)
4861 element of a block move between registers and memory. If
4862 assuming the block is in the memory, padding upward means that
4863 the last element is padded after its highest significant byte,
4864 while in downward padding, the last element is padded at the
4865 its least significant byte side.
4867 Small aggregates and small complex types are always padded
4868 upwards.
4870 We don't need to worry about homogeneous floating-point or
4871 short-vector aggregates; their move is not affected by the
4872 padding direction determined here. Regardless of endianness,
4873 each element of such an aggregate is put in the least
4874 significant bits of a fp/simd register.
4876 Return !BYTES_BIG_ENDIAN if the least significant byte of the
4877 register has useful data, and return the opposite if the most
4878 significant byte does. */
4880 bool
4883 {
4885 /* Small composite types are always padded upward. */
4887 {
4888 HOST_WIDE_INT size;
4891 else
4892 /* No frontends can create types with variable-sized modes, so we
4893 shouldn't be asked to pass or return them. */
4897 }
4899 /* Otherwise, use the default padding. */
4901 }
4903 static scalar_int_mode
4905 {
4907 }
4909 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
4911 /* We use the 12-bit shifted immediate arithmetic instructions so values
4912 must be multiple of (1 << 12), i.e. 4096. */
4913 #define ARITH_FACTOR 4096
4915 #if (PROBE_INTERVAL % ARITH_FACTOR) != 0
4916 #error Cannot use simple address calculation for stack probing
4917 #endif
4919 /* The pair of scratch registers used for stack probing. */
4920 #define PROBE_STACK_FIRST_REG R9_REGNUM
4921 #define PROBE_STACK_SECOND_REG R10_REGNUM
4923 /* Emit code to probe a range of stack addresses from FIRST to FIRST+POLY_SIZE,
4924 inclusive. These are offsets from the current stack pointer. */
4926 static void
4928 {
4929 HOST_WIDE_INT size;
4931 {
4934 }
4938 /* See the same assertion on PROBE_INTERVAL above. */
4941 /* See if we have a constant small number of probes to generate. If so,
4942 that's the easy case. */
4944 {
4951 }
4953 /* The run-time loop is made up of 8 insns in the generic case while the
4954 compile-time loop is made up of 4+2*(n-2) insns for n # of intervals. */
4956 {
4961 stack_pointer_rtx,
4965 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
4966 it exceeds SIZE. If only two probes are needed, this will not
4967 generate any code. Then probe at FIRST + SIZE. */
4969 {
4973 }
4977 {
4982 }
4983 else
4985 }
4987 /* Otherwise, do the same as above, but in a loop. Note that we must be
4988 extra careful with variables wrapping around because we might be at
4989 the very top (or the very bottom) of the address space and we have
4990 to be able to handle this case properly; in particular, we use an
4991 equality test for the loop condition. */
4992 else
4993 {
4996 /* Step 1: round SIZE to the previous multiple of the interval. */
5001 /* Step 2: compute initial and final value of the loop counter. */
5003 /* TEST_ADDR = SP + FIRST. */
5007 /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE. */
5010 {
5014 }
5015 else
5019 /* Step 3: the loop
5021 do
5022 {
5023 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
5024 probe at TEST_ADDR
5025 }
5026 while (TEST_ADDR != LAST_ADDR)
5028 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
5029 until it is equal to ROUNDED_SIZE. */
5034 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
5035 that SIZE is equal to ROUNDED_SIZE. */
5038 {
5042 {
5047 }
5048 else
5050 }
5051 }
5053 /* Make sure nothing is scheduled before we are done. */
5055 }
5057 /* Probe a range of stack addresses from REG1 to REG2 inclusive. These are
5058 absolute addresses. */
5062 {
5069 /* Loop. */
5072 HOST_WIDE_INT stack_clash_probe_interval
5075 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
5077 HOST_WIDE_INT interval;
5080 else
5088 /* If doing stack clash protection then we probe up by the ABI specified
5089 amount. We do this because we're dropping full pages at a time in the
5090 loop. But if we're doing non-stack clash probing, probe at SP 0. */
5093 else
5096 /* Probe at TEST_ADDR. If we're inside the loop it is always safe to probe
5097 by this amount for each iteration. */
5100 /* Test if TEST_ADDR == LAST_ADDR. */
5104 /* Branch. */
5110 }
5112 /* Emit the probe loop for doing stack clash probes and stack adjustments for
5113 SVE. This emits probes from BASE to BASE - ADJUSTMENT based on a guard size
5114 of GUARD_SIZE. When a probe is emitted it is done at most
5115 MIN_PROBE_THRESHOLD bytes from the current BASE at an interval of
5116 at most MIN_PROBE_THRESHOLD. By the end of this function
5117 BASE = BASE - ADJUSTMENT. */
5122 {
5123 /* This function is not allowed to use any instruction generation function
5124 like gen_ and friends. If you do you'll likely ICE during CFG validation,
5125 so instead emit the code you want using output_asm_insn. */
5130 /* The minimum required allocation before the residual requires probing. */
5133 /* Clamp the value down to the nearest value that can be used with a cmp. */
5148 /* Emit loop start label. */
5151 /* ADJUSTMENT < RESIDUAL_PROBE_GUARD. */
5156 /* Branch to end if not enough adjustment to probe. */
5161 /* BASE = BASE - RESIDUAL_PROBE_GUARD. */
5166 /* Probe at BASE. */
5170 /* ADJUSTMENT = ADJUSTMENT - RESIDUAL_PROBE_GUARD. */
5175 /* Branch to start if still more bytes to allocate. */
5180 /* No probe leave. */
5183 /* BASE = BASE - ADJUSTMENT. */
5188 }
5190 /* Determine whether a frame chain needs to be generated. */
5191 static bool
5193 {
5194 /* Force a frame chain for EH returns so the return address is at FP+8. */
5198 /* A leaf function cannot have calls or write LR. */
5201 /* Don't use a frame chain in leaf functions if leaf frame pointers
5202 are disabled. */
5207 }
5209 /* Mark the registers that need to be saved by the callee and calculate
5210 the size of the callee-saved registers area and frame record (both FP
5211 and LR may be omitted). */
5212 static void
5214 {
5221 /* Adjust the outgoing arguments size if required. Keep it in sync with what
5222 the mid-end is doing. */
5225 #define SLOT_NOT_REQUIRED (-2)
5226 #define SLOT_REQUIRED (-1)
5231 /* If this is a non-leaf simd function with calls we assume that
5232 at least one of those calls is to a non-simd function and thus
5233 we must save V8 to V23 in the prologue. */
5236 {
5240 }
5242 /* First mark all the registers that really need to be saved... */
5249 /* ... that includes the eh data registers (if needed)... */
5255 /* ... and any callee saved register that dataflow says is live. */
5266 {
5269 }
5272 {
5273 /* FP and LR are placed in the linkage record. */
5279 }
5281 /* With stack-clash, LR must be saved in non-leaf functions. */
5286 /* Now assign stack slots for them. */
5289 {
5296 }
5304 {
5305 /* If there is an alignment gap between integer and fp callee-saves,
5306 allocate the last fp register to it if possible. */
5308 && has_align_gap
5309 && !simd_function
5311 {
5314 }
5323 }
5329 HOST_WIDE_INT varargs_and_saved_regs_size
5337 /* Both these values are already aligned. */
5361 {
5362 /* Simple, small frame with no outgoing arguments:
5363 stp reg1, reg2, [sp, -frame_size]!
5364 stp reg3, reg4, [sp, 16] */
5366 }
5371 max_push_offset)))
5372 {
5373 /* Frame with small outgoing arguments:
5374 sub sp, sp, frame_size
5375 stp reg1, reg2, [sp, outgoing_args_size]
5376 stp reg3, reg4, [sp, outgoing_args_size + 16] */
5380 }
5383 {
5384 /* Frame with large outgoing arguments but a small local area:
5385 stp reg1, reg2, [sp, -hard_fp_offset]!
5386 stp reg3, reg4, [sp, 16]
5387 sub sp, sp, outgoing_args_size */
5391 }
5392 else
5393 {
5394 /* Frame with large local area and outgoing arguments using frame pointer:
5395 sub sp, sp, hard_fp_offset
5396 stp x29, x30, [sp, 0]
5397 add x29, sp, 0
5398 stp reg3, reg4, [sp, 16]
5399 sub sp, sp, outgoing_args_size */
5403 }
5406 }
5408 /* Return true if the register REGNO is saved on entry to
5409 the current function. */
5411 static bool
5413 {
5415 }
5417 /* Return the next register up from REGNO up to LIMIT for the callee
5418 to save. */
5420 static unsigned
5422 {
5424 regno ++;
5426 }
5428 /* Push the register number REGNO of mode MODE to the stack with write-back
5429 adjusting the stack by ADJUSTMENT. */
5431 static void
5433 HOST_WIDE_INT adjustment)
5434 {
5445 }
5447 /* Generate and return an instruction to store the pair of registers
5448 REG and REG2 of mode MODE to location BASE with write-back adjusting
5449 the stack location BASE by ADJUSTMENT. */
5451 static rtx
5453 HOST_WIDE_INT adjustment)
5454 {
5456 {
5471 }
5472 }
5474 /* Push registers numbered REGNO1 and REGNO2 to the stack, adjusting the
5475 stack pointer by ADJUSTMENT. */
5477 static void
5479 {
5494 }
5496 /* Load the pair of register REG, REG2 of mode MODE from stack location BASE,
5497 adjusting it by ADJUSTMENT afterwards. */
5499 static rtx
5501 HOST_WIDE_INT adjustment)
5502 {
5504 {
5516 }
5517 }
5519 /* Pop the two registers numbered REGNO1, REGNO2 from the stack, adjusting it
5520 afterwards by ADJUSTMENT and writing the appropriate REG_CFA_RESTORE notes
5521 into CFI_OPS. */
5523 static void
5526 {
5533 {
5537 }
5538 else
5539 {
5544 }
5545 }
5547 /* Generate and return a store pair instruction of mode MODE to store
5548 register REG1 to MEM1 and register REG2 to MEM2. */
5550 static rtx
5552 rtx reg2)
5553 {
5555 {
5567 }
5568 }
5570 /* Generate and regurn a load pair isntruction of mode MODE to load register
5571 REG1 from MEM1 and register REG2 from MEM2. */
5573 static rtx
5575 rtx mem2)
5576 {
5578 {
5590 }
5591 }
5593 /* Return TRUE if return address signing should be enabled for the current
5594 function, otherwise return FALSE. */
5596 bool
5598 {
5599 /* This function should only be called after frame laid out. */
5602 /* If signing scope is AARCH64_FUNCTION_NON_LEAF, we only sign a leaf function
5603 if its LR is pushed onto stack. */
5607 }
5609 /* Return TRUE if Branch Target Identification Mechanism is enabled. */
5610 bool
5612 {
5614 }
5616 /* Emit code to save the callee-saved registers from register number START
5617 to LIMIT to the stack at the location starting at offset START_OFFSET,
5618 skipping any write-back candidates if SKIP_WB is true. */
5620 static void
5623 {
5631 {
5633 poly_int64 offset;
5647 offset));
5656 {
5658 rtx mem2;
5662 offset));
5664 reg2));
5666 /* The first part of a frame-related parallel insn is
5667 always assumed to be relevant to the frame
5668 calculations; subsequent parts, are only
5669 frame-related if explicitly marked. */
5672 }
5673 else
5677 }
5678 }
5680 /* Emit code to restore the callee registers of mode MODE from register
5681 number START up to and including LIMIT. Restore from the stack offset
5682 START_OFFSET, skipping any write-back candidates if SKIP_WB is true.
5683 Write the appropriate REG_CFA_RESTORE notes into CFI_OPS. */
5685 static void
5689 {
5693 poly_int64 offset;
5698 {
5721 {
5723 rtx mem2;
5731 }
5732 else
5735 }
5736 }
5738 /* Return true if OFFSET is a signed 4-bit value multiplied by the size
5739 of MODE. */
5743 {
5744 HOST_WIDE_INT multiple;
5747 }
5749 /* Return true if OFFSET is a unsigned 6-bit value multiplied by the size
5750 of MODE. */
5754 {
5755 HOST_WIDE_INT multiple;
5758 }
5760 /* Return true if OFFSET is a signed 7-bit value multiplied by the size
5761 of MODE. */
5763 bool
5765 {
5766 HOST_WIDE_INT multiple;
5769 }
5771 /* Return true if OFFSET is a signed 9-bit value. */
5773 bool
5775 poly_int64 offset)
5776 {
5777 HOST_WIDE_INT const_offset;
5780 }
5782 /* Return true if OFFSET is a signed 9-bit value multiplied by the size
5783 of MODE. */
5787 {
5788 HOST_WIDE_INT multiple;
5791 }
5793 /* Return true if OFFSET is an unsigned 12-bit value multiplied by the size
5794 of MODE. */
5798 {
5799 HOST_WIDE_INT multiple;
5802 }
5804 /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
5806 static sbitmap
5808 {
5812 /* The registers we need saved to the frame. */
5815 {
5820 /* Check that we can access the stack slot of the register with one
5821 direct load with no adjustments needed. */
5824 }
5826 /* Don't mess with the hard frame pointer. */
5832 /* If registers have been chosen to be stored/restored with
5833 writeback don't interfere with them to avoid having to output explicit
5834 stack adjustment instructions. */
5844 }
5846 /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
5848 static sbitmap
5850 {
5859 /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
5866 {
5870 /* If there is a callee-save at an adjacent offset, add it too
5871 to increase the use of LDP/STP. */
5876 {
5880 }
5881 }
5884 }
5886 /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS.
5887 Nothing to do for aarch64. */
5889 static void
5891 {
5892 }
5894 /* Return the next set bit in BMP from START onwards. Return the total number
5895 of bits in BMP if no set bit is found at or after START. */
5897 static unsigned int
5899 {
5910 }
5912 /* Do the work for aarch64_emit_prologue_components and
5913 aarch64_emit_epilogue_components. COMPONENTS is the bitmap of registers
5914 to save/restore, PROLOGUE_P indicates whether to emit the prologue sequence
5915 for these components or the epilogue sequence. That is, it determines
5916 whether we should emit stores or loads and what kind of CFA notes to attach
5917 to the insns. Otherwise the logic for the two sequences is very
5918 similar. */
5920 static void
5922 {
5924 ? HARD_FRAME_POINTER_REGNUM
5932 {
5933 /* AAPCS64 section 5.1.2 requires only the low 64 bits to be saved
5934 so DFmode for the vector registers is enough. For simd functions
5935 we want to save the low 128 bits. */
5948 /* No more registers to handle after REGNO.
5949 Emit a single save/restore and exit. */
5951 {
5956 else
5959 }
5962 /* The next register is not of the same class or its offset is not
5963 mergeable with the current one into a pair. */
5969 {
5974 else
5979 }
5981 /* REGNO2 can be saved/restored in a pair with REGNO. */
5993 else
5998 {
6001 }
6002 else
6003 {
6006 }
6009 }
6010 }
6012 /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
6014 static void
6016 {
6018 }
6020 /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
6022 static void
6024 {
6026 }
6028 /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
6030 static void
6032 {
6036 }
6038 /* On AArch64 we have an ABI defined safe buffer. This constant is used to
6039 determining the probe offset for alloca. */
6041 static HOST_WIDE_INT
6043 {
6045 }
6048 /* Allocate POLY_SIZE bytes of stack space using TEMP1 and TEMP2 as scratch
6049 registers. If POLY_SIZE is not large enough to require a probe this function
6050 will only adjust the stack. When allocating the stack space
6051 FRAME_RELATED_P is then used to indicate if the allocation is frame related.
6052 FINAL_ADJUSTMENT_P indicates whether we are allocating the outgoing
6053 arguments. If we are then we ensure that any allocation larger than the ABI
6054 defined buffer needs a probe so that the invariant of having a 1KB buffer is
6055 maintained.
6057 We emit barriers after each stack adjustment to prevent optimizations from
6058 breaking the invariant that we never drop the stack more than a page. This
6059 invariant is needed to make it easier to correctly handle asynchronous
6060 events, e.g. if we were to allow the stack to be dropped by more than a page
6061 and then have multiple probes up and we take a signal somewhere in between
6062 then the signal handler doesn't know the state of the stack and can make no
6063 assumptions about which pages have been probed. */
6065 static void
6067 poly_int64 poly_size,
6070 {
6071 HOST_WIDE_INT guard_size
6074 /* When doing the final adjustment for the outgoing argument size we can't
6075 assume that LR was saved at position 0. So subtract it's offset from the
6076 ABI safe buffer so that we don't accidentally allow an adjustment that
6077 would result in an allocation larger than the ABI buffer without
6078 probing. */
6079 HOST_WIDE_INT min_probe_threshold
6080 = final_adjustment_p
6086 /* We should always have a positive probe threshold. */
6090 {
6095 {
6097 }
6100 {
6102 }
6103 }
6105 /* If SIZE is not large enough to require probing, just adjust the stack and
6106 exit. */
6109 {
6112 }
6114 HOST_WIDE_INT size;
6115 /* Handle the SVE non-constant case first. */
6117 {
6119 {
6123 }
6125 /* First calculate the amount of bytes we're actually spilling. */
6132 {
6133 /* This is done to provide unwinding information for the stack
6134 adjustments we're about to do, however to prevent the optimizers
6135 from removing the R11 move and leaving the CFA note (which would be
6136 very wrong) we tie the old and new stack pointer together.
6137 The tie will expand to nothing but the optimizers will not touch
6138 the instruction. */
6143 /* We want the CFA independent of the stack pointer for the
6144 duration of the loop. */
6147 }
6156 /* Now reset the CFA register if needed. */
6158 {
6163 }
6166 }
6170 "Stack clash AArch64 prologue: " HOST_WIDE_INT_PRINT_DEC
6173 /* Round size to the nearest multiple of guard_size, and calculate the
6174 residual as the difference between the original size and the rounded
6175 size. */
6179 /* We can handle a small number of allocations/probes inline. Otherwise
6180 punt to a loop. */
6182 {
6184 {
6187 guard_used_by_caller));
6189 }
6191 }
6192 else
6193 {
6194 /* Compute the ending address. */
6199 /* For the initial allocation, we don't have a frame pointer
6200 set up, so we always need CFI notes. If we're doing the
6201 final allocation, then we may have a frame pointer, in which
6202 case it is the CFA, otherwise we need CFI notes.
6204 We can determine which allocation we are doing by looking at
6205 the value of FRAME_RELATED_P since the final allocations are not
6206 frame related. */
6208 {
6209 /* We want the CFA independent of the stack pointer for the
6210 duration of the loop. */
6214 }
6216 /* This allocates and probes the stack. Note that this re-uses some of
6217 the existing Ada stack protection code. However we are guaranteed not
6218 to enter the non loop or residual branches of that code.
6220 The non-loop part won't be entered because if our allocation amount
6221 doesn't require a loop, the case above would handle it.
6223 The residual amount won't be entered because TEMP1 is a mutliple of
6224 the allocation size. The residual will always be 0. As such, the only
6225 part we are actually using from that code is the loop setup. The
6226 actual probing is done in aarch64_output_probe_stack_range. */
6230 /* Now reset the CFA register if needed. */
6232 {
6236 }
6240 }
6242 /* Handle any residuals. Residuals of at least MIN_PROBE_THRESHOLD have to
6243 be probed. This maintains the requirement that each page is probed at
6244 least once. For initial probing we probe only if the allocation is
6245 more than GUARD_SIZE - buffer, and for the outgoing arguments we probe
6246 if the amount is larger than buffer. GUARD_SIZE - buffer + buffer ==
6247 GUARD_SIZE. This works that for any allocation that is large enough to
6248 trigger a probe here, we'll have at least one, and if they're not large
6249 enough for this code to emit anything for them, The page would have been
6250 probed by the saving of FP/LR either by this function or any callees. If
6251 we don't have any callees then we won't have more stack adjustments and so
6252 are still safe. */
6254 {
6256 /* If we're doing final adjustments, and we've done any full page
6257 allocations then any residual needs to be probed. */
6260 /* If doing a small final adjustment, we always probe at offset 0.
6261 This is done to avoid issues when LR is not at position 0 or when
6262 the final adjustment is smaller than the probing offset. */
6268 {
6271 "Stack clash AArch64 prologue residuals: "
6272 HOST_WIDE_INT_PRINT_DEC " bytes, probing will be required."
6276 residual_probe_offset));
6278 }
6279 }
6280 }
6282 /* Return 1 if the register is used by the epilogue. We need to say the
6283 return register is used, but only after epilogue generation is complete.
6284 Note that in the case of sibcalls, the values "used by the epilogue" are
6285 considered live at the start of the called function.
6287 For SIMD functions we need to return 1 for FP registers that are saved and
6288 restored by a function but are not zero in call_used_regs. If we do not do
6289 this optimizations may remove the restore of the register. */
6291 int
6293 {
6295 {
6300 }
6302 }
6304 /* Add a REG_CFA_EXPRESSION note to INSN to say that register REG
6305 is saved at BASE + OFFSET. */
6307 static void
6310 {
6314 }
6316 /* AArch64 stack frames generated by this compiler look like:
6318 +-------------------------------+
6319 | |
6320 | incoming stack arguments |
6321 | |
6322 +-------------------------------+
6323 | | <-- incoming stack pointer (aligned)
6324 | callee-allocated save area |
6325 | for register varargs |
6326 | |
6327 +-------------------------------+
6328 | local variables | <-- frame_pointer_rtx
6329 | |
6330 +-------------------------------+
6331 | padding | \
6332 +-------------------------------+ |
6333 | callee-saved registers | | frame.saved_regs_size
6334 +-------------------------------+ |
6335 | LR' | |
6336 +-------------------------------+ |
6337 | FP' | / <- hard_frame_pointer_rtx (aligned)
6338 +-------------------------------+
6339 | dynamic allocation |
6340 +-------------------------------+
6341 | padding |
6342 +-------------------------------+
6343 | outgoing stack arguments | <-- arg_pointer
6344 | |
6345 +-------------------------------+
6346 | | <-- stack_pointer_rtx (aligned)
6348 Dynamic stack allocations via alloca() decrease stack_pointer_rtx
6349 but leave frame_pointer_rtx and hard_frame_pointer_rtx
6350 unchanged.
6352 By default for stack-clash we assume the guard is at least 64KB, but this
6353 value is configurable to either 4KB or 64KB. We also force the guard size to
6354 be the same as the probing interval and both values are kept in sync.
6356 With those assumptions the callee can allocate up to 63KB (or 3KB depending
6357 on the guard size) of stack space without probing.
6359 When probing is needed, we emit a probe at the start of the prologue
6360 and every PARAM_STACK_CLASH_PROTECTION_GUARD_SIZE bytes thereafter.
6362 We have to track how much space has been allocated and the only stores
6363 to the stack we track as implicit probes are the FP/LR stores.
6365 For outgoing arguments we probe if the size is larger than 1KB, such that
6366 the ABI specified buffer is maintained for the next callee.
6368 The following registers are reserved during frame layout and should not be
6369 used for any other purpose:
6371 - r11: Used by stack clash protection when SVE is enabled.
6372 - r12(EP0) and r13(EP1): Used as temporaries for stack adjustment.
6373 - r14 and r15: Used for speculation tracking.
6374 - r16(IP0), r17(IP1): Used by indirect tailcalls.
6375 - r30(LR), r29(FP): Used by standard frame layout.
6377 These registers must be avoided in frame layout related code unless the
6378 explicit intention is to interact with one of the features listed above. */
6380 /* Generate the prologue instructions for entry into a function.
6381 Establish the stack frame by decreasing the stack pointer with a
6382 properly calculated size and, if necessary, create a frame record
6383 filled with the values of LR and previous frame pointer. The
6384 current FP is also set up if it is in use. */
6386 void
6388 {
6399 /* Sign return address for functions. */
6401 {
6403 {
6412 }
6415 }
6421 {
6423 {
6427 (frame_size
6429 }
6432 }
6437 /* In theory we should never have both an initial adjustment
6438 and a callee save adjustment. Verify that is the case since the
6439 code below does not handle it for -fstack-clash-protection. */
6442 /* Will only probe if the initial adjustment is larger than the guard
6443 less the amount of the guard reserved for use by the caller's
6444 outgoing args. */
6452 {
6455 {
6460 }
6465 {
6466 /* Variable-sized frames need to describe the save slot
6467 address using DW_CFA_expression rather than DW_CFA_offset.
6468 This means that, without taking further action, the
6469 locations of the registers that we've already saved would
6470 remain based on the stack pointer even after we redefine
6471 the CFA based on the frame pointer. We therefore need new
6472 DW_CFA_expressions to re-express the save slots with addresses
6473 based on the frame pointer. */
6477 /* Add an explicit CFA definition if this was previously
6478 implicit. */
6480 {
6482 callee_offset);
6485 }
6487 /* Change the save slot expressions for the registers that
6488 we've already saved. */
6493 reg_offset);
6494 }
6496 }
6503 else
6507 /* We may need to probe the final adjustment if it is larger than the guard
6508 that is assumed by the called. */
6511 }
6513 /* Return TRUE if we can use a simple_return insn.
6515 This function checks whether the callee saved stack is empty, which
6516 means no restore actions are need. The pro_and_epilogue will use
6517 this to check whether shrink-wrapping opt is feasible. */
6519 bool
6521 {
6529 }
6531 /* Return false for non-leaf SIMD functions in order to avoid
6532 shrink-wrapping them. Doing this will lose the necessary
6533 save/restore of FP registers. */
6535 bool
6537 {
6542 }
6544 /* Generate the epilogue instructions for returning from a function.
6545 This is almost exactly the reverse of the prolog sequence, except
6546 that we need to insert barriers to avoid scheduling loads that read
6547 from a deallocated stack, and we optimize the unwind records by
6548 emitting them all together if possible. */
6549 void
6551 {
6560 /* A stack clash protection prologue may not have left EP0_REGNUM or
6561 EP1_REGNUM in a usable state. The same is true for allocations
6562 with an SVE component, since we then need both temporary registers
6563 for each allocation. For stack clash we are in a usable state if
6564 the adjustment is less than GUARD_SIZE - GUARD_USED_BY_CALLER. */
6565 HOST_WIDE_INT guard_size
6569 /* We can re-use the registers when the allocation amount is smaller than
6570 guard_size - guard_used_by_caller because we won't be doing any probes
6571 then. In such situations the register should remain live with the correct
6572 value. */
6575 && (!flag_stack_clash_protection
6579 /* We need to add memory barrier to prevent read from deallocated stack. */
6580 bool need_barrier_p
6584 /* Emit a barrier to prevent loads from a deallocated stack. */
6588 {
6591 }
6593 /* Restore the stack pointer from the frame pointer if it may not
6594 be the same as the stack pointer. */
6599 /* If writeback is used when restoring callee-saves, the CFA
6600 is restored on the instruction doing the writeback. */
6604 else
6605 /* The case where we need to re-use the register here is very rare, so
6606 avoid the complicated condition and just always emit a move if the
6607 immediate doesn't fit. */
6615 else
6626 {
6627 /* Emit delayed restores and set the CFA to be SP + initial_adjust. */
6633 }
6635 /* Liveness of EP0_REGNUM can not be trusted across function calls either, so
6636 add restriction on emit_move optimization to leaf functions. */
6642 {
6643 /* Emit delayed restores and reset the CFA to be SP. */
6648 }
6650 /* We prefer to emit the combined return/authenticate instruction RETAA,
6651 however there are three cases in which we must instead emit an explicit
6652 authentication instruction.
6654 1) Sibcalls don't return in a normal way, so if we're about to call one
6655 we must authenticate.
6657 2) The RETAA instruction is not available before ARMv8.3-A, so if we are
6658 generating code for !TARGET_ARMV8_3 we can't use it and must
6659 explicitly authenticate.
6661 3) On an eh_return path we make extra stack adjustments to update the
6662 canonical frame address to be the exception handler's CFA. We want
6663 to authenticate using the CFA of the function which calls eh_return.
6664 */
6667 {
6669 {
6678 }
6681 }
6683 /* Stack adjustment for exception handler. */
6685 {
6686 /* We need to unwind the stack by the offset computed by
6687 EH_RETURN_STACKADJ_RTX. We have already reset the CFA
6688 to be SP; letting the CFA move during this adjustment
6689 is just as correct as retaining the CFA from the body
6690 of the function. Therefore, do nothing special. */
6692 }
6697 }
6699 /* Implement EH_RETURN_HANDLER_RTX. EH returns need to either return
6700 normally or return to a previous frame after unwinding.
6702 An EH return uses a single shared return sequence. The epilogue is
6703 exactly like a normal epilogue except that it has an extra input
6704 register (EH_RETURN_STACKADJ_RTX) which contains the stack adjustment
6705 that must be applied after the frame has been destroyed. An extra label
6706 is inserted before the epilogue which initializes this register to zero,
6707 and this is the entry point for a normal return.
6709 An actual EH return updates the return address, initializes the stack
6710 adjustment and jumps directly into the epilogue (bypassing the zeroing
6711 of the adjustment). Since the return address is typically saved on the
6712 stack when a function makes a call, the saved LR must be updated outside
6713 the epilogue.
6715 This poses problems as the store is generated well before the epilogue,
6716 so the offset of LR is not known yet. Also optimizations will remove the
6717 store as it appears dead, even after the epilogue is generated (as the
6718 base or offset for loading LR is different in many cases).
6720 To avoid these problems this implementation forces the frame pointer
6721 in eh_return functions so that the location of LR is fixed and known early.
6722 It also marks the store volatile, so no optimization is permitted to
6723 remove the store. */
6724 rtx
6726 {
6730 /* Mark the store volatile, so no optimization is permitted to remove it. */
6733 }
6735 /* Output code to add DELTA to the first argument, and then jump
6736 to FUNCTION. Used for C++ multiple inheritance. */
6737 static void
6739 HOST_WIDE_INT delta,
6740 HOST_WIDE_INT vcall_offset,
6741 tree function)
6742 {
6743 /* The this pointer is always in x0. Note that this differs from
6744 Arm where the this pointer maybe bumped to r1 if r0 is required
6745 to return a pointer to an aggregate. On AArch64 a result value
6746 pointer will be in x8. */
6764 else
6765 {
6770 {
6774 else
6777 }
6781 else
6788 else
6789 {
6791 Pmode);
6793 }
6797 else
6803 }
6805 /* Generate a tail call to the target function. */
6807 {
6810 }
6825 /* Stop pretending to be a post-reload pass. */
6827 }
6829 static bool
6831 {
6836 {
6840 /* Don't recurse into UNSPEC_TLS looking for TLS symbols; these are
6841 TLS offsets, not real symbol references. */
6844 }
6846 }
6849 /* Return true if val can be encoded as a 12-bit unsigned immediate with
6850 a left shift of 0 or 12 bits. */
6851 bool
6853 {
6856 );
6857 }
6859 /* Returns the nearest value to VAL that will fit as a 12-bit unsigned immediate
6860 that can be created with a left shift of 0 or 12. */
6861 static HOST_WIDE_INT
6863 {
6864 /* Check to see if the value fits in 24 bits, as that is the maximum we can
6865 handle correctly. */
6872 }
6874 /* Return true if val is an immediate that can be loaded into a
6875 register by a MOVZ instruction. */
6876 static bool
6878 {
6880 {
6884 }
6885 else
6886 {
6887 /* Ignore sign extension. */
6889 }
6892 }
6894 /* VAL is a value with the inner mode of MODE. Replicate it to fill a
6895 64-bit (DImode) integer. */
6897 static unsigned HOST_WIDE_INT
6899 {
6902 {
6906 }
6908 }
6910 /* Multipliers for repeating bitmasks of width 32, 16, 8, 4, and 2. */
6913 {
6919 };
6922 /* Return true if val is a valid bitmask immediate. */
6924 bool
6926 {
6930 /* Check for a single sequence of one bits and return quickly if so.
6931 The special cases of all ones and all zeroes returns false. */
6938 /* Replicate 32-bit immediates so we can treat them as 64-bit. */
6942 /* Invert if the immediate doesn't start with a zero bit - this means we
6943 only need to search for sequences of one bits. */
6947 /* Find the first set bit and set tmp to val with the first sequence of one
6948 bits removed. Return success if there is a single sequence of ones. */
6955 /* Find the next set bit and compute the difference in bit position. */
6960 /* Check the bit position difference is a power of 2, and that the first
6961 sequence of one bits fits within 'bits' bits. */
6965 /* Check the sequence of one bits is repeated 64/bits times. */
6967 }
6969 /* Create mask of ones, covering the lowest to highest bits set in VAL_IN.
6970 Assumed precondition: VAL_IN Is not zero. */
6972 unsigned HOST_WIDE_INT
6974 {
6981 }
6983 /* Create constant where bits outside of lowest bit set to highest bit set
6984 are set to 1. */
6986 unsigned HOST_WIDE_INT
6988 {
6990 }
6992 /* Return true if VAL_IN is a valid 'and' bitmask immediate. */
6994 bool
6996 {
6997 scalar_int_mode int_mode;
7010 }
7012 /* Return true if val is an immediate that can be loaded into a
7013 register in a single instruction. */
7014 bool
7016 {
7017 scalar_int_mode int_mode;
7024 }
7026 static bool
7028 {
7034 /* There's no way to calculate VL-based values using relocations. */
7042 {
7046 else
7047 /* Avoid generating a 64-bit relocation in ILP32; leave
7048 to aarch64_expand_mov_immediate to handle it properly. */
7050 }
7053 }
7055 /* Implement TARGET_CASE_VALUES_THRESHOLD.
7056 The expansion for a table switch is quite expensive due to the number
7057 of instructions, the table lookup and hard to predict indirect jump.
7058 When optimizing for speed, and -O3 enabled, use the per-core tuning if
7059 set, otherwise use tables for > 16 cases as a tradeoff between size and
7060 performance. When optimizing for size, use the default setting. */
7062 static unsigned int
7064 {
7065 /* Use the specified limit for the number of cases before using jump
7066 tables at higher optimization levels. */
7070 else
7072 }
7074 /* Return true if register REGNO is a valid index register.
7075 STRICT_P is true if REG_OK_STRICT is in effect. */
7077 bool
7079 {
7081 {
7089 }
7091 }
7093 /* Return true if register REGNO is a valid base register for mode MODE.
7094 STRICT_P is true if REG_OK_STRICT is in effect. */
7096 bool
7098 {
7100 {
7108 }
7110 /* The fake registers will be eliminated to either the stack or
7111 hard frame pointer, both of which are usually valid base registers.
7112 Reload deals with the cases where the eliminated form isn't valid. */
7117 }
7119 /* Return true if X is a valid base register for mode MODE.
7120 STRICT_P is true if REG_OK_STRICT is in effect. */
7122 static bool
7124 {
7131 }
7133 /* Return true if address offset is a valid index. If it is, fill in INFO
7134 appropriately. STRICT_P is true if REG_OK_STRICT is in effect. */
7136 static bool
7139 {
7141 rtx index;
7144 /* (reg:P) */
7147 {
7151 }
7152 /* (sign_extend:DI (reg:SI)) */
7157 {
7162 }
7163 /* (mult:DI (sign_extend:DI (reg:SI)) (const_int scale)) */
7170 {
7175 }
7176 /* (ashift:DI (sign_extend:DI (reg:SI)) (const_int shift)) */
7183 {
7188 }
7189 /* (sign_extract:DI (mult:DI (reg:DI) (const_int scale)) 32+shift 0) */
7196 {
7204 }
7205 /* (and:DI (mult:DI (reg:DI) (const_int scale))
7206 (const_int 0xffffffff<<shift)) */
7213 {
7219 }
7220 /* (sign_extract:DI (ashift:DI (reg:DI) (const_int shift)) 32+shift 0) */
7227 {
7235 }
7236 /* (and:DI (ashift:DI (reg:DI) (const_int shift))
7237 (const_int 0xffffffff<<shift)) */
7244 {
7250 }
7251 /* (mult:P (reg:P) (const_int scale)) */
7256 {
7260 }
7261 /* (ashift:P (reg:P) (const_int shift)) */
7266 {
7270 }
7271 else
7280 {
7284 }
7285 else
7286 {
7291 }
7295 {
7300 }
7303 }
7305 /* Return true if MODE is one of the modes for which we
7306 support LDP/STP operations. */
7308 static bool
7310 {
7318 }
7320 /* Return true if REGNO is a virtual pointer register, or an eliminable
7321 "soft" frame register. Like REGNO_PTR_FRAME_P except that we don't
7322 include stack_pointer or hard_frame_pointer. */
7323 static bool
7325 {
7330 }
7332 /* Return true if X is a valid address of type TYPE for machine mode MODE.
7333 If it is, fill in INFO appropriately. STRICT_P is true if
7334 REG_OK_STRICT is in effect. */
7336 bool
7339 aarch64_addr_query_type type)
7340 {
7343 poly_int64 offset;
7345 HOST_WIDE_INT const_size;
7347 /* On BE, we use load/store pair for all large int mode load/stores.
7348 TI/TFmode may also use a load/store pair. */
7357 /* If we are dealing with ADDR_QUERY_LDP_STP_N that means the incoming mode
7358 corresponds to the actual size of the memory being loaded/stored and the
7359 mode of the corresponding addressing mode is half of that. */
7369 /* For SVE, only accept [Rn], [Rn, Rm, LSL #shift] and
7370 [Rn, #offset, MUL VL]. */
7375 /* On LE, for AdvSIMD, don't support anything other than POST_INC or
7376 REG addressing. */
7378 && !BYTES_BIG_ENDIAN
7386 {
7403 {
7410 }
7415 {
7421 /* TImode and TFmode values are allowed in both pairs of X
7422 registers and individual Q registers. The available
7423 address modes are:
7424 X,X: 7-bit signed scaled offset
7425 Q: 9-bit signed offset
7426 We conservatively require an offset representable in either mode.
7427 When performing the check for pairs of X registers i.e. LDP/STP
7428 pass down DImode since that is the natural size of the LDP/STP
7429 instruction memory accesses. */
7435 /* A 7bit offset check because OImode will emit a ldp/stp
7436 instruction (only big endian will get here).
7437 For ldp/stp instructions, the offset is scaled for the size of a
7438 single element of the pair. */
7442 /* Three 9/12 bit offsets checks because CImode will emit three
7443 ldr/str instructions (only big endian will get here). */
7451 /* Two 7bit offsets checks because XImode will emit two ldp/stp
7452 instructions (only big endian will get here). */
7458 /* Make "m" use the LD1 offset range for SVE data modes, so
7459 that pre-RTL optimizers like ivopts will work to that
7460 instead of the wider LDR/STR range. */
7467 {
7468 poly_int64 end_offset = (offset
7475 end_offset)));
7476 }
7486 else
7489 }
7492 {
7493 /* Look for base + (scaled/extended) index register. */
7496 {
7499 }
7502 {
7505 }
7506 }
7527 {
7531 /* TImode and TFmode values are allowed in both pairs of X
7532 registers and individual Q registers. The available
7533 address modes are:
7534 X,X: 7-bit signed scaled offset
7535 Q: 9-bit signed offset
7536 We conservatively require an offset representable in either mode.
7537 */
7547 else
7549 }
7555 /* load literal: pc-relative constant pool entry. Only supported
7556 for SI mode or larger. */
7562 {
7570 }
7579 {
7585 {
7586 /* The symbol and offset must be aligned to the access size. */
7592 {
7596 }
7602 else
7611 }
7612 }
7617 }
7618 }
7620 /* Return true if the address X is valid for a PRFM instruction.
7621 STRICT_P is true if we should do strict checking with
7622 aarch64_classify_address. */
7624 bool
7626 {
7629 /* PRFM accepts the same addresses as DImode... */
7634 /* ... except writeback forms. */
7636 }
7638 bool
7640 {
7641 rtx offset;
7645 }
7647 /* Classify the base of symbolic expression X. */
7649 enum aarch64_symbol_type
7651 {
7652 rtx offset;
7656 }
7659 /* Return TRUE if X is a legitimate address for accessing memory in
7660 mode MODE. */
7661 static bool
7663 {
7667 }
7669 /* Return TRUE if X is a legitimate address of type TYPE for accessing
7670 memory in mode MODE. STRICT_P is true if REG_OK_STRICT is in effect. */
7671 bool
7673 aarch64_addr_query_type type)
7674 {
7678 }
7680 /* Implement TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT. */
7682 static bool
7684 poly_int64 orig_offset,
7685 machine_mode mode)
7686 {
7687 HOST_WIDE_INT size;
7689 {
7692 /* A general SVE offset is A * VQ + B. Remove the A component from
7693 coefficient 0 in order to get the constant B. */
7696 /* Split an out-of-range address displacement into a base and
7697 offset. Use 4KB range for 1- and 2-byte accesses and a 16KB
7698 range otherwise to increase opportunities for sharing the base
7699 address of different sizes. Unaligned accesses use the signed
7700 9-bit range, TImode/TFmode use the intersection of signed
7701 scaled 7-bit and signed 9-bit offset. */
7706 else
7712 /* Split the offset into second_offset and the rest. */
7716 }
7717 else
7718 {
7719 /* Get the mode we should use as the basis of the range. For structure
7720 modes this is the mode of one vector. */
7722 machine_mode step_mode
7725 /* Get the "mul vl" multiplier we'd like to use. */
7729 /* LDR supports a 9-bit range, but the move patterns for
7730 structure modes require all vectors to be in range of the
7731 same base. The simplest way of accomodating that while still
7732 promoting reuse of anchor points between different modes is
7733 to use an 8-bit range unconditionally. */
7735 else
7736 /* Predicates are only handled singly, so we might as well use
7737 the full range. */
7742 /* Convert the "mul vl" multiplier into a byte offset. */
7747 /* Split the offset into second_offset and the rest. */
7751 }
7752 }
7754 /* Return the binary representation of floating point constant VALUE in INTVAL.
7755 If the value cannot be converted, return false without setting INTVAL.
7756 The conversion is done in the given MODE. */
7757 bool
7759 {
7761 /* We make a general exception for 0. */
7763 {
7766 }
7768 scalar_float_mode mode;
7772 /* Only support up to DF mode. */
7784 {
7788 }
7789 else
7794 }
7796 /* Return TRUE if rtx X is an immediate constant that can be moved using a
7797 single MOV(+MOVK) followed by an FMOV. */
7798 bool
7800 {
7805 /* Determine whether it's cheaper to write float constants as
7806 mov/movk pairs over ldr/adrp pairs. */
7812 {
7814 ? SImode
7819 }
7822 }
7824 /* Return TRUE if rtx X is immediate constant 0.0 */
7825 bool
7827 {
7834 }
7836 /* Return TRUE if rtx X is immediate constant that fits in a single
7837 MOVI immediate operation. */
7838 bool
7840 {
7844 machine_mode vmode;
7845 scalar_int_mode imode;
7850 {
7854 /* We make a general exception for 0. */
7859 }
7863 else
7866 /* use a 64 bit mode for everything except for DI/DF mode, where we use
7867 a 128 bit vector mode. */
7874 }
7877 /* Return the fixed registers used for condition codes. */
7879 static bool
7881 {
7885 }
7887 /* This function is used by the call expanders of the machine description.
7888 RESULT is the register in which the result is returned. It's NULL for
7889 "call" and "sibcall".
7890 MEM is the location of the function call.
7891 SIBCALL indicates whether this function call is normal call or sibling call.
7892 It will generate different pattern accordingly. */
7894 void
7896 {
7898 rtvec vec;
7899 machine_mode mode;
7906 /* Decide if we should generate indirect calls by loading the
7907 address of the callee into a register before performing
7908 the branch-and-link. */
7922 else
7929 }
7931 /* Emit call insn with PAT and do aarch64-specific handling. */
7933 void
7935 {
7941 }
7943 machine_mode
7945 {
7949 /* All floating point compares return CCFP if it is an equality
7950 comparison, and CCFPE otherwise. */
7952 {
7954 {
7975 }
7976 }
7978 /* Equality comparisons of short modes against zero can be performed
7979 using the TST instruction with the appropriate bitmask. */
7985 /* Similarly, comparisons of zero_extends from shorter modes can
7986 be performed using an ANDS with an immediate mask. */
8002 /* A compare with a shifted operand. Because of canonicalization,
8003 the comparison will have to be swapped when we emit the assembly
8004 code. */
8012 /* Similarly for a negated operand, but we can only do this for
8013 equalities. */
8020 /* A test for unsigned overflow from an addition. */
8027 /* A test for unsigned overflow from an add with carry. */
8037 /* A test for signed overflow. */
8044 /* For everything else, return CCmode. */
8046 }
8048 static int
8051 int
8053 {
8060 }
8062 static int
8064 {
8066 {
8070 {
8084 }
8089 {
8101 }
8106 {
8118 }
8123 {
8133 }
8138 {
8144 }
8149 {
8153 }
8158 {
8162 }
8167 {
8171 }
8176 {
8180 }
8185 }
8188 }
8190 bool
8192 HOST_WIDE_INT minval,
8193 HOST_WIDE_INT maxval)
8194 {
8195 rtx elt;
8199 }
8201 bool
8203 {
8205 }
8207 /* Return true if VEC is a constant in which every element is in the range
8208 [MINVAL, MAXVAL]. The elements do not need to have the same value. */
8210 static bool
8212 HOST_WIDE_INT minval,
8213 HOST_WIDE_INT maxval)
8214 {
8226 {
8231 }
8233 }
8235 /* N Z C V. */
8236 #define AARCH64_CC_V 1
8237 #define AARCH64_CC_C (1 << 1)
8238 #define AARCH64_CC_Z (1 << 2)
8239 #define AARCH64_CC_N (1 << 3)
8241 /* N Z C V flags for ccmp. Indexed by AARCH64_COND_CODE. */
8243 {
8260 };
8262 /* Print floating-point vector immediate operand X to F, negating it
8263 first if NEGATE is true. Return true on success, false if it isn't
8264 a constant we can handle. */
8266 static bool
8268 {
8269 rtx elt;
8278 /* We only handle the SVE single-bit immediates here. */
8285 else
8289 }
8291 /* Return the equivalent letter for size. */
8292 static char
8294 {
8296 {
8302 }
8303 }
8305 /* Print operand X to file F in a target specific manner according to CODE.
8306 The acceptable formatting commands given by CODE are:
8307 'c': An integer or symbol address without a preceding #
8308 sign.
8309 'C': Take the duplicated element in a vector constant
8310 and print it in hex.
8311 'D': Take the duplicated element in a vector constant
8312 and print it as an unsigned integer, in decimal.
8313 'e': Print the sign/zero-extend size as a character 8->b,
8314 16->h, 32->w.
8315 'p': Prints N such that 2^N == X (X must be power of 2 and
8316 const int).
8317 'P': Print the number of non-zero bits in X (a const_int).
8318 'H': Print the higher numbered register of a pair (TImode)
8319 of regs.
8320 'm': Print a condition (eq, ne, etc).
8321 'M': Same as 'm', but invert condition.
8322 'N': Take the duplicated element in a vector constant
8323 and print the negative of it in decimal.
8324 'b/h/s/d/q': Print a scalar FP/SIMD register name.
8325 'S/T/U/V': Print a FP/SIMD register name for a register list.
8326 The register printed is the FP/SIMD register name
8327 of X + 0/1/2/3 for S/T/U/V.
8328 'R': Print a scalar FP/SIMD register name + 1.
8329 'X': Print bottom 16 bits of integer constant in hex.
8330 'w/x': Print a general register name or the zero register
8331 (32-bit or 64-bit).
8332 '0': Print a normal operand, if it's a general register,
8333 then we assume DImode.
8334 'k': Print NZCV for conditional compare instructions.
8335 'A': Output address constant representing the first
8336 argument of X, specifying a relocation offset
8337 if appropriate.
8338 'L': Output constant address specified by X
8339 with a relocation offset if appropriate.
8340 'G': Prints address of X, specifying a PC relative
8341 relocation mode if appropriate.
8342 'y': Output address of LDP or STP - this is used for
8343 some LDP/STPs which don't use a PARALLEL in their
8344 pattern (so the mode needs to be adjusted).
8345 'z': Output address of a typical LDP or STP. */
8347 static void
8349 {
8350 rtx elt;
8352 {
8355 {
8367 {
8370 }
8371 /* Fall through. */
8375 }
8379 {
8384 {
8387 }
8390 {
8403 }
8404 }
8408 {
8412 {
8415 }
8418 }
8423 {
8426 }
8433 {
8436 }
8439 {
8442 }
8449 {
8451 /* CONST_TRUE_RTX means al/nv (al is the default, don't print it). */
8453 {
8457 }
8460 {
8463 }
8471 else
8473 }
8478 {
8481 }
8487 ;
8488 else
8489 {
8492 }
8501 {
8502 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
8504 }
8513 {
8514 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
8516 }
8524 {
8525 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
8527 }
8533 {
8536 }
8541 {
8542 /* Print a replicated constant in hex. */
8544 {
8547 }
8550 }
8554 {
8555 /* Print a replicated constant in decimal, treating it as
8556 unsigned. */
8558 {
8561 }
8564 }
8571 {
8574 }
8577 {
8580 }
8583 {
8586 }
8588 /* Fall through */
8592 {
8595 }
8598 {
8601 {
8604 else
8605 {
8606 char suffix
8611 }
8612 }
8613 else
8632 {
8635 }
8636 /* fall through */
8640 {
8643 }
8649 ;
8650 else
8651 {
8654 }
8658 /* Since we define TARGET_SUPPORTS_WIDE_INT we shouldn't ever
8659 be getting CONST_DOUBLEs holding integers. */
8662 {
8665 }
8667 {
8668 #define buf_size 20
8676 #undef buf_size
8677 }
8683 }
8691 {
8718 }
8724 {
8759 }
8765 {
8771 }
8776 {
8777 HOST_WIDE_INT cond_code;
8780 {
8783 }
8788 }
8793 {
8798 {
8801 }
8805 ? ADDR_QUERY_LDP_STP_N
8808 }
8814 }
8815 }
8817 /* Print address 'x' of a memory access with mode 'mode'.
8818 'op' is the context required by aarch64_classify_address. It can either be
8819 MEM for a normal memory access or PARALLEL for LDP/STP. */
8820 static bool
8822 aarch64_addr_query_type type)
8823 {
8827 /* Check all addresses are Pmode - including ILP32. */
8831 {
8834 }
8838 {
8843 {
8844 HOST_WIDE_INT vnum
8849 }
8851 {
8852 HOST_WIDE_INT vnum
8857 }
8858 else
8867 else
8876 else
8885 else
8891 /* Writeback is only supported for fixed-width modes. */
8894 {
8917 }
8929 }
8932 }
8934 /* Print address 'x' of a memory access with mode 'mode'. */
8935 static void
8937 {
8940 }
8942 bool
8944 {
8951 /* UNSPEC_TLS entries for a symbol include a LABEL_REF for the
8952 referencing instruction, but they are constant offsets, not
8953 symbols. */
8959 {
8961 {
8967 }
8970 }
8973 }
8975 /* Implement REGNO_REG_CLASS. */
8977 enum reg_class
8979 {
8998 }
9000 /* OFFSET is an address offset for mode MODE, which has SIZE bytes.
9001 If OFFSET is out of range, return an offset of an anchor point
9002 that is in range. Return 0 otherwise. */
9004 static HOST_WIDE_INT
9006 machine_mode mode)
9007 {
9008 /* Does it look like we'll need a 16-byte load/store-pair operation? */
9012 /* For offsets that aren't a multiple of the access size, the limit is
9013 -256...255. */
9015 {
9016 /* BLKmode typically uses LDP of X-registers. */
9020 }
9022 /* Small negative offsets are supported. */
9029 /* Use 12-bit offset by access size. */
9031 }
9033 static rtx
9035 {
9036 /* Try to split X+CONST into Y=X+(CONST & ~mask), Y+(CONST&mask),
9037 where mask is selected by alignment and size of the offset.
9038 We try to pick as large a range for the offset as possible to
9039 maximize the chance of a CSE. However, for aligned addresses
9040 we limit the range to 4k so that structures with different sized
9041 elements are likely to use the same base. We need to be careful
9042 not to split a CONST for some forms of address expression, otherwise
9043 it will generate sub-optimal code. */
9046 {
9052 {
9056 /* Force any scaling into a temp for CSE. */
9060 /* Let the pointer register be in op0. */
9064 /* If the pointer is virtual or frame related, then we know that
9065 virtual register instantiation or register elimination is going
9066 to apply a second constant. We want the two constants folded
9067 together easily. Therefore, emit as (OP0 + CONST) + OP1. */
9069 {
9073 }
9075 /* Otherwise, in order to encourage CSE (and thence loop strength
9076 reduce) scaled addresses, emit as (OP0 + OP1) + CONST. */
9080 }
9082 HOST_WIDE_INT size;
9084 {
9086 mode);
9088 {
9092 }
9093 }
9094 }
9097 }
9099 static reg_class_t
9101 reg_class_t rclass,
9102 machine_mode mode,
9104 {
9105 /* Use aarch64_sve_reload_be for SVE reloads that cannot be handled
9106 directly by the *aarch64_sve_mov<mode>_be move pattern. See the
9107 comment at the head of aarch64-sve.md for more details about the
9108 big-endian handling. */
9114 {
9117 }
9119 /* If we have to disable direct literal pool loads and stores because the
9120 function is too big, then we need a scratch register. */
9125 {
9128 }
9130 /* Without the TARGET_SIMD instructions we cannot move a Q register
9131 to a Q register directly. We need a scratch. */
9135 {
9138 }
9140 /* A TFmode or TImode memory access should be handled via an FP_REGS
9141 because AArch64 has richer addressing modes for LDR/STR instructions
9142 than LDP/STP instructions. */
9151 }
9153 static bool
9155 {
9158 /* If we need a frame pointer, ARG_POINTER_REGNUM and FRAME_POINTER_REGNUM
9159 can only eliminate to HARD_FRAME_POINTER_REGNUM. */
9163 }
9165 poly_int64
9167 {
9169 {
9176 }
9179 {
9183 }
9186 }
9188 /* Implement RETURN_ADDR_RTX. We do not support moving back to a
9189 previous frame. */
9191 rtx
9193 {
9197 }
9200 static void
9202 {
9207 {
9211 }
9214 {
9217 offset1);
9218 }
9219 else
9220 {
9223 offset2);
9224 }
9227 /* The trampoline needs an extra padding instruction. In case if BTI is
9228 enabled the padding instruction is replaced by the BTI instruction at
9229 the beginning. */
9235 }
9237 static void
9239 {
9243 /* Don't need to copy the trailing D-words, we fill those in below. */
9255 /* XXX We should really define a "clear_cache" pattern and use
9256 gen_clear_cache(). */
9261 ptr_mode);
9262 }
9264 static unsigned char
9266 {
9267 /* ??? Logically we should only need to provide a value when
9268 HARD_REGNO_MODE_OK says that at least one register in REGCLASS
9269 can hold MODE, but at the moment we need to handle all modes.
9270 Just ignore any runtime parts for registers that can't store them. */
9274 {
9301 }
9303 }
9305 static reg_class_t
9307 {
9312 {
9318 }
9320 /* Register eliminiation can result in a request for
9321 SP+constant->FP_REGS. We cannot support such operations which
9322 use SP as source and an FP_REG as destination, so reject out
9323 right now. */
9325 {
9328 /* Look through a possible SUBREG introduced by ILP32. */
9334 POINTER_REGS));
9336 }
9339 }
9341 void
9343 {
9345 }
9347 static void
9349 {
9352 else
9353 {
9355 /* While priority is known to be in range [0, 65535], so 18 bytes
9356 would be enough, the compiler might not know that. To avoid
9357 -Wformat-truncation false positive, use a larger size. */
9364 }
9365 }
9367 static void
9369 {
9372 else
9373 {
9375 /* While priority is known to be in range [0, 65535], so 18 bytes
9376 would be enough, the compiler might not know that. To avoid
9377 -Wformat-truncation false positive, use a larger size. */
9384 }
9385 }
9389 {
9395 {
9396 {
9399 },
9400 {
9403 },
9404 {
9407 },
9408 /* We assume that DImode is only generated when not optimizing and
9409 that we don't really need 64-bit address offsets. That would
9410 imply an object file with 8GB of code in a single function! */
9411 {
9414 }
9415 };
9424 /* Need to implement table size reduction, by chaning the code below. */
9434 }
9437 /* Return size in bits of an arithmetic operand which is shifted/scaled and
9438 masked such that it is suitable for a UXTB, UXTH, or UXTW extend
9439 operator. */
9441 int
9443 {
9445 {
9448 {
9452 }
9453 }
9455 }
9457 /* Constant pools are per function only when PC relative
9458 literal loads are true or we are in the large memory
9459 model. */
9463 {
9466 }
9468 static bool
9470 {
9471 /* We can't use blocks for constants when we're using a per-function
9472 constant pool. */
9474 }
9476 /* Select appropriate section for constants depending
9477 on where we place literal pools. */
9481 rtx x,
9483 {
9488 }
9490 /* Implement ASM_OUTPUT_POOL_EPILOGUE. */
9491 void
9493 HOST_WIDE_INT offset)
9494 {
9495 /* When using per-function literal pools, we must ensure that any code
9496 section is aligned to the minimal instruction length, lest we get
9497 errors from the assembler re "unaligned instructions". */
9500 }
9502 /* Costs. */
9504 /* Helper function for rtx cost calculation. Strip a shift expression
9505 from X. Returns the inner operand if successful, or the original
9506 expression on failure. */
9507 static rtx
9509 {
9512 /* We accept both ROTATERT and ROTATE: since the RHS must be a constant
9513 we can convert both to ROR during final output. */
9528 }
9530 /* Helper function for rtx cost calculation. Strip an extend
9531 expression from X. Returns the inner operand if successful, or the
9532 original expression on failure. We deal with a number of possible
9533 canonicalization variations here. If STRIP_SHIFT is true, then
9534 we can strip off a shift also. */
9535 static rtx
9537 {
9538 scalar_int_mode mode;
9544 /* Zero and sign extraction of a widened value. */
9552 /* It can also be represented (for zero-extend) as an AND with an
9553 immediate. */
9562 /* Now handle extended register, as this may also have an optional
9563 left shift by 1..4. */
9578 }
9580 /* Return true iff CODE is a shift supported in combination
9581 with arithmetic instructions. */
9583 static bool
9585 {
9587 }
9590 /* Return true iff X is a cheap shift without a sign extend. */
9592 static bool
9594 {
9620 }
9622 /* Helper function for rtx cost calculation. Calculate the cost of
9623 a MULT or ASHIFT, which may be part of a compound PLUS/MINUS rtx.
9624 Return the calculated cost of the expression, recursing manually in to
9625 operands where needed. */
9627 static int
9629 {
9645 /* Integer multiply/fma. */
9647 {
9648 /* The multiply will be canonicalized as a shift, cost it as such. */
9652 {
9656 {
9658 {
9659 /* If the shift is considered cheap,
9660 then don't add any cost. */
9662 ;
9664 /* ARITH + shift-by-register. */
9667 /* ARITH + extended register. We don't have a cost field
9668 for ARITH+EXTEND+SHIFT, so use extend_arith here. */
9670 else
9671 /* ARITH + shift-by-immediate. */
9673 }
9674 else
9675 /* LSL (immediate). */
9678 }
9679 /* Strip extends as we will have costed them in the case above. */
9686 }
9688 /* MNEG or [US]MNEGL. Extract the NEG operand and indicate that it's a
9689 compound and let the below cases handle it. After all, MNEG is a
9690 special-case alias of MSUB. */
9692 {
9695 }
9697 /* Integer multiplies or FMAs have zero/sign extending variants. */
9702 {
9707 {
9709 /* SMADDL/UMADDL/UMSUBL/SMSUBL. */
9711 else
9712 /* MUL/SMULL/UMULL. */
9714 }
9717 }
9719 /* This is either an integer multiply or a MADD. In both cases
9720 we want to recurse and cost the operands. */
9725 {
9727 /* MADD/MSUB. */
9729 else
9730 /* MUL. */
9732 }
9735 }
9736 else
9737 {
9739 {
9740 /* Floating-point FMA/FMUL can also support negations of the
9741 operands, unless the rounding mode is upward or downward in
9742 which case FNMUL is different than FMUL with operand negation. */
9746 {
9751 }
9754 /* FMADD/FNMADD/FNMSUB/FMSUB. */
9756 else
9757 /* FMUL/FNMUL. */
9759 }
9764 }
9765 }
9767 static int
9769 machine_mode mode,
9770 addr_space_t as ATTRIBUTE_UNUSED,
9772 {
9780 {
9782 {
9783 /* This is a CONST or SYMBOL ref which will be split
9784 in a different way depending on the code model in use.
9785 Cost it through the generic infrastructure. */
9787 /* Divide through by the cost of one instruction to
9788 bring it to the same units as the address costs. */
9790 /* The cost is then the cost of preparing the address,
9791 followed by an immediate (possibly 0) offset. */
9793 }
9794 else
9795 {
9796 /* This is most likely a jump table from a case
9797 statement. */
9799 }
9800 }
9803 {
9815 else
9834 }
9838 {
9839 /* For the sake of calculating the cost of the shifted register
9840 component, we can treat same sized modes in the same way. */
9847 else
9848 /* We can't tell, or this is a 128-bit vector. */
9850 }
9853 }
9855 /* Return the cost of a branch. If SPEED_P is true then the compiler is
9856 optimizing for speed. If PREDICTABLE_P is true then the branch is predicted
9857 to be taken. */
9859 int
9861 {
9862 /* When optimizing for speed, use the cost of unpredictable branches. */
9868 else
9870 }
9872 /* Return true if the RTX X in mode MODE is a zero or sign extract
9873 usable in an ADD or SUB (extended register) instruction. */
9874 static bool
9876 {
9877 /* Catch add with a sign extract.
9878 This is add_<optab><mode>_multp2. */
9881 {
9892 op1))
9893 {
9895 }
9896 }
9897 /* The simple case <ARITH>, XD, XN, XM, [us]xt.
9898 No shift. */
9904 }
9906 static bool
9908 {
9910 {
9922 }
9923 }
9925 /* Return true iff X is an rtx that will match an extr instruction
9926 i.e. as described in the *extr<mode>5_insn family of patterns.
9927 OP0 and OP1 will be set to the operands of the shifts involved
9928 on success and will be NULL_RTX otherwise. */
9930 static bool
9932 {
9934 scalar_int_mode mode;
9949 {
9950 /* Canonicalise locally to ashift in op0, lshiftrt in op1. */
9962 {
9966 }
9967 }
9970 }
9972 /* Calculate the cost of calculating (if_then_else (OP0) (OP1) (OP2)),
9973 storing it in *COST. Result is true if the total cost of the operation
9974 has now been calculated. */
9975 static bool
9977 {
9978 rtx inner;
9979 rtx comparator;
9983 {
9987 }
9988 else
9989 {
9993 }
9996 {
9997 /* Conditional branch. */
10000 else
10001 {
10003 {
10005 {
10006 /* TBZ/TBNZ/CBZ/CBNZ. */
10008 /* TBZ/TBNZ. */
10011 else
10012 /* CBZ/CBNZ. */
10016 }
10017 }
10019 {
10020 /* TBZ/TBNZ. */
10023 }
10024 }
10025 }
10027 {
10028 /* CCMP. */
10030 {
10031 /* Increase cost of CCMP reg, 0, imm, CC to prefer CMP reg, 0. */
10035 {
10042 else
10044 }
10046 }
10048 /* It's a conditional operation based on the status flags,
10049 so it must be some flavor of CSEL. */
10051 /* CSNEG, CSINV, and CSINC are handled for free as part of CSEL. */
10057 {
10058 /* CSEL with zero-extension (*cmovdi_insn_uxtw). */
10061 }
10066 }
10068 /* We don't know what this is, cost all operands. */
10070 }
10072 /* Check whether X is a bitfield operation of the form shift + extend that
10073 maps down to a UBFIZ/SBFIZ/UBFX/SBFX instruction. If so, return the
10074 operand to which the bitfield operation is applied. Otherwise return
10075 NULL_RTX. */
10077 static rtx
10079 {
10093 {
10111 }
10114 }
10116 /* Return true if the mask and a shift amount from an RTX of the form
10117 (x << SHFT_AMNT) & MASK are valid to combine into a UBFIZ instruction of
10118 mode MODE. See the *andim_ashift<mode>_bfiz pattern. */
10120 bool
10122 rtx shft_amnt)
10123 {
10129 }
10131 /* Return true if the masks and a shift amount from an RTX of the form
10132 ((x & MASK1) | ((y << SHIFT_AMNT) & MASK2)) are valid to combine into
10133 a BFI instruction of mode MODE. See *arch64_bfi patterns. */
10135 bool
10140 {
10143 /* Verify that there is no overlap in what bits are set in the two masks. */
10147 /* Verify that mask2 is not all zeros or ones. */
10151 /* The shift amount should always be less than the mode size. */
10154 /* Verify that the mask being shifted is contiguous and would be in the
10155 least significant bits after shifting by shft_amnt. */
10158 }
10160 /* Calculate the cost of calculating X, storing it in *COST. Result
10161 is true if the total cost of the operation has now been calculated. */
10162 static bool
10165 {
10170 scalar_int_mode int_mode;
10172 /* By default, assume that everything has equivalent cost to the
10173 cheapest instruction. Any additional costs are applied as a delta
10174 above this default. */
10178 {
10180 /* The cost depends entirely on the operands to SET. */
10186 {
10189 {
10203 }
10212 /* Fall through. */
10214 /* The cost is one per vector-register copied. */
10216 {
10219 }
10220 /* const0_rtx is in general free, but we will use an
10221 instruction to set a register to 0. */
10223 {
10224 /* The cost is 1 per register copied. */
10227 }
10228 else
10229 /* Cost is just the cost of the RHS of the set. */
10235 /* Bit-field insertion. Strip any redundant widening of
10236 the RHS to meet the width of the target. */
10247 {
10248 /* MOV immediate is assumed to always be cheap. */
10250 }
10251 else
10252 {
10253 /* BFM. */
10257 }
10262 /* We can't make sense of this, assume default cost. */
10265 }
10269 /* If an instruction can incorporate a constant within the
10270 instruction, the instruction's expression avoids calling
10271 rtx_cost() on the constant. If rtx_cost() is called on a
10272 constant, then it is usually because the constant must be
10273 moved into a register by one or more instructions.
10275 The exception is constant 0, which can be expressed
10276 as XZR/WZR and is therefore free. The exception to this is
10277 if we have (set (reg) (const0_rtx)) in which case we must cost
10278 the move. However, we can catch that when we cost the SET, so
10279 we don't need to consider that here. */
10282 else
10283 {
10284 /* To an approximation, building any other constant is
10285 proportionally expensive to the number of instructions
10286 required to build that constant. This is true whether we
10287 are compiling for SPEED or otherwise. */
10292 }
10297 /* First determine number of instructions to do the move
10298 as an integer constant. */
10302 {
10308 ? SImode
10314 }
10317 {
10318 /* mov[df,sf]_aarch64. */
10320 /* FMOV (scalar immediate). */
10323 {
10324 /* This will be a load from memory. */
10327 else
10329 }
10330 else
10331 /* Otherwise this is +0.0. We get this using MOVI d0, #0
10332 or MOV v0.s[0], wzr - neither of which are modeled by the
10333 cost tables. Just use the default cost. */
10334 {
10335 }
10336 }
10342 {
10343 /* For loads we want the base cost of a load, plus an
10344 approximation for the additional cost of the addressing
10345 mode. */
10359 }
10367 {
10369 {
10370 /* FNEG. */
10372 }
10374 }
10377 {
10380 {
10381 /* CSETM. */
10384 }
10386 /* Cost this as SUB wzr, X. */
10390 }
10393 {
10394 /* Support (neg(fma...)) as a single instruction only if
10395 sign of zeros is unimportant. This matches the decision
10396 making in aarch64.md. */
10398 {
10399 /* FNMADD. */
10402 }
10404 {
10405 /* FNMUL. */
10408 }
10410 /* FNEG. */
10413 }
10420 {
10423 else
10425 }
10435 {
10439 }
10442 {
10443 /* TODO: A write to the CC flags possibly costs extra, this
10444 needs encoding in the cost tables. */
10447 /* ANDS. */
10449 {
10452 }
10455 {
10456 /* ADDS (and CMN alias). */
10459 }
10462 {
10463 /* SUBS. */
10466 }
10471 {
10472 /* COMPARE of ZERO_EXTRACT form of TST-immediate.
10473 Handle it here directly rather than going to cost_logic
10474 since we know the immediate generated for the TST is valid
10475 so we can avoid creating an intermediate rtx for it only
10476 for costing purposes. */
10483 }
10486 {
10487 /* CMN. */
10494 }
10496 /* CMP.
10498 Compare can freely swap the order of operands, and
10499 canonicalization puts the more complex operation first.
10500 But the integer MINUS logic expects the shift/extend
10501 operation in op1. */
10504 {
10507 }
10509 }
10512 {
10513 /* FCMP. */
10518 {
10520 /* FCMP supports constant 0.0 for no extra cost. */
10522 }
10524 }
10527 {
10528 /* Vector compare. */
10533 {
10534 /* Vector cm (eq|ge|gt|lt|le) supports constant 0.0 for no extra
10535 cost. */
10537 }
10539 }
10543 {
10547 cost_minus:
10550 /* Detect valid immediates. */
10556 {
10558 /* SUB(S) (immediate). */
10561 }
10563 /* Look for SUB (extended register). */
10566 {
10574 }
10578 /* Cost this as an FMA-alike operation. */
10582 {
10585 speed);
10587 }
10592 {
10594 {
10595 /* Vector SUB. */
10597 }
10599 {
10600 /* SUB(S). */
10602 }
10604 {
10605 /* FSUB. */
10607 }
10608 }
10610 }
10613 {
10614 rtx new_op0;
10619 cost_plus:
10622 {
10623 /* CSINC. */
10627 }
10632 {
10636 /* ADD (immediate). */
10639 }
10643 /* Look for ADD (extended register). */
10646 {
10654 }
10656 /* Strip any extend, leave shifts behind as we will
10657 cost them through mult_cost. */
10662 {
10664 speed);
10666 }
10671 {
10673 {
10674 /* Vector ADD. */
10676 }
10678 {
10679 /* ADD. */
10681 }
10683 {
10684 /* FADD. */
10686 }
10687 }
10689 }
10695 {
10698 else
10700 }
10705 {
10709 {
10712 else
10714 }
10716 }
10719 {
10726 }
10727 /* Fall through. */
10730 cost_logic:
10735 {
10739 }
10747 {
10748 /* This is a UBFM/SBFM. */
10753 }
10756 {
10758 {
10759 /* We have a mask + shift version of a UBFIZ
10760 i.e. the *andim_ashift<mode>_bfiz pattern. */
10764 {
10771 }
10773 {
10774 /* We possibly get the immediate for free, this is not
10775 modelled. */
10782 }
10783 }
10784 else
10785 {
10788 /* Handle ORN, EON, or BIC. */
10794 /* If we had a shift on op0 then this is a logical-shift-
10795 by-register/immediate operation. Otherwise, this is just
10796 a logical operation. */
10798 {
10800 {
10801 /* Shift by immediate. */
10804 else
10806 }
10807 else
10809 }
10811 /* In both cases we want to cost both operands. */
10818 }
10819 }
10827 {
10828 /* Vector NOT. */
10831 }
10833 /* MVN-shifted-reg. */
10835 {
10842 }
10843 /* EON can have two forms: (xor (not a) b) but also (not (xor a b)).
10844 Handle the second form here taking care that 'a' in the above can
10845 be a shift. */
10847 {
10856 {
10859 else
10861 }
10864 }
10865 /* MVN. */
10874 /* If a value is written in SI mode, then zero extended to DI
10875 mode, the operation will in general be free as a write to
10876 a 'w' register implicitly zeroes the upper bits of an 'x'
10877 register. However, if this is
10879 (set (reg) (zero_extend (reg)))
10881 we must cost the explicit register move. */
10885 {
10888 /* If OP_COST is non-zero, then the cost of the zero extend
10889 is effectively the cost of the inner operation. Otherwise
10890 we have a MOV instruction and we take the cost from the MOV
10891 itself. This is true independently of whether we are
10892 optimizing for space or time. */
10897 }
10899 {
10900 /* All loads can zero extend to any size for free. */
10903 }
10907 {
10912 }
10915 {
10917 {
10918 /* UMOV. */
10920 }
10921 else
10922 {
10923 /* We generate an AND instead of UXTB/UXTH. */
10925 }
10926 }
10931 {
10932 /* LDRSH. */
10934 {
10941 }
10943 }
10947 {
10952 }
10955 {
10958 else
10960 }
10968 {
10970 {
10972 {
10973 /* Vector shift (immediate). */
10975 }
10976 else
10977 {
10978 /* LSL (immediate), UBMF, UBFIZ and friends. These are all
10979 aliases. */
10981 }
10982 }
10984 /* We can incorporate zero/sign extend for free. */
10991 }
10992 else
10993 {
10995 {
10997 /* Vector shift (register). */
10999 }
11000 else
11001 {
11003 /* LSLV. */
11010 {
11012 /* We already demanded XEXP (op1, 0) to be REG_P, so
11013 don't recurse into it. */
11015 }
11016 }
11018 }
11028 {
11029 /* ASR (immediate) and friends. */
11031 {
11034 else
11036 }
11040 }
11041 else
11042 {
11044 {
11046 /* Vector shift (register). */
11048 }
11049 else
11050 {
11052 /* ASR (register) and friends. */
11059 {
11061 /* We already demanded XEXP (op1, 0) to be REG_P, so
11062 don't recurse into it. */
11064 }
11065 }
11067 }
11073 {
11074 /* LDR. */
11077 }
11080 {
11081 /* ADRP, followed by ADD. */
11085 }
11088 {
11089 /* ADR. */
11092 }
11095 {
11096 /* One extra load instruction, after accessing the GOT. */
11100 }
11105 /* ADRP/ADD (immediate). */
11112 /* UBFX/SBFX. */
11114 {
11117 else
11119 }
11121 /* We can trust that the immediates used will be correct (there
11122 are no by-register forms), so we need only cost op0. */
11128 /* aarch64_rtx_mult_cost always handles recursion to its
11129 operands. */
11133 /* We can expand signed mod by power of 2 using a NEGS, two parallel
11134 ANDs and a CSNEG. Assume here that CSNEG is the same as the cost of
11135 an unconditional negate. This case should only ever be reached through
11136 the set_smod_pow2_cheap check in expmed.c. */
11140 {
11141 /* We expand to 4 instructions. Reset the baseline. */
11149 }
11151 /* Fall-through. */
11154 {
11155 /* Slighly prefer UMOD over SMOD. */
11162 }
11169 {
11173 /* There is no integer SQRT, so only DIV and UDIV can get
11174 here. */
11176 /* Slighly prefer UDIV over SDIV. */
11178 else
11180 }
11206 {
11209 else
11211 }
11213 /* FMSUB, FNMADD, and FNMSUB are free. */
11220 /* aarch64_fnma4_elt_to_64v2df has the NEG as operand 1,
11221 and the by-element operand as operand 0. */
11225 /* Catch vector-by-element operations. The by-element operand can
11226 either be (vec_duplicate (vec_select (x))) or just
11227 (vec_select (x)), depending on whether we are multiplying by
11228 a vector or a scalar.
11230 Canonicalization is not very good in these cases, FMA4 will put the
11231 by-element operand as operand 0, FNMA4 will have it as operand 1. */
11242 /* If the remaining parameters are not registers,
11243 get the cost to put them into registers. */
11257 {
11259 {
11260 /*Vector truncate. */
11262 }
11263 else
11265 }
11270 {
11272 {
11273 /*Vector conversion. */
11275 }
11276 else
11278 }
11284 /* Strip the rounding part. They will all be implemented
11285 by the fcvt* family of instructions anyway. */
11287 {
11296 }
11299 {
11302 else
11304 }
11306 /* We can combine fmul by a power of 2 followed by a fcvt into a single
11307 fixed-point fcvt. */
11312 {
11316 }
11323 {
11324 /* ABS (vector). */
11327 }
11329 {
11332 /* FABD, which is analogous to FADD. */
11334 {
11341 }
11342 /* Simple FABS is analogous to FNEG. */
11345 }
11346 else
11347 {
11348 /* Integer ABS will either be split to
11349 two arithmetic instructions, or will be an ABS
11350 (scalar), which we don't model. */
11354 }
11360 {
11363 else
11364 {
11365 /* FMAXNM/FMINNM/FMAX/FMIN.
11366 TODO: This may not be accurate for all implementations, but
11367 we do not model this in the cost tables. */
11369 }
11370 }
11374 /* The floating point round to integer frint* instructions. */
11376 {
11381 }
11384 {
11389 }
11394 /* Decompose <su>muldi3_highpart. */
11396 mode == DImode
11397 /* (lshiftrt:TI */
11400 /* (mult:TI */
11402 /* (ANY_EXTEND:TI (reg:DI))
11403 (ANY_EXTEND:TI (reg:DI))) */
11410 /* (const_int 64) */
11413 {
11414 /* UMULH/SMULH. */
11422 }
11424 /* Fall through. */
11427 }
11435 }
11437 /* Wrapper around aarch64_rtx_costs, dumps the partial, or total cost
11438 calculated for X. This cost is stored in *COST. Returns true
11439 if the total cost of X was calculated. */
11440 static bool
11443 {
11448 {
11453 }
11456 }
11458 static int
11461 {
11467 /* Caller save and pointer regs are equivalent to GENERAL_REGS. */
11474 /* Moving between GPR and stack cost is the same as GP2GP. */
11479 /* To/From the stack register, we move via the gprs. */
11485 {
11486 /* 128-bit operations on general registers require 2 instructions. */
11494 /* When AdvSIMD instructions are disabled it is not possible to move
11495 a 128-bit value directly between Q registers. This is handled in
11496 secondary reload. A general register is used as a scratch to move
11497 the upper DI value and the lower DI value is moved directly,
11498 hence the cost is the sum of three moves. */
11503 }
11513 }
11515 static int
11517 reg_class_t rclass ATTRIBUTE_UNUSED,
11519 {
11521 }
11523 /* Return true if it is safe and beneficial to use the approximate rsqrt optabs
11524 to optimize 1.0/sqrt. */
11526 static bool
11528 {
11530 && flag_unsafe_math_optimizations
11534 }
11536 /* Function to decide when to use the approximate reciprocal square root
11537 builtin. */
11539 static tree
11541 {
11547 }
11549 /* Emit instruction sequence to compute either the approximate square root
11550 or its approximate reciprocal, depending on the flag RECP, and return
11551 whether the sequence was emitted or not. */
11553 bool
11555 {
11559 {
11562 }
11565 {
11572 || flag_trapping_math
11573 || !flag_unsafe_math_optimizations
11576 }
11577 else
11578 /* Caller assumes we cannot fail. */
11584 /* When calculating the approximate square root, compare the
11585 argument with 0.0 and create a mask. */
11591 /* Estimate the approximate reciprocal square root. */
11595 /* Iterate over the series twice for SF and thrice for DF. */
11598 /* Optionally iterate over the series once less for faster performance
11599 while sacrificing the accuracy. */
11602 iterations--;
11604 /* Iterate over the series to calculate the approximate reciprocal square
11605 root. */
11608 {
11616 }
11619 {
11620 /* Qualify the approximate reciprocal square root when the argument is
11621 0.0 by squashing the intermediary result to 0.0. */
11627 /* Calculate the approximate square root. */
11629 }
11631 /* Finalize the approximation. */
11635 }
11637 /* Emit the instruction sequence to compute the approximation for the division
11638 of NUM by DEN in QUO and return whether the sequence was emitted or not. */
11640 bool
11642 {
11653 || flag_trapping_math
11654 || !flag_unsafe_math_optimizations
11662 /* Estimate the approximate reciprocal. */
11666 /* Iterate over the series twice for SF and thrice for DF. */
11669 /* Optionally iterate over the series once less for faster performance,
11670 while sacrificing the accuracy. */
11672 iterations--;
11674 /* Iterate over the series to calculate the approximate reciprocal. */
11677 {
11682 }
11685 {
11686 /* As the approximate reciprocal of DEN is already calculated, only
11687 calculate the approximate division when NUM is not 1.0. */
11690 }
11692 /* Finalize the approximation. */
11695 }
11697 /* Return the number of instructions that can be issued per cycle. */
11698 static int
11700 {
11702 }
11704 static int
11706 {
11710 }
11713 /* Implement TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD as
11714 autopref_multipass_dfa_lookahead_guard from haifa-sched.c. It only
11715 has an effect if PARAM_SCHED_AUTOPREF_QUEUE_DEPTH > 0. */
11717 static int
11720 {
11722 }
11725 /* Vectorizer cost model target hooks. */
11727 /* Implement targetm.vectorize.builtin_vectorization_cost. */
11728 static int
11730 tree vectype,
11732 {
11741 {
11792 }
11793 }
11795 /* Implement targetm.vectorize.add_stmt_cost. */
11796 static unsigned
11800 {
11805 {
11810 /* Statements in an inner loop relative to the loop being
11811 vectorized are weighted more heavily. The value here is
11812 arbitrary and could potentially be improved with analysis. */
11818 }
11821 }
11825 /* Parse the TO_PARSE string and put the architecture struct that it
11826 selects into RES and the architectural features into ISA_FLAGS.
11827 Return an aarch64_parse_opt_result describing the parse result.
11828 If there is an error parsing, RES and ISA_FLAGS are left unchanged.
11829 When the TO_PARSE string contains an invalid extension,
11830 a copy of the string is created and stored to INVALID_EXTENSION. */
11832 static enum aarch64_parse_opt_result
11835 {
11844 else
11851 /* Loop through the list of supported ARCHes to find a match. */
11853 {
11856 {
11860 {
11861 /* TO_PARSE string contains at least one extension. */
11862 enum aarch64_parse_opt_result ext_res
11867 }
11868 /* Extension parsing was successful. Confirm the result
11869 arch and ISA flags. */
11873 }
11874 }
11876 /* ARCH name not found in list. */
11878 }
11880 /* Parse the TO_PARSE string and put the result tuning in RES and the
11881 architecture flags in ISA_FLAGS. Return an aarch64_parse_opt_result
11882 describing the parse result. If there is an error parsing, RES and
11883 ISA_FLAGS are left unchanged.
11884 When the TO_PARSE string contains an invalid extension,
11885 a copy of the string is created and stored to INVALID_EXTENSION. */
11887 static enum aarch64_parse_opt_result
11890 {
11899 else
11906 /* Loop through the list of supported CPUs to find a match. */
11908 {
11910 {
11915 {
11916 /* TO_PARSE string contains at least one extension. */
11917 enum aarch64_parse_opt_result ext_res
11922 }
11923 /* Extension parsing was successfull. Confirm the result
11924 cpu and ISA flags. */
11928 }
11929 }
11931 /* CPU name not found in list. */
11933 }
11935 /* Parse the TO_PARSE string and put the cpu it selects into RES.
11936 Return an aarch64_parse_opt_result describing the parse result.
11937 If the parsing fails the RES does not change. */
11939 static enum aarch64_parse_opt_result
11941 {
11944 /* Loop through the list of supported CPUs to find a match. */
11946 {
11948 {
11951 }
11952 }
11954 /* CPU name not found in list. */
11956 }
11958 /* Parse TOKEN, which has length LENGTH to see if it is an option
11959 described in FLAG. If it is, return the index bit for that fusion type.
11960 If not, error (printing OPTION_NAME) and return zero. */
11962 static unsigned int
11967 {
11969 {
11973 }
11977 }
11979 /* Parse OPTION which is a comma-separated list of flags to enable.
11980 FLAGS gives the list of flags we understand, INITIAL_STATE gives any
11981 default state we inherit from the CPU tuning structures. OPTION_NAME
11982 gives the top-level option we are parsing in the -moverride string,
11983 for use in error messages. */
11985 static unsigned int
11990 {
11997 {
12000 token_length,
12001 flags,
12002 option_name);
12003 /* If we find "none" (or, for simplicity's sake, an error) anywhere
12004 in the token stream, reset the supported operations. So:
12006 adrp+add.cmp+branch.none.adrp+add
12008 would have the result of turning on only adrp+add fusion. */
12014 }
12016 /* We ended with a comma, print something. */
12018 {
12021 }
12023 /* We still have one more token to parse. */
12026 token_length,
12027 flags,
12028 option_name);
12034 }
12036 /* Support for overriding instruction fusion. */
12038 static void
12041 {
12043 aarch64_fusible_pairs,
12046 }
12048 /* Support for overriding other tuning flags. */
12050 static void
12053 {
12054 tune->extra_tuning_flags
12056 aarch64_tuning_flags,
12059 }
12061 /* Parse the sve_width tuning moverride string in TUNE_STRING.
12062 Accept the valid SVE vector widths allowed by
12063 aarch64_sve_vector_bits_enum and use it to override sve_width
12064 in TUNE. */
12066 static void
12069 {
12074 {
12077 }
12079 {
12088 }
12090 }
12092 /* Parse TOKEN, which has length LENGTH to see if it is a tuning option
12093 we understand. If it is, extract the option string and handoff to
12094 the appropriate function. */
12096 void
12100 {
12106 {
12109 }
12111 /* Get the length of the option name. */
12113 /* Skip the '=' to get to the option string. */
12114 option_part++;
12117 {
12119 {
12122 }
12123 }
12127 }
12129 /* A checking mechanism for the implementation of the tls size. */
12131 static void
12133 {
12138 {
12140 /* Both the default and maximum TLS size allowed under tiny is 1M which
12141 needs two instructions to address, so we clamp the size to 24. */
12146 /* The maximum TLS size allowed under small is 4G. */
12151 /* The maximum TLS size allowed under large is 16E.
12152 FIXME: 16E should be 64bit, we only support 48bit offset now. */
12158 }
12161 }
12163 /* Parse STRING looking for options in the format:
12164 string :: option:string
12165 option :: name=substring
12166 name :: {a-z}
12167 substring :: defined by option. */
12169 static void
12172 {
12183 {
12185 /* Make this substring look like a string. */
12189 }
12191 /* One last option to parse. */
12194 }
12197 static void
12199 {
12201 {
12202 opts->x_aarch64_branch_protection_string
12204 }
12206 /* PR 70044: We have to be careful about being called multiple times for the
12207 same function. This means all changes should be repeatable. */
12209 /* Set aarch64_use_frame_pointer based on -fno-omit-frame-pointer.
12210 Disable the frame pointer flag so the mid-end will not use a frame
12211 pointer in leaf functions in order to support -fomit-leaf-frame-pointer.
12212 Set x_flag_omit_frame_pointer to the special value 2 to differentiate
12213 between -fomit-frame-pointer (1) and -fno-omit-frame-pointer (2). */
12218 /* If not optimizing for size, set the default
12219 alignment to what the target wants. */
12221 {
12228 }
12230 /* We default to no pc-relative literal loads. */
12234 /* If -mpc-relative-literal-loads is set on the command line, this
12235 implies that the user asked for PC relative literal loads. */
12239 /* In the tiny memory model it makes no sense to disallow PC relative
12240 literal pool loads. */
12245 /* When enabling the lower precision Newton series for the square root, also
12246 enable it for the reciprocal square root, since the latter is an
12247 intermediary step for the former. */
12250 }
12252 /* 'Unpack' up the internal tuning structs and update the options
12253 in OPTS. The caller must have set up selected_tune and selected_arch
12254 as all the other target-specific codegen decisions are
12255 derived from them. */
12257 void
12259 {
12262 /* Make a copy of the tuning parameters attached to the core, which
12263 we may later overwrite. */
12271 /* This target defaults to strict volatile bitfields. */
12277 {
12280 aarch64_stack_protector_guard_offset_str);
12281 }
12286 {
12288 "%<-mstack-protector-guard-reg%> must be used "
12290 }
12293 {
12296 }
12299 {
12308 }
12315 {
12327 }
12329 /* We don't mind passing in global_options_set here as we don't use
12330 the *options_set structs anyway. */
12332 queue_depth,
12336 /* Set up parameters to be used in prefetching algorithm. Do not
12337 override the defaults unless we are tuning for a core we have
12338 researched values for. */
12370 /* Use the alternative scheduling-pressure algorithm by default. */
12375 /* If the user hasn't changed it via configure then set the default to 64 KB
12376 for the backend. */
12383 /* Validate the guard size. */
12386 /* Enforce that interval is the same size as size so the mid-end does the
12387 right thing. */
12389 guard_size,
12393 /* The maybe_set calls won't update the value if the user has explicitly set
12394 one. Which means we need to validate that probing interval and guard size
12395 are equal. */
12396 int probe_interval
12402 /* Enable sw prefetching at specified optimization level for
12403 CPUS that have prefetch. Lower optimization level threshold by 1
12404 when profiling is enabled. */
12419 }
12421 /* Print a hint with a suggestion for a core or architecture name that
12422 most closely resembles what the user passed in STR. ARCH is true if
12423 the user is asking for an architecture name. ARCH is false if the user
12424 is asking for a core name. */
12426 static void
12428 {
12434 #ifdef HAVE_LOCAL_CPU_DETECT
12435 /* Add also "native" as possible value. */
12438 #endif
12445 else
12449 }
12451 /* Print a hint with a suggestion for a core name that most closely resembles
12452 what the user passed in STR. */
12456 {
12458 }
12460 /* Print a hint with a suggestion for an architecture name that most closely
12461 resembles what the user passed in STR. */
12465 {
12467 }
12470 /* Print a hint with a suggestion for an extension name
12471 that most closely resembles what the user passed in STR. */
12473 void
12475 {
12483 else
12487 }
12489 /* Validate a command-line -mcpu option. Parse the cpu and extensions (if any)
12490 specified in STR and throw errors if appropriate. Put the results if
12491 they are valid in RES and ISA_FLAGS. Return whether the option is
12492 valid. */
12494 static bool
12497 {
12499 enum aarch64_parse_opt_result parse_res
12506 {
12521 }
12524 }
12526 /* Parses CONST_STR for branch protection features specified in
12527 aarch64_branch_protect_types, and set any global variables required. Returns
12528 the parsing result and assigns LAST_STR to the last processed token from
12529 CONST_STR so that it can be used for error reporting. */
12531 static enum
12534 {
12541 else
12542 {
12544 /* Reset the branch protection features to their defaults. */
12548 {
12551 /* Search for this type. */
12553 {
12555 {
12560 }
12561 else
12562 type++;
12563 }
12565 {
12567 /* Loop through each token until we find one that isn't a
12568 subtype. */
12570 {
12573 /* Search for the subtype. */
12576 {
12578 {
12583 }
12584 else
12585 subtype++;
12586 }
12587 }
12588 }
12591 }
12592 }
12593 /* Copy the last processed token into the argument to pass it back.
12594 Used by option and attribute validation to print the offending token. */
12596 {
12599 }
12601 {
12602 /* If needed, alloc the accepted string then copy in const_str.
12603 Used by override_option_after_change_1. */
12606 BRANCH_PROTECT_STR_MAX
12610 /* Forcibly null-terminate. */
12612 }
12614 }
12616 static bool
12618 {
12628 }
12630 /* Validate a command-line -march option. Parse the arch and extensions
12631 (if any) specified in STR and throw errors if appropriate. Put the
12632 results, if they are valid, in RES and ISA_FLAGS. Return whether the
12633 option is valid. */
12635 static bool
12638 {
12640 enum aarch64_parse_opt_result parse_res
12647 {
12662 }
12665 }
12667 /* Validate a command-line -mtune option. Parse the cpu
12668 specified in STR and throw errors if appropriate. Put the
12669 result, if it is valid, in RES. Return whether the option is
12670 valid. */
12672 static bool
12674 {
12675 enum aarch64_parse_opt_result parse_res
12682 {
12692 }
12694 }
12696 /* Return the CPU corresponding to the enum CPU.
12697 If it doesn't specify a cpu, return the default. */
12701 {
12705 /* The & 0x3f is to extract the bottom 6 bits that encode the
12706 default cpu as selected by the --with-cpu GCC configure option
12707 in config.gcc.
12708 ???: The whole TARGET_CPU_DEFAULT and AARCH64_CPU_DEFAULT_FLAGS
12709 flags mechanism should be reworked to make it more sane. */
12711 }
12713 /* Return the architecture corresponding to the enum ARCH.
12714 If it doesn't specify a valid architecture, return the default. */
12718 {
12725 }
12727 /* Return the VG value associated with -msve-vector-bits= value VALUE. */
12729 static poly_uint16
12731 {
12732 /* For now generate vector-length agnostic code for -msve-vector-bits=128.
12733 This ensures we can clearly distinguish SVE and Advanced SIMD modes when
12734 deciding which .md file patterns to use and when deciding whether
12735 something is a legitimate address or constant. */
12738 else
12740 }
12742 /* Implement TARGET_OPTION_OVERRIDE. This is called once in the beginning
12743 and is used to parse the -m{cpu,tune,arch} strings and setup the initial
12744 tuning structs. In particular it must set selected_tune and
12745 aarch64_isa_flags that define the available ISA features and tuning
12746 decisions. It must also set selected_arch as this will be used to
12747 output the .arch asm tags for each function. */
12749 static void
12751 {
12767 /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
12768 If either of -march or -mtune is given, they override their
12769 respective component of -mcpu. */
12781 #ifdef SUBTARGET_OVERRIDE_OPTIONS
12782 SUBTARGET_OVERRIDE_OPTIONS;
12783 #endif
12785 /* If the user did not specify a processor, choose the default
12786 one for them. This will be the CPU set during configuration using
12787 --with-cpu, otherwise it is "generic". */
12789 {
12791 {
12795 }
12796 else
12797 {
12798 /* Get default configure-time CPU. */
12801 }
12805 }
12806 /* If both -mcpu and -march are specified check that they are architecturally
12807 compatible, warn if they're not and prefer the -march ISA flags. */
12809 {
12811 {
12815 }
12820 }
12821 else
12822 {
12823 /* -mcpu but no -march. */
12830 }
12832 /* Set the arch as well as we will need it when outputing
12833 the .arch directive in assembly. */
12835 {
12838 }
12844 {
12845 #ifdef TARGET_ENABLE_BTI
12847 #else
12849 #endif
12850 }
12852 /* Return address signing is currently not supported for ILP32 targets. For
12853 LP64 targets use the configured option in the absence of a command-line
12854 option for -mbranch-protection. */
12856 {
12857 #ifdef TARGET_ENABLE_PAC_RET
12859 #else
12861 #endif
12862 }
12864 #ifndef HAVE_AS_MABI_OPTION
12865 /* The compiler may have been configured with 2.23.* binutils, which does
12866 not have support for ILP32. */
12869 #endif
12871 /* Convert -msve-vector-bits to a VG count. */
12877 /* Make sure we properly set up the explicit options. */
12886 /* The pass to insert speculation tracking runs before
12887 shrink-wrapping and the latter does not know how to update the
12888 tracking status. So disable it in this case. */
12894 /* Save these options as the default ones in case we push and pop them later
12895 while processing functions with potential target attributes. */
12896 target_option_default_node = target_option_current_node
12898 }
12900 /* Implement targetm.override_options_after_change. */
12902 static void
12904 {
12906 }
12910 {
12914 }
12916 void
12918 {
12920 }
12922 /* A checking mechanism for the implementation of the various code models. */
12923 static void
12925 {
12927 {
12929 {
12934 #ifdef HAVE_AS_SMALL_PIC_RELOCS
12936 ? AARCH64_CMODEL_SMALL_PIC
12938 #else
12940 #endif
12948 }
12949 }
12950 else
12952 }
12954 /* Implement TARGET_OPTION_SAVE. */
12956 static void
12958 {
12960 ptr->x_aarch64_branch_protection_string
12962 }
12964 /* Implements TARGET_OPTION_RESTORE. Restore the backend codegen decisions
12965 using the information saved in PTR. */
12967 static void
12969 {
12975 opts->x_aarch64_branch_protection_string
12978 {
12980 NULL);
12981 }
12984 }
12986 /* Implement TARGET_OPTION_PRINT. */
12988 static void
12990 {
13001 }
13005 void
13007 {
13009 }
13011 /* Restore or save the TREE_TARGET_GLOBALS from or to NEW_TREE.
13012 Used by aarch64_set_current_function and aarch64_pragma_target_parse to
13013 make sure optab availability predicates are recomputed when necessary. */
13015 void
13017 {
13022 else
13024 }
13026 /* Implement TARGET_SET_CURRENT_FUNCTION. Unpack the codegen decisions
13027 like tuning and ISA features from the DECL_FUNCTION_SPECIFIC_TARGET
13028 of the function, if such exists. This function may be called multiple
13029 times on a single function so use aarch64_previous_fndecl to avoid
13030 setting up identical state. */
13032 static void
13034 {
13038 tree old_tree = (aarch64_previous_fndecl
13044 /* If current function has no attributes but the previous one did,
13045 use the default node. */
13049 /* If nothing to do, return. #pragma GCC reset or #pragma GCC pop to
13050 the default have been handled by aarch64_save_restore_target_globals from
13051 aarch64_pragma_target_parse. */
13057 /* First set the target options. */
13061 }
13063 /* Enum describing the various ways we can handle attributes.
13064 In many cases we can reuse the generic option handling machinery. */
13066 enum aarch64_attr_opt_type
13067 {
13071 aarch64_attr_custom /* Attribute requires a custom handling function. */
13072 };
13074 /* All the information needed to handle a target attribute.
13075 NAME is the name of the attribute.
13076 ATTR_TYPE specifies the type of behavior of the attribute as described
13077 in the definition of enum aarch64_attr_opt_type.
13078 ALLOW_NEG is true if the attribute supports a "no-" form.
13079 HANDLER is the function that takes the attribute string as an argument
13080 It is needed only when the ATTR_TYPE is aarch64_attr_custom.
13081 OPT_NUM is the enum specifying the option that the attribute modifies.
13082 This is needed for attributes that mirror the behavior of a command-line
13083 option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
13084 aarch64_attr_enum. */
13086 struct aarch64_attribute_info
13087 {
13093 };
13095 /* Handle the ARCH_STR argument to the arch= target attribute. */
13097 static bool
13099 {
13102 enum aarch64_parse_opt_result parse_res
13106 {
13111 }
13114 {
13129 }
13132 }
13134 /* Handle the argument CPU_STR to the cpu= target attribute. */
13136 static bool
13138 {
13141 enum aarch64_parse_opt_result parse_res
13145 {
13153 }
13156 {
13171 }
13174 }
13176 /* Handle the argument STR to the branch-protection= attribute. */
13178 static bool
13180 {
13186 {
13197 /* Fall through. */
13202 }
13205 }
13207 /* Handle the argument STR to the tune= target attribute. */
13209 static bool
13211 {
13213 enum aarch64_parse_opt_result parse_res
13217 {
13222 }
13225 {
13232 }
13235 }
13237 /* Parse an architecture extensions target attribute string specified in STR.
13238 For example "+fp+nosimd". Show any errors if needed. Return TRUE
13239 if successful. Update aarch64_isa_flags to reflect the ISA features
13240 modified. */
13242 static bool
13244 {
13248 /* We allow "+nothing" in the beginning to clear out all architectural
13249 features if the user wants to handpick specific features. */
13251 {
13254 }
13260 {
13263 }
13266 {
13278 }
13281 }
13283 /* The target attributes that we support. On top of these we also support just
13284 ISA extensions, like __attribute__ ((target ("+crc"))), but that case is
13285 handled explicitly in aarch64_process_one_target_attr. */
13288 {
13290 OPT_mgeneral_regs_only },
13292 OPT_mfix_cortex_a53_835769 },
13294 OPT_mfix_cortex_a53_843419 },
13298 OPT_momit_leaf_frame_pointer },
13301 OPT_march_ },
13304 OPT_mtune_ },
13308 OPT_msign_return_address_ },
13310 };
13312 /* Parse ARG_STR which contains the definition of one target attribute.
13313 Show appropriate errors if any or return true if the attribute is valid. */
13315 static bool
13317 {
13323 {
13326 }
13331 /* We have something like __attribute__ ((target ("+fp+nosimd"))).
13332 It is easier to detect and handle it explicitly here rather than going
13333 through the machinery for the rest of the target attributes in this
13334 function. */
13339 {
13342 }
13345 /* If we found opt=foo then terminate STR_TO_CHECK at the '='
13346 and point ARG to "foo". */
13348 {
13350 arg++;
13351 }
13355 {
13356 /* If the names don't match up, or the user has given an argument
13357 to an attribute that doesn't accept one, or didn't give an argument
13358 to an attribute that expects one, fail to match. */
13367 {
13370 }
13372 /* If the name matches but the attribute does not allow "no-" versions
13373 then we can't match. */
13375 {
13378 }
13381 {
13382 /* Has a custom handler registered.
13383 For example, cpu=, arch=, tune=. */
13390 /* Either set or unset a boolean option. */
13392 {
13400 }
13401 /* Set or unset a bit in the target_flags. aarch64_handle_option
13402 should know what mask to apply given the option number. */
13404 {
13406 /* We only need to specify the option number.
13407 aarch64_handle_option will know which mask to apply. */
13413 }
13414 /* Use the option setting machinery to set an option to an enum. */
13416 {
13423 {
13426 global_dc);
13427 }
13428 else
13429 {
13431 }
13433 }
13436 }
13437 }
13439 /* If we reached here we either have found an attribute and validated
13440 it or didn't match any. If we matched an attribute but its arguments
13441 were malformed we will have returned false already. */
13443 }
13445 /* Count how many times the character C appears in
13446 NULL-terminated string STR. */
13448 static unsigned int
13450 {
13453 {
13455 res++;
13457 str++;
13458 }
13461 }
13463 /* Parse the tree in ARGS that contains the target attribute information
13464 and update the global target options space. */
13466 bool
13468 {
13470 {
13471 do
13472 {
13475 {
13478 }
13483 }
13486 {
13489 }
13496 {
13499 }
13501 /* Used to catch empty spaces between commas i.e.
13502 attribute ((target ("attr1,,attr2"))). */
13505 /* Handle multiple target attributes separated by ','. */
13510 {
13511 num_attrs++;
13513 {
13516 }
13519 }
13522 {
13525 }
13528 }
13530 /* Implement TARGET_OPTION_VALID_ATTRIBUTE_P. This is used to
13531 process attribute ((target ("..."))). */
13533 static bool
13535 {
13538 tree old_optimize;
13542 /* If what we're processing is the current pragma string then the
13543 target option node is already stored in target_option_current_node
13544 by aarch64_pragma_target_parse in aarch64-c.c. Use that to avoid
13545 having to re-parse the string. This is especially useful to keep
13546 arm_neon.h compile times down since that header contains a lot
13547 of intrinsics enclosed in pragmas. */
13549 {
13552 }
13558 /* If the function changed the optimization levels as well as setting
13559 target options, start with the optimizations specified. */
13564 /* Save the current target options to restore at the end. */
13567 /* If fndecl already has some target attributes applied to it, unpack
13568 them so that we add this attribute on top of them, rather than
13569 overwriting them. */
13571 {
13577 }
13578 else
13584 /* Set up any additional state. */
13586 {
13588 /* Initialize SIMD builtins if we haven't already.
13589 Set current_target_pragma to NULL for the duration so that
13590 the builtin initialization code doesn't try to tag the functions
13591 being built with the attributes specified by any current pragma, thus
13592 going into an infinite recursion. */
13594 {
13599 }
13601 }
13602 else
13608 {
13613 }
13621 }
13623 /* Helper for aarch64_can_inline_p. In the case where CALLER and CALLEE are
13624 tri-bool options (yes, no, don't care) and the default value is
13625 DEF, determine whether to reject inlining. */
13627 static bool
13630 {
13631 /* If the callee doesn't care, always allow inlining. */
13635 /* If the caller doesn't care, always allow inlining. */
13639 /* Otherwise, allow inlining if either the callee and caller values
13640 agree, or if the callee is using the default value. */
13642 }
13644 /* Implement TARGET_CAN_INLINE_P. Decide whether it is valid
13645 to inline CALLEE into CALLER based on target-specific info.
13646 Make sure that the caller and callee have compatible architectural
13647 features. Then go through the other possible target attributes
13648 and see if they can block inlining. Try not to reject always_inline
13649 callees unless they are incompatible architecturally. */
13651 static bool
13653 {
13665 /* Callee's ISA flags should be a subset of the caller's. */
13670 /* Allow non-strict aligned functions inlining into strict
13671 aligned ones. */
13681 /* If the architectural features match up and the callee is always_inline
13682 then the other attributes don't matter. */
13694 /* Honour explicit requests to workaround errata. */
13707 /* If the user explicitly specified -momit-leaf-frame-pointer for the
13708 caller and calle and they don't match up, reject inlining. */
13715 /* If the callee has specific tuning overrides, respect them. */
13720 /* If the user specified tuning override strings for the
13721 caller and callee and they don't match up, reject inlining.
13722 We just do a string compare here, we don't analyze the meaning
13723 of the string, as it would be too costly for little gain. */
13731 }
13733 /* Return true if SYMBOL_REF X binds locally. */
13735 static bool
13737 {
13741 }
13743 /* Return true if SYMBOL_REF X is thread local */
13744 static bool
13746 {
13754 }
13756 /* Classify a TLS symbol into one of the TLS kinds. */
13757 enum aarch64_symbol_type
13759 {
13763 {
13770 {
13776 }
13787 else
13796 }
13797 }
13799 /* Return the correct method for accessing X + OFFSET, where X is either
13800 a SYMBOL_REF or LABEL_REF. */
13802 enum aarch64_symbol_type
13804 {
13806 {
13808 {
13823 }
13824 }
13827 {
13832 {
13834 /* When we retrieve symbol + offset address, we have to make sure
13835 the offset does not cause overflow of the final address. But
13836 we have no way of knowing the address of symbol at compile time
13837 so we can't accurately say if the distance between the PC and
13838 symbol + offset is outside the addressible range of +/-1M in the
13839 TINY code model. So we rely on images not being greater than
13840 1M and cap the offset at 1M and anything beyond 1M will have to
13841 be loaded using an alternative mechanism. Furthermore if the
13842 symbol is a weak reference to something that isn't known to
13843 resolve to a symbol in this module, then force to memory. */
13851 /* Same reasoning as the tiny code model, but the offset cap here is
13852 4G. */
13873 /* This is alright even in PIC code as the constant
13874 pool reference is always PC relative and within
13875 the same translation unit. */
13878 else
13883 }
13884 }
13886 /* By default push everything into the constant pool. */
13888 }
13890 bool
13892 {
13894 }
13896 bool
13898 {
13906 }
13908 /* Implement TARGET_LEGITIMATE_CONSTANT_P hook. Return true for constants
13909 that should be rematerialized rather than spilled. */
13911 static bool
13913 {
13914 /* Support CSE and rematerialization of common constants. */
13920 /* Do not allow vector struct mode constants for Advanced SIMD.
13921 We could support 0 and -1 easily, but they need support in
13922 aarch64-simd.md. */
13927 /* Only accept variable-length vector constants if they can be
13928 handled directly.
13930 ??? It would be possible to handle rematerialization of other
13931 constants via secondary reloads. */
13938 /* Accept polynomial constants that can be calculated by using the
13939 destination of a move as the sole temporary. Constants that
13940 require a second temporary cannot be rematerialized (they can't be
13941 forced to memory and also aren't legitimate constants). */
13942 poly_int64 offset;
13946 /* If an offset is being added to something else, we need to allow the
13947 base to be moved into the destination register, meaning that there
13948 are no free temporaries for the offset. */
13953 /* Do not allow const (plus (anchor_symbol, const_int)). */
13957 /* Treat symbols as constants. Avoid TLS symbols as they are complex,
13958 so spilling them is better than rematerialization. */
13962 /* Label references are always constant. */
13967 }
13969 rtx
13971 {
13977 /* Can return in any reg. */
13980 }
13982 /* On AAPCS systems, this is the "struct __va_list". */
13985 /* Implement TARGET_BUILD_BUILTIN_VA_LIST.
13986 Return the type to use as __builtin_va_list.
13988 AAPCS64 \S 7.1.4 requires that va_list be a typedef for a type defined as:
13990 struct __va_list
13991 {
13992 void *__stack;
13993 void *__gr_top;
13994 void *__vr_top;
13995 int __gr_offs;
13996 int __vr_offs;
13997 }; */
13999 static tree
14001 {
14002 tree va_list_name;
14005 /* Create the type. */
14007 /* Give it the required name. */
14009 TYPE_DECL,
14011 va_list_type);
14016 /* Create the fields. */
14019 ptr_type_node);
14022 ptr_type_node);
14025 ptr_type_node);
14028 integer_type_node);
14031 integer_type_node);
14033 /* Tell tree-stdarg pass about our internal offset fields.
14034 NOTE: va_list_gpr/fpr_counter_field are only used for tree comparision
14035 purpose to identify whether the code is updating va_list internal
14036 offset fields through irregular way. */
14058 /* Compute its layout. */
14062 }
14064 /* Implement TARGET_EXPAND_BUILTIN_VA_START. */
14065 static void
14067 {
14071 tree t;
14085 {
14088 }
14097 NULL_TREE);
14099 NULL_TREE);
14101 NULL_TREE);
14103 NULL_TREE);
14105 NULL_TREE);
14107 /* Emit code to initialize STACK, which points to the next varargs stack
14108 argument. CUM->AAPCS_STACK_SIZE gives the number of stack words used
14109 by named arguments. STACK is 8-byte aligned. */
14116 /* Emit code to initialize GRTOP, the top of the GR save area.
14117 virtual_incoming_args_rtx should have been 16 byte aligned. */
14122 /* Emit code to initialize VRTOP, the top of the VR save area.
14123 This address is gr_save_area_bytes below GRTOP, rounded
14124 down to the next 16-byte boundary. */
14134 /* Emit code to initialize GROFF, the offset from GRTOP of the
14135 next GPR argument. */
14140 /* Likewise emit code to initialize VROFF, the offset from FTOP
14141 of the next VR argument. */
14145 }
14147 /* Implement TARGET_GIMPLIFY_VA_ARG_EXPR. */
14149 static tree
14152 {
14153 tree addr;
14159 machine_mode mode;
14183 align
14189 type,
14193 {
14194 /* No frontends can create types with variable-sized modes, so we
14195 shouldn't be asked to pass or return them. */
14198 /* TYPE passed in fp/simd registers. */
14210 {
14213 }
14216 {
14218 }
14219 }
14220 else
14221 {
14222 /* TYPE passed in general registers. */
14231 {
14236 }
14240 {
14242 }
14243 }
14245 /* Get a local temporary for the field value. */
14248 /* Emit code to branch if off >= 0. */
14254 {
14255 /* Emit: offs = (offs + 15) & -16. */
14261 }
14262 else
14265 /* Update ap.__[g|v]r_offs */
14270 /* String up. */
14274 /* [cond2] if (ap.__[g|v]r_offs > 0) */
14279 /* String up: make sure the assignment happens before the use. */
14283 /* Prepare the trees handling the argument that is passed on the stack;
14284 the top level node will store in ON_STACK. */
14287 {
14288 /* if (alignof(type) > 8) (arg = arg + 15) & -16; */
14293 }
14294 else
14296 /* Advance ap.__stack */
14301 /* String up roundup and advance. */
14304 /* String up with arg */
14306 /* Big-endianness related address adjustment. */
14309 {
14313 }
14318 /* Adjustment to OFFSET in the case of BIG_ENDIAN. */
14328 {
14329 /* type ha; // treat as "struct {ftype field[n];}"
14330 ... [computing offs]
14331 for (i = 0; i <nregs; ++i, offs += 16)
14332 ha.field[i] = *((ftype *)(ap.__vr_top + offs));
14333 return ha; */
14337 /* Declare a local variable. */
14341 /* Establish the base type. */
14343 {
14362 {
14366 }
14370 }
14372 /* *(field_ptr_t)&ha = *((field_ptr_t)vr_saved_area */
14380 /* ha.field[i] = *((field_ptr_t)vr_saved_area + i) */
14382 {
14392 }
14396 }
14406 }
14408 /* Implement TARGET_SETUP_INCOMING_VARARGS. */
14410 static void
14414 {
14416 CUMULATIVE_ARGS local_cum;
14420 /* The caller has advanced CUM up to, but not beyond, the last named
14421 argument. Advance a local copy of CUM past the last "real" named
14422 argument, to find out how many registers are left over. */
14426 /* Found out how many registers we need to save.
14427 Honor tree-stdvar analysis results. */
14436 {
14439 }
14442 {
14444 {
14447 /* virtual_incoming_args_rtx should have been 16-byte aligned. */
14455 }
14457 {
14458 /* We can't use move_block_from_reg, because it will use
14459 the wrong mode, storing D regs only. */
14463 /* Set OFF to the offset from virtual_incoming_args_rtx of
14464 the first vector register. The VR save area lies below
14465 the GR one, and is aligned to 16 bytes. */
14472 {
14480 }
14481 }
14482 }
14484 /* We don't save the size into *PRETEND_SIZE because we want to avoid
14485 any complication of having crtl->args.pretend_args_size changed. */
14490 }
14492 static void
14494 {
14497 {
14499 {
14502 }
14503 }
14506 {
14509 }
14511 /* When tracking speculation, we need a couple of call-clobbered registers
14512 to track the speculation state. It would be nice to just use
14513 IP0 and IP1, but currently there are numerous places that just
14514 assume these registers are free for other uses (eg pointer
14515 authentication). */
14517 {
14522 }
14523 }
14525 /* Walk down the type tree of TYPE counting consecutive base elements.
14526 If *MODEP is VOIDmode, then set it to the first valid floating point
14527 type. If a non-floating point type is found, or if a floating point
14528 type that doesn't match a non-VOIDmode *MODEP is found, then return -1,
14529 otherwise return the count in the sub-tree. */
14530 static int
14532 {
14533 machine_mode mode;
14534 HOST_WIDE_INT size;
14537 {
14567 /* Use V2SImode and V4SImode as representatives of all 64-bit
14568 and 128-bit vector types. */
14571 {
14580 }
14585 /* Vector modes are considered to be opaque: two vectors are
14586 equivalent for the purposes of being homogeneous aggregates
14587 if they are the same size. */
14594 {
14598 /* Can't handle incomplete types nor sizes that are not
14599 fixed. */
14606 || !index
14617 /* There must be no padding. */
14623 }
14626 {
14629 tree field;
14631 /* Can't handle incomplete types nor sizes that are not
14632 fixed. */
14638 {
14646 }
14648 /* There must be no padding. */
14654 }
14658 {
14659 /* These aren't very interesting except in a degenerate case. */
14662 tree field;
14664 /* Can't handle incomplete types nor sizes that are not
14665 fixed. */
14671 {
14679 }
14681 /* There must be no padding. */
14687 }
14691 }
14694 }
14696 /* Return TRUE if the type, as described by TYPE and MODE, is a short vector
14697 type as described in AAPCS64 \S 4.1.2.
14699 See the comment above aarch64_composite_type_p for the notes on MODE. */
14701 static bool
14703 machine_mode mode)
14704 {
14714 }
14716 /* Return TRUE if the type, as described by TYPE and MODE, is a composite
14717 type as described in AAPCS64 \S 4.3. This includes aggregate, union and
14718 array types. The C99 floating-point complex types are also considered
14719 as composite types, according to AAPCS64 \S 7.1.1. The complex integer
14720 types, which are GCC extensions and out of the scope of AAPCS64, are
14721 treated as composite types here as well.
14723 Note that MODE itself is not sufficient in determining whether a type
14724 is such a composite type or not. This is because
14725 stor-layout.c:compute_record_mode may have already changed the MODE
14726 (BLKmode) of a RECORD_TYPE TYPE to some other mode. For example, a
14727 structure with only one field may have its MODE set to the mode of the
14728 field. Also an integer mode whose size matches the size of the
14729 RECORD_TYPE type may be used to substitute the original mode
14730 (i.e. BLKmode) in certain circumstances. In other words, MODE cannot be
14731 solely relied on. */
14733 static bool
14735 machine_mode mode)
14736 {
14749 }
14751 /* Return TRUE if an argument, whose type is described by TYPE and MODE,
14752 shall be passed or returned in simd/fp register(s) (providing these
14753 parameter passing registers are available).
14755 Upon successful return, *COUNT returns the number of needed registers,
14756 *BASE_MODE returns the mode of the individual register and when IS_HAF
14757 is not NULL, *IS_HA indicates whether or not the argument is a homogeneous
14758 floating-point aggregate or a homogeneous short-vector aggregate. */
14760 static bool
14762 const_tree type,
14766 {
14774 {
14777 }
14779 {
14783 }
14785 {
14789 {
14792 }
14793 else
14795 }
14796 else
14801 }
14803 /* Implement TARGET_STRUCT_VALUE_RTX. */
14805 static rtx
14808 {
14810 }
14812 /* Implements target hook vector_mode_supported_p. */
14813 static bool
14815 {
14818 }
14820 /* Return the full-width SVE vector mode for element mode MODE, if one
14821 exists. */
14822 opt_machine_mode
14824 {
14826 {
14843 }
14844 }
14846 /* Return the 128-bit Advanced SIMD vector mode for element mode MODE,
14847 if it exists. */
14848 opt_machine_mode
14850 {
14852 {
14869 }
14870 }
14872 /* Return appropriate SIMD container
14873 for MODE within a vector of WIDTH bits. */
14874 static machine_mode
14876 {
14882 {
14885 else
14887 {
14900 }
14901 }
14903 }
14905 /* Return 128-bit container as the preferred SIMD mode for MODE. */
14906 static machine_mode
14908 {
14911 }
14913 /* Return a list of possible vector sizes for the vectorizer
14914 to iterate over. */
14915 static void
14917 {
14922 }
14924 /* Implement TARGET_MANGLE_TYPE. */
14928 {
14929 /* The AArch64 ABI documents say that "__va_list" has to be
14930 mangled as if it is in the "std" namespace. */
14934 /* Half-precision float. */
14938 /* Mangle AArch64-specific internal types. TYPE_NAME is non-NULL_TREE for
14939 builtin types. */
14943 /* Use the default mangling. */
14945 }
14947 /* Find the first rtx_insn before insn that will generate an assembly
14948 instruction. */
14952 {
14956 do
14957 {
14959 }
14963 }
14965 static bool
14967 {
14969 /* A number of these may be AArch32 only. */
14974 };
14977 {
14980 }
14983 }
14985 /* Check if there is a register dependency between a load and the insn
14986 for which we hold recog_data. */
14988 static bool
14990 {
14991 rtx load_reg;
15001 {
15007 }
15009 }
15012 /* When working around the Cortex-A53 erratum 835769,
15013 given rtx_insn INSN, return true if it is a 64-bit multiply-accumulate
15014 instruction and has a preceding memory instruction such that a NOP
15015 should be inserted between them. */
15017 bool
15019 {
15022 rtx body;
15035 /* aarch64_prev_real_insn can call recog_memoized on insns other than INSN.
15036 Restore recog state to INSN to avoid state corruption. */
15044 /* If the previous insn is a memory op and there is no dependency between
15045 it and the DImode madd, emit a NOP between them. If body is NULL then we
15046 have a complex memory operation, probably a load/store pair.
15047 Be conservative for now and emit a NOP. */
15054 }
15057 /* Implement FINAL_PRESCAN_INSN. */
15059 void
15061 {
15064 }
15067 /* Return true if BASE_OR_STEP is a valid immediate operand for an SVE INDEX
15068 instruction. */
15070 bool
15072 {
15075 }
15077 /* Return true if X is a valid immediate for the SVE ADD and SUB
15078 instructions. Negate X first if NEGATE_P is true. */
15080 bool
15082 {
15083 rtx elt;
15097 }
15099 /* Return true if X is a valid immediate operand for an SVE logical
15100 instruction such as AND. */
15102 bool
15104 {
15105 rtx elt;
15111 }
15113 /* Return true if X is a valid immediate for the SVE DUP and CPY
15114 instructions. */
15116 bool
15118 {
15119 rtx elt;
15129 }
15131 /* Return true if X is a valid immediate operand for an SVE CMP instruction.
15132 SIGNED_P says whether the operand is signed rather than unsigned. */
15134 bool
15136 {
15137 rtx elt;
15141 && (signed_p
15144 }
15146 /* Return true if X is a valid immediate operand for an SVE FADD or FSUB
15147 instruction. Negate X first if NEGATE_P is true. */
15149 bool
15151 {
15152 rtx elt;
15153 REAL_VALUE_TYPE r;
15169 }
15171 /* Return true if X is a valid immediate operand for an SVE FMUL
15172 instruction. */
15174 bool
15176 {
15177 rtx elt;
15179 /* GCC will never generate a multiply with an immediate of 2, so there is no
15180 point testing for it (even though it is a valid constant). */
15184 }
15186 /* Return true if replicating VAL32 is a valid 2-byte or 4-byte immediate
15187 for the Advanced SIMD operation described by WHICH and INSN. If INFO
15188 is nonnull, use it to describe valid immediates. */
15189 static bool
15194 {
15195 /* Try a 4-byte immediate with LSL. */
15198 {
15203 }
15205 /* Try a 2-byte immediate with LSL. */
15210 {
15215 }
15217 /* Try a 4-byte immediate with MSL, except for cases that MVN
15218 can handle. */
15221 {
15224 {
15229 }
15230 }
15233 }
15235 /* Return true if replicating VAL64 is a valid immediate for the
15236 Advanced SIMD operation described by WHICH. If INFO is nonnull,
15237 use it to describe valid immediates. */
15238 static bool
15242 {
15248 {
15259 /* Try using a replicated byte. */
15263 {
15267 }
15268 }
15270 /* Try using a bit-to-bytemask. */
15272 {
15275 {
15279 }
15281 {
15285 }
15286 }
15288 }
15290 /* Return true if replicating VAL64 gives a valid immediate for an SVE MOV
15291 instruction. If INFO is nonnull, use it to describe valid immediates. */
15293 static bool
15296 {
15300 {
15304 {
15309 }
15310 }
15313 {
15314 /* DUP with no shift. */
15318 }
15320 {
15321 /* DUP with LSL #8. */
15325 }
15327 {
15328 /* DUPM. */
15332 }
15334 }
15336 /* Return true if X is a valid SVE predicate. If INFO is nonnull, use
15337 it to describe valid immediates. */
15339 static bool
15341 {
15343 {
15347 }
15349 /* Analyze the value as a VNx16BImode. This should be relatively
15350 efficient, since rtx_vector_builder has enough built-in capacity
15351 to store all VLA predicate constants without needing the heap. */
15352 rtx_vector_builder builder;
15358 {
15362 {
15364 {
15367 }
15369 }
15370 }
15372 }
15374 /* Return true if OP is a valid SIMD immediate for the operation
15375 described by WHICH. If INFO is nonnull, use it to describe valid
15376 immediates. */
15377 bool
15380 {
15397 {
15406 }
15410 else
15413 scalar_float_mode elt_float_mode;
15416 {
15420 {
15424 }
15425 }
15433 /* Expand the vector constant out into a byte vector, with the least
15434 significant byte of the register first. */
15438 {
15439 /* The vector is provided in gcc endian-neutral fashion.
15440 For aarch64_be Advanced SIMD, it must be laid out in the vector
15441 register in reverse order. */
15453 {
15456 }
15457 }
15459 /* The immediate must repeat every eight bytes. */
15465 /* Get the repeating 8-byte value as an integer. No endian correction
15466 is needed here because bytes is already in lsb-first order. */
15474 else
15476 }
15478 /* Check whether X is a VEC_SERIES-like constant that starts at 0 and
15479 has a step in the range of INDEX. Return the index expression if so,
15480 otherwise return null. */
15481 rtx
15483 {
15490 }
15492 /* Check of immediate shift constants are within range. */
15493 bool
15495 {
15499 else
15501 }
15503 /* Return the bitmask CONST_INT to select the bits required by a zero extract
15504 operation of width WIDTH at bit position POS. */
15506 rtx
15508 {
15512 unsigned HOST_WIDE_INT mask
15515 }
15517 bool
15519 {
15528 {
15529 /* Require predicate constants to be VNx16BI before RA, so that we
15530 force everything to have a canonical form. */
15532 && !reload_completed
15538 }
15548 }
15550 /* Return a const_int vector of VAL. */
15551 rtx
15553 {
15556 }
15558 /* Check OP is a legal scalar immediate for the MOVI instruction. */
15560 bool
15562 {
15563 machine_mode vmode;
15568 }
15570 /* Construct and return a PARALLEL RTX vector with elements numbering the
15571 lanes of either the high (HIGH == TRUE) or low (HIGH == FALSE) half of
15572 the vector - from the perspective of the architecture. This does not
15573 line up with GCC's perspective on lane numbers, so we end up with
15574 different masks depending on our target endian-ness. The diagram
15575 below may help. We must draw the distinction when building masks
15576 which select one half of the vector. An instruction selecting
15577 architectural low-lanes for a big-endian target, must be described using
15578 a mask selecting GCC high-lanes.
15580 Big-Endian Little-Endian
15582 GCC 0 1 2 3 3 2 1 0
15583 | x | x | x | x | | x | x | x | x |
15584 Architecture 3 2 1 0 3 2 1 0
15586 Low Mask: { 2, 3 } { 0, 1 }
15587 High Mask: { 0, 1 } { 2, 3 }
15589 MODE Is the mode of the vector and NUNITS is the number of units in it. */
15591 rtx
15593 {
15598 rtx t1;
15603 else
15611 }
15613 /* Check OP for validity as a PARALLEL RTX vector with elements
15614 numbering the lanes of either the high (HIGH == TRUE) or low lanes,
15615 from the perspective of the architecture. See the diagram above
15616 aarch64_simd_vect_par_cnst_half for more details. */
15618 bool
15621 {
15635 {
15642 }
15644 }
15646 /* Return a PARALLEL containing NELTS elements, with element I equal
15647 to BASE + I * STEP. */
15649 rtx
15651 {
15656 }
15658 /* Return true if OP is a PARALLEL of CONST_INTs that form a linear
15659 series with step STEP. */
15661 bool
15663 {
15674 }
15676 /* Bounds-check lanes. Ensure OPERAND lies between LOW (inclusive) and
15677 HIGH (exclusive). */
15678 void
15680 const_tree exp)
15681 {
15682 HOST_WIDE_INT lane;
15687 {
15690 else
15692 }
15693 }
15695 /* Peform endian correction on lane number N, which indexes a vector
15696 of mode MODE, and return the result as an SImode rtx. */
15698 rtx
15700 {
15702 }
15704 /* Return TRUE if OP is a valid vector addressing mode. */
15706 bool
15708 {
15711 }
15713 /* Return true if OP is a valid MEM operand for an SVE LD1R instruction. */
15715 bool
15717 {
15719 scalar_mode mode;
15726 }
15728 /* Return true if OP is a valid MEM operand for an SVE LD1RQ instruction. */
15729 bool
15731 {
15745 }
15747 /* Return true if OP is a valid MEM operand for an SVE LDR instruction.
15748 The conditions for STR are the same. */
15749 bool
15751 {
15758 }
15760 /* Return true if OP is a valid MEM operand for an SVE_STRUCT mode.
15761 We need to be able to access the individual pieces, so the range
15762 is different from LD[234] and ST[234]. */
15763 bool
15765 {
15772 ADDR_QUERY_ANY)
15780 }
15782 /* Emit a register copy from operand to operand, taking care not to
15783 early-clobber source registers in the process.
15785 COUNT is the number of components into which the copy needs to be
15786 decomposed. */
15787 void
15790 {
15800 else
15804 }
15806 /* Compute and return the length of aarch64_simd_reglist<mode>, where <mode> is
15807 one of VSTRUCT modes: OI, CI, or XI. */
15808 int
15810 {
15811 /* This is only used (and only meaningful) for Advanced SIMD, not SVE. */
15813 }
15815 /* Implement target hook TARGET_VECTOR_ALIGNMENT. The AAPCS64 sets the maximum
15816 alignment of a vector to 128 bits. SVE predicates have an alignment of
15817 16 bits. */
15818 static HOST_WIDE_INT
15820 {
15822 /* ??? Checking the mode isn't ideal, but VECTOR_BOOLEAN_TYPE_P can
15823 be set for non-predicate vectors of booleans. Modes are the most
15824 direct way we have of identifying real SVE predicate types. */
15827 }
15829 /* Implement target hook TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT. */
15830 static poly_uint64
15832 {
15834 {
15835 /* If the length of the vector is fixed, try to align to that length,
15836 otherwise don't try to align at all. */
15837 HOST_WIDE_INT result;
15841 }
15843 }
15845 /* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE. */
15846 static bool
15848 {
15852 /* For fixed-length vectors, check that the vectorizer will aim for
15853 full-vector alignment. This isn't true for generic GCC vectors
15854 that are wider than the ABI maximum of 128 bits. */
15855 poly_uint64 preferred_alignment =
15859 preferred_alignment))
15862 /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned. */
15864 }
15866 /* Return true if the vector misalignment factor is supported by the
15867 target. */
15868 static bool
15872 {
15874 {
15875 /* Return if movmisalign pattern is not supported for this mode. */
15879 /* Misalignment factor is unknown at compile time. */
15882 }
15884 is_packed);
15885 }
15887 /* If VALS is a vector constant that can be loaded into a register
15888 using DUP, generate instructions to do so and return an RTX to
15889 assign to the register. Otherwise return NULL_RTX. */
15890 static rtx
15892 {
15895 rtx x;
15900 /* We can load this constant by using DUP and a constant in a
15901 single ARM register. This will be cheaper than a vector
15902 load. */
15905 }
15908 /* Generate code to load VALS, which is a PARALLEL containing only
15909 constants (for vec_init) or CONST_VECTOR, efficiently into a
15910 register. Returns an RTX to copy into the register, or NULL_RTX
15911 for a PARALLEL that cannot be converted into a CONST_VECTOR. */
15912 static rtx
15914 {
15916 rtx const_dup;
15924 {
15925 /* A CONST_VECTOR must contain only CONST_INTs and
15926 CONST_DOUBLEs, but CONSTANT_P allows more (e.g. SYMBOL_REF).
15927 Only store valid constants in a CONST_VECTOR. */
15930 {
15933 n_const++;
15934 }
15937 }
15938 else
15943 /* Load using MOVI/MVNI. */
15946 /* Loaded using DUP. */
15949 /* Load from constant pool. We cannot take advantage of single-cycle
15950 LD1 because we need a PC-relative addressing mode. */
15952 else
15953 /* A PARALLEL containing something not valid inside CONST_VECTOR.
15954 We cannot construct an initializer. */
15956 }
15958 /* Expand a vector initialisation sequence, such that TARGET is
15959 initialised to contain VALS. */
15961 void
15963 {
15966 /* The number of vector elements. */
15968 /* The number of vector elements which are not constant. */
15971 /* The first element of vals. */
15975 /* This is a special vec_init<M><N> where N is not an element mode but a
15976 vector mode with half the elements of M. We expect to find two entries
15977 of mode N in VALS and we must put their concatentation into TARGET. */
15979 {
15988 /* When we want to concatenate a half-width vector with zeroes we can
15989 use the aarch64_combinez[_be] patterns. Just make sure that the
15990 zeroes are in the right half. */
15999 else
16000 {
16001 /* Else create the two half-width registers and combine them. */
16010 }
16012 }
16014 /* Count the number of variable elements to initialise. */
16016 {
16020 else
16024 }
16026 /* No variable elements, hand off to aarch64_simd_make_constant which knows
16027 how best to handle this. */
16029 {
16032 {
16035 }
16036 }
16038 /* Splat a single non-constant element if we can. */
16040 {
16044 }
16049 /* If there are only variable elements, try to optimize
16050 the insertion using dup for the most common element
16051 followed by insertions. */
16053 /* The algorithm will fill matches[*][0] with the earliest matching element,
16054 and matches[X][1] with the count of duplicate elements (if X is the
16055 earliest element which has duplicates). */
16058 {
16061 {
16063 {
16065 {
16069 }
16070 }
16071 }
16076 {
16079 }
16081 /* Create a duplicate of the most common element, unless all elements
16082 are equally useless to us, in which case just immediately set the
16083 vector register using the first element. */
16086 {
16087 /* For vectors of two 64-bit elements, we can do even better. */
16092 {
16095 /* Combine can pick up this case, but handling it directly
16096 here leaves clearer RTL.
16098 This is load_pair_lanes<mode>, and also gives us a clean-up
16099 for store_pair_lanes<mode>. */
16102 && !STRICT_ALIGNMENT
16107 {
16108 rtx t;
16111 else
16115 }
16116 }
16117 /* The subreg-move sequence below will move into lane zero of the
16118 vector register. For big-endian we want that position to hold
16119 the last element of VALS. */
16123 }
16124 else
16125 {
16128 }
16130 /* Insert the rest. */
16132 {
16138 }
16140 }
16142 /* Initialise a vector which is part-variable. We want to first try
16143 to build those lanes which are constant in the most efficient way we
16144 can. */
16146 {
16149 /* Load constant part of vector. We really don't care what goes into the
16150 parts we will overwrite, but we're more likely to be able to load the
16151 constant efficiently if it has fewer, larger, repeating parts
16152 (see aarch64_simd_valid_immediate). */
16154 {
16160 {
16161 /* Look in the copied vector, as more elements are const. */
16164 {
16167 }
16168 }
16170 }
16172 }
16174 /* Insert the variable lanes directly. */
16176 {
16182 }
16183 }
16185 /* Emit RTL corresponding to:
16186 insr TARGET, ELEM. */
16188 static void
16190 {
16198 }
16200 /* Subroutine of aarch64_sve_expand_vector_init for handling
16201 trailing constants.
16202 This function works as follows:
16203 (a) Create a new vector consisting of trailing constants.
16204 (b) Initialize TARGET with the constant vector using emit_move_insn.
16205 (c) Insert remaining elements in TARGET using insr.
16206 NELTS is the total number of elements in original vector while
16207 while NELTS_REQD is the number of elements that are actually
16208 significant.
16210 ??? The heuristic used is to do above only if number of constants
16211 is at least half the total number of elements. May need fine tuning. */
16213 static bool
16214 aarch64_sve_expand_vector_init_handle_trailing_constants
16216 {
16223 i--)
16224 n_trailing_constants++;
16227 {
16238 }
16241 }
16243 /* Subroutine of aarch64_sve_expand_vector_init.
16244 Works as follows:
16245 (a) Initialize TARGET by broadcasting element NELTS_REQD - 1 of BUILDER.
16246 (b) Skip trailing elements from BUILDER, which are the same as
16247 element NELTS_REQD - 1.
16248 (c) Insert earlier elements in reverse order in TARGET using insr. */
16250 static void
16254 {
16269 }
16271 /* Subroutine of aarch64_sve_expand_vector_init to handle case
16272 when all trailing elements of builder are same.
16273 This works as follows:
16274 (a) Use expand_insn interface to broadcast last vector element in TARGET.
16275 (b) Insert remaining elements in TARGET using insr.
16277 ??? The heuristic used is to do above if number of same trailing elements
16278 is at least 3/4 of total number of elements, loosely based on
16279 heuristic from mostly_zeros_p. May need fine-tuning. */
16281 static bool
16282 aarch64_sve_expand_vector_init_handle_trailing_same_elem
16284 {
16287 {
16291 }
16294 }
16296 /* Initialize register TARGET from BUILDER. NELTS is the constant number
16297 of elements in BUILDER.
16299 The function tries to initialize TARGET from BUILDER if it fits one
16300 of the special cases outlined below.
16302 Failing that, the function divides BUILDER into two sub-vectors:
16303 v_even = even elements of BUILDER;
16304 v_odd = odd elements of BUILDER;
16306 and recursively calls itself with v_even and v_odd.
16308 if (recursive call succeeded for v_even or v_odd)
16309 TARGET = zip (v_even, v_odd)
16311 The function returns true if it managed to build TARGET from BUILDER
16312 with one of the special cases, false otherwise.
16314 Example: {a, 1, b, 2, c, 3, d, 4}
16316 The vector gets divided into:
16317 v_even = {a, b, c, d}
16318 v_odd = {1, 2, 3, 4}
16320 aarch64_sve_expand_vector_init(v_odd) hits case 1 and
16321 initialize tmp2 from constant vector v_odd using emit_move_insn.
16323 aarch64_sve_expand_vector_init(v_even) fails since v_even contains
16324 4 elements, so we construct tmp1 from v_even using insr:
16325 tmp1 = dup(d)
16326 insr tmp1, c
16327 insr tmp1, b
16328 insr tmp1, a
16330 And finally:
16331 TARGET = zip (tmp1, tmp2)
16332 which sets TARGET to {a, 1, b, 2, c, 3, d, 4}. */
16334 static bool
16337 {
16340 /* Case 1: Vector contains trailing constants. */
16346 /* Case 2: Vector contains leading constants. */
16355 {
16358 }
16360 /* Case 3: Vector contains trailing same element. */
16366 /* Case 4: Vector contains leading same element. */
16370 {
16373 }
16375 /* Avoid recursing below 4-elements.
16376 ??? The threshold 4 may need fine-tuning. */
16385 {
16388 }
16404 /* Initialize v_even and v_odd using INSR if it didn't match any of the
16405 special cases and zip v_even, v_odd. */
16416 }
16418 /* Initialize register TARGET from the elements in PARALLEL rtx VALS. */
16420 void
16422 {
16431 /* If neither sub-vectors of v could be initialized specially,
16432 then use INSR to insert all elements from v into TARGET.
16433 ??? This might not be optimal for vectors with large
16434 initializers like 16-element or above.
16435 For nelts < 4, it probably isn't useful to handle specially. */
16440 }
16442 static unsigned HOST_WIDE_INT
16444 {
16448 }
16450 /* Select a format to encode pointers in exception handling data. */
16451 int
16453 {
16456 {
16462 /* text+got+data < 4Gb. 4-byte signed relocs are sufficient
16463 for everything. */
16467 /* No assumptions here. 8-byte relocs required. */
16470 }
16472 }
16474 /* Output .variant_pcs for aarch64_vector_pcs function symbols. */
16476 static void
16478 {
16480 {
16484 }
16485 }
16487 /* The last .arch and .tune assembly strings that we printed. */
16491 /* Implement ASM_DECLARE_FUNCTION_NAME. Output the ISA features used
16492 by the function fndecl. */
16494 void
16496 tree fndecl)
16497 {
16503 else
16514 /* Only update the assembler .arch string if it is distinct from the last
16515 such string we printed. */
16518 {
16521 }
16523 /* Print the cpu name we're tuning for in the comments, might be
16524 useful to readers of the generated asm. Do it only when it changes
16525 from function to function and verbose assembly is requested. */
16530 {
16534 }
16538 /* Don't forget the type directive for ELF. */
16541 }
16543 /* Implement ASM_OUTPUT_DEF_FROM_DECLS. Output .variant_pcs for aliases. */
16545 void
16547 {
16552 }
16554 /* Implement ASM_OUTPUT_EXTERNAL. Output .variant_pcs for undefined
16555 function symbol references. */
16557 void
16559 {
16562 }
16564 /* Triggered after a .cfi_startproc directive is emitted into the assembly file.
16565 Used to output the .cfi_b_key_frame directive when signing the current
16566 function with the B key. */
16568 void
16570 {
16574 }
16576 /* Implements TARGET_ASM_FILE_START. Output the assembly header. */
16578 static void
16580 {
16597 }
16599 /* Emit load exclusive. */
16601 static void
16604 {
16606 }
16608 /* Emit store exclusive. */
16610 static void
16613 {
16615 }
16617 /* Mark the previous jump instruction as unlikely. */
16619 static void
16621 {
16624 }
16626 /* Expand a compare and swap pattern. */
16628 void
16630 {
16644 /* Normally the succ memory model must be stronger than fail, but in the
16645 unlikely event of fail being ACQUIRE and succ being RELEASE we need to
16646 promote succ to ACQ_REL so that we don't lose the acquire semantics. */
16653 {
16656 }
16659 {
16660 /* The CAS insn requires oldval and rval overlap, but we need to
16661 have a copy of oldval saved across the operation to tell if
16662 the operation is successful. */
16665 else
16671 }
16672 else
16673 {
16674 /* The oldval predicate varies by mode. Test it and force to reg. */
16682 }
16690 }
16692 /* Emit a barrier, that is appropriate for memory model MODEL, at the end of a
16693 sequence implementing an atomic operation. */
16695 static void
16697 {
16704 {
16706 }
16707 }
16709 /* Split a compare and swap pattern. */
16711 void
16713 {
16715 machine_mode mode;
16720 rtx model_rtx;
16732 /* When OLDVAL is zero and we want the strong version we can emit a tighter
16733 loop:
16734 .label1:
16735 LD[A]XR rval, [mem]
16736 CBNZ rval, .label2
16737 ST[L]XR scratch, newval, [mem]
16738 CBNZ scratch, .label1
16739 .label2:
16740 CMP rval, 0. */
16745 {
16748 }
16751 /* The initial load can be relaxed for a __sync operation since a final
16752 barrier will be emitted to stop code hoisting. */
16756 else
16760 {
16762 {
16763 /* Emit an explicit compare instruction, so that we can correctly
16764 track the condition codes. */
16767 }
16768 else
16774 }
16775 else
16776 {
16782 }
16787 {
16789 {
16790 /* Emit an explicit compare instruction, so that we can correctly
16791 track the condition codes. */
16794 }
16795 else
16801 }
16802 else
16803 {
16807 }
16810 /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL
16811 to set the condition flags. If this is not used it will be removed by
16812 later passes. */
16814 {
16818 }
16819 /* Emit any final barrier needed for a __sync operation. */
16822 }
16824 /* Split an atomic operation. */
16826 void
16829 {
16835 rtx x;
16837 /* Split the atomic operation into a sequence. */
16845 else
16849 /* The initial load can be relaxed for a __sync operation since a final
16850 barrier will be emitted to stop code hoisting. */
16854 else
16858 {
16872 {
16875 }
16876 /* Fall through. */
16882 }
16888 {
16889 /* Emit an explicit compare instruction, so that we can correctly
16890 track the condition codes. */
16893 }
16894 else
16901 /* Emit any final barrier needed for a __sync operation. */
16904 }
16906 static void
16908 {
16909 /* Half-precision float operations. The compiler handles all operations
16910 with NULL libfuncs by converting to SFmode. */
16912 /* Conversions. */
16916 /* Arithmetic. */
16923 /* Comparisons. */
16931 }
16933 /* Target hook for c_mode_for_suffix. */
16934 static machine_mode
16936 {
16941 }
16943 /* We can only represent floating point constants which will fit in
16944 "quarter-precision" values. These values are characterised by
16945 a sign bit, a 4-bit mantissa and a 3-bit exponent. And are given
16946 by:
16948 (-1)^s * (n/16) * 2^r
16950 Where:
16951 's' is the sign bit.
16952 'n' is an integer in the range 16 <= n <= 31.
16953 'r' is an integer in the range -3 <= r <= 4. */
16955 /* Return true iff X can be represented by a quarter-precision
16956 floating point immediate operand X. Note, we cannot represent 0.0. */
16957 bool
16959 {
16960 /* This represents our current view of how many bits
16961 make up the mantissa. */
16977 /* We cannot represent infinities, NaNs or +/-zero. We won't
16978 know if we have +zero until we analyse the mantissa, but we
16979 can reject the other invalid values. */
16984 /* Extract exponent. */
16988 /* For the mantissa, we expand into two HOST_WIDE_INTS, apart from the
16989 highest (sign) bit, with a fixed binary point at bit point_pos.
16990 m1 holds the low part of the mantissa, m2 the high part.
16991 WARNING: If we ever have a representation using more than 2 * H_W_I - 1
16992 bits for the mantissa, this can fail (low bits will be lost). */
16996 /* If the low part of the mantissa has bits set we cannot represent
16997 the value. */
17000 /* We have rejected the lower HOST_WIDE_INT, so update our
17001 understanding of how many bits lie in the mantissa and
17002 look only at the high HOST_WIDE_INT. */
17006 /* We can only represent values with a mantissa of the form 1.xxxx. */
17011 /* Having filtered unrepresentable values, we may now remove all
17012 but the highest 5 bits. */
17015 /* We cannot represent the value 0.0, so reject it. This is handled
17016 elsewhere. */
17020 /* Then, as bit 4 is always set, we can mask it off, leaving
17021 the mantissa in the range [0, 15]. */
17025 /* GCC internally does not use IEEE754-like encoding (where normalized
17026 significands are in the range [1, 2). GCC uses [0.5, 1) (see real.c).
17027 Our mantissa values are shifted 4 places to the left relative to
17028 normalized IEEE754 so we must modify the exponent returned by REAL_EXP
17029 by 5 places to correct for GCC's representation. */
17033 }
17035 /* Returns the string with the instruction for AdvSIMD MOVI, MVNI, ORR or BIC
17036 immediate with a CONST_VECTOR of MODE and WIDTH. WHICH selects whether to
17037 output MOVI/MVNI, ORR or BIC immediate. */
17041 {
17051 /* This will return true to show const_vector is legal for use as either
17052 a AdvSIMD MOVI instruction (or, implicitly, MVNI), ORR or BIC immediate.
17053 It will also update INFO to show how the immediate should be generated.
17054 WHICH selects whether to check for MOVI/MVNI, ORR or BIC. */
17062 {
17065 /* For FP zero change it to a CONST_INT 0 and use the integer SIMD
17066 move immediate path. */
17069 else
17070 {
17079 else
17083 }
17084 }
17089 {
17101 else
17105 }
17106 else
17107 {
17108 /* For AARCH64_CHECK_BIC and AARCH64_CHECK_ORR. */
17115 else
17119 }
17121 }
17125 {
17127 /* If a floating point number was passed and we desire to use it in an
17128 integer mode do the conversion to integer. */
17130 {
17135 }
17137 machine_mode vmode;
17138 /* use a 64 bit mode for everything except for DI/DF mode, where we use
17139 a 128 bit vector mode. */
17145 }
17147 /* Return the output string to use for moving immediate CONST_VECTOR
17148 into an SVE register. */
17152 {
17164 {
17167 {
17170 }
17171 else
17172 {
17179 else
17182 }
17184 }
17187 {
17193 }
17196 {
17199 else
17200 {
17210 }
17211 }
17216 }
17218 /* Split operands into moves from op[1] + op[2] into op[0]. */
17220 void
17222 {
17233 {
17234 /* No-op move. Can't split to nothing; emit something. */
17237 }
17239 /* Preserve register attributes for variable tracking. */
17244 /* Special case of reversed high/low parts. */
17247 {
17251 }
17253 {
17254 /* Try to avoid unnecessary moves if part of the result
17255 is in the right place already. */
17260 }
17261 else
17262 {
17267 }
17268 }
17270 /* vec_perm support. */
17272 struct expand_vec_perm_d
17273 {
17275 vec_perm_indices perm;
17276 machine_mode vmode;
17280 };
17282 /* Generate a variable permutation. */
17284 static void
17286 {
17297 {
17299 {
17300 /* Expand the argument to a V16QI mode by duplicating it. */
17304 }
17305 else
17306 {
17308 }
17309 }
17310 else
17311 {
17312 rtx pair;
17315 {
17319 }
17320 else
17321 {
17325 }
17326 }
17327 }
17329 /* Expand a vec_perm with the operands given by TARGET, OP0, OP1 and SEL.
17330 NELT is the number of elements in the vector. */
17332 void
17335 {
17338 rtx mask;
17340 /* The TBL instruction does not use a modulo index, so we must take care
17341 of that ourselves. */
17346 /* For big-endian, we also need to reverse the index within the vector
17347 (but not which vector). */
17349 {
17350 /* If one_vector_p, mask is a vector of (nelt - 1)'s already. */
17355 }
17357 }
17359 /* Generate (set TARGET (unspec [OP0 OP1] CODE)). */
17361 static void
17363 {
17367 }
17369 /* Expand an SVE vec_perm with the given operands. */
17371 void
17373 {
17376 /* Enforced by the pattern condition. */
17379 /* Note: vec_perm indices are supposed to wrap when they go beyond the
17380 size of the two value vectors, i.e. the upper bits of the indices
17381 are effectively ignored. SVE TBL instead produces 0 for any
17382 out-of-range indices, so we need to modulo all the vec_perm indices
17383 to ensure they are all in range. */
17386 /* Check if the sel only references the first values vector. */
17389 {
17392 }
17394 /* Check if the two values vectors are the same. */
17396 {
17402 }
17404 /* Run TBL on for each value vector and combine the results. */
17411 {
17416 }
17423 else
17425 }
17427 /* Recognize patterns suitable for the TRN instructions. */
17428 static bool
17430 {
17431 HOST_WIDE_INT odd;
17439 /* Note that these are little-endian tests.
17440 We correct for big-endian later. */
17447 /* Success! */
17453 /* We don't need a big-endian lane correction for SVE; see the comment
17454 at the head of aarch64-sve.md for details. */
17456 {
17459 }
17465 }
17467 /* Recognize patterns suitable for the UZP instructions. */
17468 static bool
17470 {
17471 HOST_WIDE_INT odd;
17478 /* Note that these are little-endian tests.
17479 We correct for big-endian later. */
17485 /* Success! */
17491 /* We don't need a big-endian lane correction for SVE; see the comment
17492 at the head of aarch64-sve.md for details. */
17494 {
17497 }
17503 }
17505 /* Recognize patterns suitable for the ZIP instructions. */
17506 static bool
17508 {
17517 /* Note that these are little-endian tests.
17518 We correct for big-endian later. */
17526 /* Success! */
17532 /* We don't need a big-endian lane correction for SVE; see the comment
17533 at the head of aarch64-sve.md for details. */
17535 {
17538 }
17544 }
17546 /* Recognize patterns for the EXT insn. */
17548 static bool
17550 {
17551 HOST_WIDE_INT location;
17552 rtx offset;
17554 /* The first element always refers to the first vector.
17555 Check if the extracted indices are increasing by one. */
17561 /* Success! */
17565 /* The case where (location == 0) is a no-op for both big- and little-endian,
17566 and is removed by the mid-end at optimization levels -O1 and higher.
17568 We don't need a big-endian lane correction for SVE; see the comment
17569 at the head of aarch64-sve.md for details. */
17571 {
17572 /* After setup, we want the high elements of the first vector (stored
17573 at the LSB end of the register), and the low elements of the second
17574 vector (stored at the MSB end of the register). So swap. */
17576 /* location != 0 (above), so safe to assume (nelt - location) < nelt.
17577 to_constant () is safe since this is restricted to Advanced SIMD
17578 vectors. */
17580 }
17586 UNSPEC_EXT));
17588 }
17590 /* Recognize patterns for the REV{64,32,16} insns, which reverse elements
17591 within each 64-bit, 32-bit or 16-bit granule. */
17593 static bool
17595 {
17596 HOST_WIDE_INT diff;
17598 machine_mode pred_mode;
17607 {
17610 }
17612 {
17615 }
17617 {
17620 }
17621 else
17629 /* Success! */
17635 {
17638 UNSPEC_PRED_X);
17639 }
17642 }
17644 /* Recognize patterns for the REV insn, which reverses elements within
17645 a full vector. */
17647 static bool
17649 {
17658 /* Success! */
17665 }
17667 static bool
17669 {
17671 rtx in0;
17672 HOST_WIDE_INT elt;
17674 rtx lane;
17684 /* Success! */
17688 /* The generic preparation in aarch64_expand_vec_perm_const_1
17689 swaps the operand order and the permute indices if it finds
17690 d->perm[0] to be in the second operand. Thus, we can always
17691 use d->op0 and need not do any extra arithmetic to get the
17692 correct lane number. */
17700 }
17702 static bool
17704 {
17708 /* Make sure that the indices are constant. */
17717 /* Generic code will try constant permutation twice. Once with the
17718 original mode and again with the elements lowered to QImode.
17719 So wait and don't do the selector expansion ourselves. */
17723 /* to_constant is safe since this routine is specific to Advanced SIMD
17724 vectors. */
17727 /* If big-endian and two vectors we end up with a weird mixed-endian
17728 mode on NEON. Reverse the index within each word but not the word
17729 itself. to_constant is safe because we checked is_constant above. */
17739 }
17741 /* Try to implement D using an SVE TBL instruction. */
17743 static bool
17745 {
17748 /* Permuting two variable-length vectors could overflow the
17749 index range. */
17760 else
17763 }
17765 static bool
17767 {
17768 /* The pattern matching functions above are written to look for a small
17769 number to begin the sequence (0, 1, N/2). If we begin with an index
17770 from the second operand, we can swap the operands. */
17773 {
17776 }
17782 {
17801 }
17803 }
17805 /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
17807 static bool
17810 {
17813 /* Check whether the mask can be applied to a single vector. */
17818 {
17821 }
17823 {
17826 }
17827 else
17847 }
17849 /* Generate a byte permute mask for a register of mode MODE,
17850 which has NUNITS units. */
17852 rtx
17854 {
17855 /* We have to reverse each vector because we dont have
17856 a permuted load that can reverse-load according to ABI rules. */
17857 rtx mask;
17870 }
17872 /* Expand an SVE integer comparison using the SVE equivalent of:
17874 (set TARGET (CODE OP0 OP1)). */
17876 void
17878 {
17885 }
17887 /* Return the UNSPEC_COND_* code for comparison CODE. */
17889 static unsigned int
17891 {
17893 {
17910 }
17911 }
17913 /* Emit:
17915 (set TARGET (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X>))
17917 where <X> is the operation associated with comparison CODE.
17918 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
17920 static void
17923 {
17929 }
17931 /* Emit the SVE equivalent of:
17933 (set TMP1 (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X1>))
17934 (set TMP2 (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X2>))
17935 (set TARGET (ior:PRED_MODE TMP1 TMP2))
17937 where <Xi> is the operation associated with comparison CODEi.
17938 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
17940 static void
17943 {
17950 }
17952 /* Emit the SVE equivalent of:
17954 (set TMP (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X>))
17955 (set TARGET (not TMP))
17957 where <X> is the operation associated with comparison CODE.
17958 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
17960 static void
17963 {
17968 }
17970 /* Expand an SVE floating-point comparison using the SVE equivalent of:
17972 (set TARGET (CODE OP0 OP1))
17974 If CAN_INVERT_P is true, the caller can also handle inverted results;
17975 return true if the result is in fact inverted. */
17977 bool
17980 {
17986 {
17988 /* UNORDERED has no immediate form. */
17990 /* fall through */
17997 {
17998 /* There is native support for the comparison. */
18001 }
18004 /* This is a trapping operation (LT or GT). */
18010 {
18011 /* This would trap for signaling NaNs. */
18016 }
18017 /* fall through */
18023 {
18024 /* Work out which elements are ordered. */
18030 /* Test the opposite condition for the ordered elements,
18031 then invert the result. */
18034 else
18037 {
18041 }
18045 }
18049 /* ORDERED has no immediate form. */
18055 }
18057 /* There is native support for the inverse comparison. */
18060 {
18063 }
18066 }
18068 /* Expand an SVE vcond pattern with operands OPS. DATA_MODE is the mode
18069 of the data being selected and CMP_MODE is the mode of the values being
18070 compared. */
18072 void
18075 {
18076 machine_mode pred_mode
18081 {
18085 }
18086 else
18091 }
18093 /* Implement TARGET_MODES_TIEABLE_P. In principle we should always return
18094 true. However due to issues with register allocation it is preferable
18095 to avoid tieing integer scalar and FP scalar modes. Executing integer
18096 operations in general registers is better than treating them as scalar
18097 vector operations. This reduces latency and avoids redundant int<->FP
18098 moves. So tie modes if they are either the same class, or vector modes
18099 with other vector modes, vector structs or any scalar mode. */
18101 static bool
18103 {
18107 /* We specifically want to allow elements of "structure" modes to
18108 be tieable to the structure. This more general condition allows
18109 other rarer situations too. The reason we don't extend this to
18110 predicate modes is that there are no predicate structure modes
18111 nor any specific instructions for extracting part of a predicate
18112 register. */
18117 /* Also allow any scalar modes with vectors. */
18123 }
18125 /* Return a new RTX holding the result of moving POINTER forward by
18126 AMOUNT bytes. */
18128 static rtx
18130 {
18135 }
18137 /* Return a new RTX holding the result of moving POINTER forward by the
18138 size of the mode it points to. */
18140 static rtx
18142 {
18144 }
18146 /* Copy one MODE sized block from SRC to DST, then progress SRC and DST by
18147 MODE bytes. */
18149 static void
18151 machine_mode mode)
18152 {
18155 /* "Cast" the pointers to the correct mode. */
18158 /* Emit the memcpy. */
18161 /* Move the pointers forward. */
18164 }
18166 /* Expand cpymem, as if from a __builtin_memcpy. Return true if
18167 we succeed, otherwise return false. */
18169 bool
18171 {
18175 rtx base;
18179 /* When optimizing for size, give a better estimate of the length of a
18180 memcpy call, but use the default otherwise. Moves larger than 8 bytes
18181 will always require an even number of instructions to do now. And each
18182 operation requires both a load+store, so devide the max number by 2. */
18185 /* We can't do anything smart if the amount to copy is not constant. */
18191 /* Try to keep the number of instructions low. For all cases we will do at
18192 most two moves for the residual amount, since we'll always overlap the
18193 remainder. */
18203 /* Convert n to bits to make the rest of the code simpler. */
18206 /* Maximum amount to copy in one go. The AArch64 back-end has integer modes
18207 larger than TImode, but we should not use them for loads/stores here. */
18211 {
18212 /* Find the largest mode in which to do the copy in without over reading
18213 or writing. */
18214 opt_scalar_int_mode mode_iter;
18226 /* Do certain trailing copies as overlapping if it's going to be
18227 cheaper. i.e. less instructions to do so. For instance doing a 15
18228 byte copy it's more efficient to do two overlapping 8 byte copies than
18229 8 + 6 + 1. */
18231 {
18237 }
18238 }
18241 }
18243 /* Split a DImode store of a CONST_INT SRC to MEM DST as two
18244 SImode stores. Handle the case when the constant has identical
18245 bottom and top halves. This is beneficial when the two stores can be
18246 merged into an STP and we avoid synthesising potentially expensive
18247 immediates twice. Return true if such a split is possible. */
18249 bool
18251 {
18260 unsigned int orig_cost
18262 unsigned int lo_cost
18265 /* We want to transform:
18266 MOV x1, 49370
18267 MOVK x1, 0x140, lsl 16
18268 MOVK x1, 0xc0da, lsl 32
18269 MOVK x1, 0x140, lsl 48
18270 STR x1, [x0]
18271 into:
18272 MOV w1, 49370
18273 MOVK w1, 0x140, lsl 16
18274 STP w1, w1, [x0]
18275 So we want to perform this only when we save two instructions
18276 or more. When optimizing for size, however, accept any code size
18277 savings we can. */
18292 /* Don't emit an explicit store pair as this may not be always profitable.
18293 Let the sched-fusion logic decide whether to merge them. */
18298 }
18300 /* Generate RTL for a conditional branch with rtx comparison CODE in
18301 mode CC_MODE. The destination of the unlikely conditional branch
18302 is LABEL_REF. */
18304 void
18306 rtx label_ref)
18307 {
18308 rtx x;
18311 const0_rtx);
18315 pc_rtx);
18317 }
18319 /* Generate DImode scratch registers for 128-bit (TImode) addition.
18321 OP1 represents the TImode destination operand 1
18322 OP2 represents the TImode destination operand 2
18323 LOW_DEST represents the low half (DImode) of TImode operand 0
18324 LOW_IN1 represents the low half (DImode) of TImode operand 1
18325 LOW_IN2 represents the low half (DImode) of TImode operand 2
18326 HIGH_DEST represents the high half (DImode) of TImode operand 0
18327 HIGH_IN1 represents the high half (DImode) of TImode operand 1
18328 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
18330 void
18335 {
18344 }
18346 /* Generate DImode scratch registers for 128-bit (TImode) subtraction.
18348 This function differs from 'arch64_addti_scratch_regs' in that
18349 OP1 can be an immediate constant (zero). We must call
18350 subreg_highpart_offset with DImode and TImode arguments, otherwise
18351 VOIDmode will be used for the const_int which generates an internal
18352 error from subreg_size_highpart_offset which does not expect a size of zero.
18354 OP1 represents the TImode destination operand 1
18355 OP2 represents the TImode destination operand 2
18356 LOW_DEST represents the low half (DImode) of TImode operand 0
18357 LOW_IN1 represents the low half (DImode) of TImode operand 1
18358 LOW_IN2 represents the low half (DImode) of TImode operand 2
18359 HIGH_DEST represents the high half (DImode) of TImode operand 0
18360 HIGH_IN1 represents the high half (DImode) of TImode operand 1
18361 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
18364 void
18369 {
18382 }
18384 /* Generate RTL for 128-bit (TImode) subtraction with overflow.
18386 OP0 represents the TImode destination operand 0
18387 LOW_DEST represents the low half (DImode) of TImode operand 0
18388 LOW_IN1 represents the low half (DImode) of TImode operand 1
18389 LOW_IN2 represents the low half (DImode) of TImode operand 2
18390 HIGH_DEST represents the high half (DImode) of TImode operand 0
18391 HIGH_IN1 represents the high half (DImode) of TImode operand 1
18392 HIGH_IN2 represents the high half (DImode) of TImode operand 2
18393 UNSIGNED_P is true if the operation is being performed on unsigned
18394 values. */
18395 void
18399 {
18401 {
18406 else
18408 }
18409 else
18410 {
18412 {
18416 }
18417 else
18422 else
18424 }
18429 }
18431 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
18433 static unsigned HOST_WIDE_INT
18435 {
18438 else
18440 }
18442 static rtx
18445 {
18449 insn_code icode;
18460 {
18488 }
18493 {
18496 }
18505 {
18508 }
18514 }
18516 static rtx
18519 {
18523 insn_code icode;
18535 {
18563 }
18568 {
18571 }
18579 {
18584 }
18595 {
18598 }
18604 }
18606 #undef TARGET_GEN_CCMP_FIRST
18607 #define TARGET_GEN_CCMP_FIRST aarch64_gen_ccmp_first
18609 #undef TARGET_GEN_CCMP_NEXT
18610 #define TARGET_GEN_CCMP_NEXT aarch64_gen_ccmp_next
18612 /* Implement TARGET_SCHED_MACRO_FUSION_P. Return true if target supports
18613 instruction fusion of some sort. */
18615 static bool
18617 {
18619 }
18622 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P. Return true if PREV and CURR
18623 should be kept together during scheduling. */
18625 static bool
18627 {
18628 rtx set_dest;
18631 /* prev and curr are simple SET insns i.e. no flag setting or branching. */
18638 {
18639 /* We are trying to match:
18640 prev (mov) == (set (reg r0) (const_int imm16))
18641 curr (movk) == (set (zero_extract (reg r0)
18642 (const_int 16)
18643 (const_int 16))
18644 (const_int imm16_1)) */
18656 {
18658 }
18659 }
18662 {
18664 /* We're trying to match:
18665 prev (adrp) == (set (reg r1)
18666 (high (symbol_ref ("SYM"))))
18667 curr (add) == (set (reg r0)
18668 (lo_sum (reg r1)
18669 (symbol_ref ("SYM"))))
18670 Note that r0 need not necessarily be the same as r1, especially
18671 during pre-regalloc scheduling. */
18675 {
18683 }
18684 }
18687 {
18689 /* We're trying to match:
18690 prev (movk) == (set (zero_extract (reg r0)
18691 (const_int 16)
18692 (const_int 32))
18693 (const_int imm16_1))
18694 curr (movk) == (set (zero_extract (reg r0)
18695 (const_int 16)
18696 (const_int 48))
18697 (const_int imm16_2)) */
18713 }
18715 {
18716 /* We're trying to match:
18717 prev (adrp) == (set (reg r0)
18718 (high (symbol_ref ("SYM"))))
18719 curr (ldr) == (set (reg r1)
18720 (mem (lo_sum (reg r0)
18721 (symbol_ref ("SYM")))))
18722 or
18723 curr (ldr) == (set (reg r1)
18724 (zero_extend (mem
18725 (lo_sum (reg r0)
18726 (symbol_ref ("SYM")))))) */
18729 {
18742 }
18743 }
18747 {
18749 rtx cc_reg_1;
18754 && prev
18756 {
18759 /* FIXME: this misses some which is considered simple arthematic
18760 instructions for ThunderX. Simple shifts are missed here. */
18766 }
18767 }
18770 && curr_set
18773 {
18774 /* We're trying to match:
18775 prev (alu_insn) == (set (r0) plus ((r0) (r1/imm)))
18776 curr (cbz) == (set (pc) (if_then_else (eq/ne) (r0)
18777 (const_int 0))
18778 (label_ref ("SYM"))
18779 (pc)) */
18786 {
18787 /* Fuse ALU operations followed by conditional branch instruction. */
18789 {
18810 }
18811 }
18812 }
18815 }
18817 /* Return true iff the instruction fusion described by OP is enabled. */
18819 bool
18821 {
18823 }
18825 /* If MEM is in the form of [base+offset], extract the two parts
18826 of address and set to BASE and OFFSET, otherwise return false
18827 after clearing BASE and OFFSET. */
18829 bool
18831 {
18832 rtx addr;
18839 {
18843 }
18847 {
18851 }
18857 }
18859 /* Types for scheduling fusion. */
18860 enum sched_fusion_type
18861 {
18863 SCHED_FUSION_LD_SIGN_EXTEND,
18864 SCHED_FUSION_LD_ZERO_EXTEND,
18865 SCHED_FUSION_LD,
18866 SCHED_FUSION_ST,
18867 SCHED_FUSION_NUM
18868 };
18870 /* If INSN is a load or store of address in the form of [base+offset],
18871 extract the two parts and set to BASE and OFFSET. Return scheduling
18872 fusion type this INSN is. */
18874 static enum sched_fusion_type
18876 {
18894 {
18899 }
18901 {
18906 }
18911 {
18914 }
18915 else
18922 }
18924 /* Implement the TARGET_SCHED_FUSION_PRIORITY hook.
18926 Currently we only support to fuse ldr or str instructions, so FUSION_PRI
18927 and PRI are only calculated for these instructions. For other instruction,
18928 FUSION_PRI and PRI are simply set to MAX_PRI - 1. In the future, other
18929 type instruction fusion can be added by returning different priorities.
18931 It's important that irrelevant instructions get the largest FUSION_PRI. */
18933 static void
18936 {
18946 {
18950 }
18952 /* Set FUSION_PRI according to fusion type and base register. */
18955 /* Calculate PRI. */
18958 /* INSN with smaller offset goes first. */
18962 else
18967 }
18969 /* Implement the TARGET_SCHED_ADJUST_PRIORITY hook.
18970 Adjust priority of sha1h instructions so they are scheduled before
18971 other SHA1 instructions. */
18973 static int
18975 {
18979 {
18984 }
18987 }
18989 /* Given OPERANDS of consecutive load/store, check if we can merge
18990 them into ldp/stp. LOAD is true if they are load instructions.
18991 MODE is the mode of memory operands. */
18993 bool
18995 machine_mode mode)
18996 {
19002 {
19010 }
19011 else
19012 {
19017 }
19019 /* The mems cannot be volatile. */
19023 /* If we have SImode and slow unaligned ldp,
19024 check the alignment to be at least 8 byte. */
19028 && !optimize_size
19032 /* Check if the addresses are in the form of [base+offset]. */
19040 /* Check if the bases are same. */
19044 /* The operands must be of the same size. */
19050 /* We should only be trying this for fixed-sized modes. There is no
19051 SVE LDP/STP instruction. */
19053 /* Check if the offsets are consecutive. */
19057 /* Check if the addresses are clobbered by load. */
19059 {
19063 /* In increasing order, the last load can clobber the address. */
19066 }
19068 /* One of the memory accesses must be a mempair operand.
19069 If it is not the first one, they need to be swapped by the
19070 peephole. */
19077 else
19082 else
19085 /* Check if the registers are of same class. */
19090 }
19092 /* Given OPERANDS of consecutive load/store that can be merged,
19093 swap them if they are not in ascending order. */
19094 void
19096 {
19101 {
19104 }
19105 else
19106 {
19109 }
19118 {
19119 /* Irrespective of whether this is a load or a store,
19120 we do the same swap. */
19123 }
19124 }
19126 /* Taking X and Y to be HOST_WIDE_INT pointers, return the result of a
19127 comparison between the two. */
19128 int
19130 {
19133 }
19135 /* Taking X and Y to be pairs of RTX, one pointing to a MEM rtx and the
19136 other pointing to a REG rtx containing an offset, compare the offsets
19137 of the two pairs.
19139 Return:
19141 1 iff offset (X) > offset (Y)
19142 0 iff offset (X) == offset (Y)
19143 -1 iff offset (X) < offset (Y) */
19144 int
19146 {
19153 else
19158 else
19161 /* Extract the offsets. */
19168 }
19170 /* Given OPERANDS of consecutive load/store, check if we can merge
19171 them into ldp/stp by adjusting the offset. LOAD is true if they
19172 are load instructions. MODE is the mode of memory operands.
19174 Given below consecutive stores:
19176 str w1, [xb, 0x100]
19177 str w1, [xb, 0x104]
19178 str w1, [xb, 0x108]
19179 str w1, [xb, 0x10c]
19181 Though the offsets are out of the range supported by stp, we can
19182 still pair them after adjusting the offset, like:
19184 add scratch, xb, 0x100
19185 stp w1, w1, [scratch]
19186 stp w1, w1, [scratch, 0x8]
19188 The peephole patterns detecting this opportunity should guarantee
19189 the scratch register is avaliable. */
19191 bool
19193 scalar_mode mode)
19194 {
19201 {
19203 {
19208 }
19210 /* Do not attempt to merge the loads if the loads clobber each other. */
19215 }
19216 else
19218 {
19221 }
19223 /* Skip if memory operand is by itself valid for ldp/stp. */
19228 {
19229 /* The mems cannot be volatile. */
19233 /* Check if the addresses are in the form of [base+offset]. */
19237 }
19239 /* Check if the registers are of same class. */
19245 {
19248 }
19249 else
19250 {
19253 }
19255 /* Only the last register in the order in which they occur
19256 may be clobbered by the load. */
19262 /* Check if the bases are same. */
19272 /* Check if the offsets can be put in the right order to do a ldp/stp. */
19274 aarch64_host_wide_int_compare);
19280 /* Check that offsets are within range of each other. The ldp/stp
19281 instructions have 7 bit immediate offsets, so use 0x80. */
19285 /* The offsets must be aligned with respect to each other. */
19289 /* If we have SImode and slow unaligned ldp,
19290 check the alignment to be at least 8 byte. */
19294 && !optimize_size
19299 }
19301 /* Given OPERANDS of consecutive load/store, this function pairs them
19302 into LDP/STP after adjusting the offset. It depends on the fact
19303 that the operands can be sorted so the offsets are correct for STP.
19304 MODE is the mode of memory operands. CODE is the rtl operator
19305 which should be applied to all memory operands, it's SIGN_EXTEND,
19306 ZERO_EXTEND or UNKNOWN. */
19308 bool
19311 {
19318 /* We make changes on a copy as we may still bail out. */
19322 /* Sort the operands. */
19325 /* Copy the memory operands so that if we have to bail for some
19326 reason the original addresses are unchanged. */
19328 {
19333 }
19334 else
19335 {
19341 }
19348 /* Adjust offset so it can fit in LDP/STP instruction. */
19356 /* The base offset is optimally half way between the two STP/LDP offsets. */
19359 else
19360 /* However, due to issues with negative LDP/STP offset generation for
19361 larger modes, for DF, DI and vector modes. we must not use negative
19362 addresses smaller than 9 signed unadjusted bits can store. This
19363 provides the most range in this case. */
19366 /* Adjust the base so that it is aligned with the addresses but still
19367 optimal. */
19369 /* Fix the offset, bearing in mind we want to make it bigger not
19370 smaller. */
19373 /* The negative range of LDP/STP is one larger than the positive range. */
19376 /* Check if base offset is too big or too small. We can attempt to resolve
19377 this issue by setting it to the maximum value and seeing if the offsets
19378 still fit. */
19380 {
19382 /* We must still make sure that the base offset is aligned with respect
19383 to the address. But it may may not be made any bigger. */
19385 }
19387 /* Likewise for the case where the base is too small. */
19389 {
19392 }
19394 /* Offset of the first STP/LDP. */
19397 /* Offset of the second STP/LDP. */
19400 /* The offsets must be within the range of the LDP/STP instructions. */
19419 {
19424 }
19426 {
19431 }
19434 {
19443 }
19444 else
19445 {
19454 }
19456 /* Emit adjusting instruction. */
19458 /* Emit ldp/stp instructions. */
19466 }
19468 /* Implement TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE. Assume for now that
19469 it isn't worth branching around empty masked ops (including masked
19470 stores). */
19472 static bool
19474 {
19476 }
19478 /* Return 1 if pseudo register should be created and used to hold
19479 GOT address for PIC code. */
19481 bool
19483 {
19485 }
19487 /* Implement TARGET_UNSPEC_MAY_TRAP_P. */
19489 static int
19491 {
19493 {
19500 }
19503 }
19506 /* If X is a positive CONST_DOUBLE with a value that is a power of 2
19507 return the log2 of that value. Otherwise return -1. */
19509 int
19511 {
19526 }
19528 /* If X is a vector of equal CONST_DOUBLE values and that value is
19529 Y, return the aarch64_fpconst_pow_of_2 of Y. Otherwise return -1. */
19531 int
19533 {
19551 }
19553 /* Implement TARGET_PROMOTED_TYPE to promote 16-bit floating point types
19554 to float.
19556 __fp16 always promotes through this hook.
19557 _Float16 may promote if TARGET_FLT_EVAL_METHOD is 16, but we do that
19558 through the generic excess precision logic rather than here. */
19560 static tree
19562 {
19568 }
19570 /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
19572 static bool
19574 optimization_type opt_type)
19575 {
19577 {
19583 }
19584 }
19586 /* Implement the TARGET_DWARF_POLY_INDETERMINATE_VALUE hook. */
19588 static unsigned int
19591 {
19592 /* Polynomial invariant 1 == (VG / 2) - 1. */
19597 }
19599 /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
19600 if MODE is HFmode, and punt to the generic implementation otherwise. */
19602 static bool
19604 {
19608 }
19610 /* Implement TARGET_SCALAR_MODE_SUPPORTED_P - return TRUE
19611 if MODE is HFmode, and punt to the generic implementation otherwise. */
19613 static bool
19615 {
19619 }
19621 /* Set the value of FLT_EVAL_METHOD.
19622 ISO/IEC TS 18661-3 defines two values that we'd like to make use of:
19624 0: evaluate all operations and constants, whose semantic type has at
19625 most the range and precision of type float, to the range and
19626 precision of float; evaluate all other operations and constants to
19627 the range and precision of the semantic type;
19629 N, where _FloatN is a supported interchange floating type
19630 evaluate all operations and constants, whose semantic type has at
19631 most the range and precision of _FloatN type, to the range and
19632 precision of the _FloatN type; evaluate all other operations and
19633 constants to the range and precision of the semantic type;
19635 If we have the ARMv8.2-A extensions then we support _Float16 in native
19636 precision, so we should set this to 16. Otherwise, we support the type,
19637 but want to evaluate expressions in float precision, so set this to
19638 0. */
19640 static enum flt_eval_method
19642 {
19644 {
19647 /* We can calculate either in 16-bit range and precision or
19648 32-bit range and precision. Make that decision based on whether
19649 we have native support for the ARMv8.2-A 16-bit floating-point
19650 instructions or not. */
19652 ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
19658 }
19660 }
19662 /* Implement TARGET_SCHED_CAN_SPECULATE_INSN. Return true if INSN can be
19663 scheduled for speculative execution. Reject the long-running division
19664 and square-root instructions. */
19666 static bool
19668 {
19670 {
19688 }
19689 }
19691 /* Implement TARGET_COMPUTE_PRESSURE_CLASSES. */
19693 static int
19695 {
19699 /* PR_REGS isn't a useful pressure class because many predicate pseudo
19700 registers need to go in PR_LO_REGS at some point during their
19701 lifetime. Splitting it into two halves has the effect of making
19702 all predicates count against PR_LO_REGS, so that we try whenever
19703 possible to restrict the number of live predicates to 8. This
19704 greatly reduces the amount of spilling in certain loops. */
19708 }
19710 /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
19712 static bool
19715 {
19717 {
19721 /* Don't allow changes between SVE data modes and non-SVE modes.
19722 See the comment at the head of aarch64-sve.md for details. */
19726 /* Don't allow changes in element size: lane 0 of the new vector
19727 would not then be lane 0 of the old vector. See the comment
19728 above aarch64_maybe_expand_sve_subreg_move for a more detailed
19729 description.
19731 In the worst case, this forces a register to be spilled in
19732 one mode and reloaded in the other, which handles the
19733 endianness correctly. */
19736 }
19738 }
19740 /* Implement TARGET_EARLY_REMAT_MODES. */
19742 static void
19744 {
19745 /* SVE values are not normally live across a call, so it should be
19746 worth doing early rematerialization even in VL-specific mode. */
19748 {
19753 }
19754 }
19756 /* Override the default target speculation_safe_value. */
19757 static rtx
19760 {
19761 /* Maybe we should warn if falling back to hard barriers. They are
19762 likely to be noticably more expensive than the alternative below. */
19774 }
19776 /* Implement TARGET_ESTIMATED_POLY_VALUE.
19777 Look into the tuning structure for an estimate.
19778 VAL.coeffs[1] is multiplied by the number of VQ chunks over the initial
19779 Advanced SIMD 128 bits. */
19781 static HOST_WIDE_INT
19783 {
19784 enum aarch64_sve_vector_bits_enum width_source
19787 /* If we still don't have an estimate, use the default. */
19793 }
19796 /* Return true for types that could be supported as SIMD return or
19797 argument types. */
19799 static bool
19801 {
19803 {
19806 }
19808 }
19810 /* Return true for types that currently are supported as SIMD return
19811 or argument types. */
19813 static bool
19815 {
19823 }
19825 /* Implement TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN. */
19827 static int
19831 {
19842 {
19846 }
19851 {
19854 "GCC does not currently support mixed size types "
19858 "GCC does not currently support return type %qT "
19860 else
19863 ret_type);
19865 }
19868 {
19872 {
19875 "GCC does not currently support mixed size types "
19877 else
19879 "GCC does not currently support argument type %qT "
19882 }
19883 }
19889 {
19893 }
19894 else
19895 {
19899 {
19904 }
19905 }
19909 }
19911 /* Implement TARGET_SIMD_CLONE_ADJUST. */
19913 static void
19915 {
19916 /* Add aarch64_vector_pcs target attribute to SIMD clones so they
19917 use the correct ABI. */
19922 }
19924 /* Implement TARGET_SIMD_CLONE_USABLE. */
19926 static int
19928 {
19930 {
19937 }
19938 }
19940 /* Implement TARGET_COMP_TYPE_ATTRIBUTES */
19942 static int
19944 {
19949 }
19951 /* Implement TARGET_GET_MULTILIB_ABI_NAME */
19955 {
19959 }
19961 /* Implement TARGET_STACK_PROTECT_GUARD. In case of a
19962 global variable based guard use the default else
19963 return a null tree. */
19964 static tree
19966 {
19971 }
19973 /* Implement TARGET_ASM_FILE_END for AArch64. This adds the AArch64 GNU NOTE
19974 section at the end if needed. */
19975 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
19976 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
19977 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1)
19978 void
19980 {
19991 {
19992 /* Generate .note.gnu.property section. */
19996 /* PT_NOTE header: namesz, descsz, type.
19997 namesz = 4 ("GNU\0")
19998 descsz = 16 (Size of the program property array)
19999 [(12 + padding) * Number of array elements]
20000 type = 5 (NT_GNU_PROPERTY_TYPE_0). */
20006 /* PT_NOTE name. */
20009 /* PT_NOTE contents for NT_GNU_PROPERTY_TYPE_0:
20010 type = GNU_PROPERTY_AARCH64_FEATURE_1_AND
20011 datasz = 4
20012 data = feature_1_and. */
20017 /* Pad the size of the note to the required alignment. */
20019 }
20020 }
20021 #undef GNU_PROPERTY_AARCH64_FEATURE_1_PAC
20022 #undef GNU_PROPERTY_AARCH64_FEATURE_1_BTI
20023 #undef GNU_PROPERTY_AARCH64_FEATURE_1_AND
20025 /* Target-specific selftests. */
20027 #if CHECKING_P
20031 /* Selftest for the RTL loader.
20032 Verify that the RTL loader copes with a dump from
20033 print_rtx_function. This is essentially just a test that class
20034 function_reader can handle a real dump, but it also verifies
20035 that lookup_reg_by_dump_name correctly handles hard regs.
20036 The presence of hard reg names in the dump means that the test is
20037 target-specific, hence it is in this file. */
20039 static void
20041 {
20053 /* Verify crtl->return_rtx. */
20057 }
20059 /* Run all target-specific selftests. */
20061 static void
20063 {
20065 }
20071 #undef TARGET_STACK_PROTECT_GUARD
20072 #define TARGET_STACK_PROTECT_GUARD aarch64_stack_protect_guard
20074 #undef TARGET_ADDRESS_COST
20075 #define TARGET_ADDRESS_COST aarch64_address_cost
20077 /* This hook will determines whether unnamed bitfields affect the alignment
20078 of the containing structure. The hook returns true if the structure
20079 should inherit the alignment requirements of an unnamed bitfield's
20080 type. */
20081 #undef TARGET_ALIGN_ANON_BITFIELD
20082 #define TARGET_ALIGN_ANON_BITFIELD hook_bool_void_true
20084 #undef TARGET_ASM_ALIGNED_DI_OP
20087 #undef TARGET_ASM_ALIGNED_HI_OP
20090 #undef TARGET_ASM_ALIGNED_SI_OP
20093 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
20094 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK \
20095 hook_bool_const_tree_hwi_hwi_const_tree_true
20097 #undef TARGET_ASM_FILE_START
20098 #define TARGET_ASM_FILE_START aarch64_start_file
20100 #undef TARGET_ASM_OUTPUT_MI_THUNK
20101 #define TARGET_ASM_OUTPUT_MI_THUNK aarch64_output_mi_thunk
20103 #undef TARGET_ASM_SELECT_RTX_SECTION
20104 #define TARGET_ASM_SELECT_RTX_SECTION aarch64_select_rtx_section
20106 #undef TARGET_ASM_TRAMPOLINE_TEMPLATE
20107 #define TARGET_ASM_TRAMPOLINE_TEMPLATE aarch64_asm_trampoline_template
20109 #undef TARGET_BUILD_BUILTIN_VA_LIST
20110 #define TARGET_BUILD_BUILTIN_VA_LIST aarch64_build_builtin_va_list
20112 #undef TARGET_CALLEE_COPIES
20113 #define TARGET_CALLEE_COPIES hook_bool_CUMULATIVE_ARGS_mode_tree_bool_false
20115 #undef TARGET_CAN_ELIMINATE
20116 #define TARGET_CAN_ELIMINATE aarch64_can_eliminate
20118 #undef TARGET_CAN_INLINE_P
20119 #define TARGET_CAN_INLINE_P aarch64_can_inline_p
20121 #undef TARGET_CANNOT_FORCE_CONST_MEM
20122 #define TARGET_CANNOT_FORCE_CONST_MEM aarch64_cannot_force_const_mem
20124 #undef TARGET_CASE_VALUES_THRESHOLD
20125 #define TARGET_CASE_VALUES_THRESHOLD aarch64_case_values_threshold
20127 #undef TARGET_CONDITIONAL_REGISTER_USAGE
20128 #define TARGET_CONDITIONAL_REGISTER_USAGE aarch64_conditional_register_usage
20130 /* Only the least significant bit is used for initialization guard
20131 variables. */
20132 #undef TARGET_CXX_GUARD_MASK_BIT
20133 #define TARGET_CXX_GUARD_MASK_BIT hook_bool_void_true
20135 #undef TARGET_C_MODE_FOR_SUFFIX
20136 #define TARGET_C_MODE_FOR_SUFFIX aarch64_c_mode_for_suffix
20138 #ifdef TARGET_BIG_ENDIAN_DEFAULT
20139 #undef TARGET_DEFAULT_TARGET_FLAGS
20140 #define TARGET_DEFAULT_TARGET_FLAGS (MASK_BIG_END)
20141 #endif
20143 #undef TARGET_CLASS_MAX_NREGS
20144 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
20146 #undef TARGET_BUILTIN_DECL
20147 #define TARGET_BUILTIN_DECL aarch64_builtin_decl
20149 #undef TARGET_BUILTIN_RECIPROCAL
20150 #define TARGET_BUILTIN_RECIPROCAL aarch64_builtin_reciprocal
20152 #undef TARGET_C_EXCESS_PRECISION
20153 #define TARGET_C_EXCESS_PRECISION aarch64_excess_precision
20155 #undef TARGET_EXPAND_BUILTIN
20156 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
20158 #undef TARGET_EXPAND_BUILTIN_VA_START
20159 #define TARGET_EXPAND_BUILTIN_VA_START aarch64_expand_builtin_va_start
20161 #undef TARGET_FOLD_BUILTIN
20162 #define TARGET_FOLD_BUILTIN aarch64_fold_builtin
20164 #undef TARGET_FUNCTION_ARG
20165 #define TARGET_FUNCTION_ARG aarch64_function_arg
20167 #undef TARGET_FUNCTION_ARG_ADVANCE
20168 #define TARGET_FUNCTION_ARG_ADVANCE aarch64_function_arg_advance
20170 #undef TARGET_FUNCTION_ARG_BOUNDARY
20171 #define TARGET_FUNCTION_ARG_BOUNDARY aarch64_function_arg_boundary
20173 #undef TARGET_FUNCTION_ARG_PADDING
20174 #define TARGET_FUNCTION_ARG_PADDING aarch64_function_arg_padding
20176 #undef TARGET_GET_RAW_RESULT_MODE
20177 #define TARGET_GET_RAW_RESULT_MODE aarch64_get_reg_raw_mode
20178 #undef TARGET_GET_RAW_ARG_MODE
20179 #define TARGET_GET_RAW_ARG_MODE aarch64_get_reg_raw_mode
20181 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
20182 #define TARGET_FUNCTION_OK_FOR_SIBCALL aarch64_function_ok_for_sibcall
20184 #undef TARGET_FUNCTION_VALUE
20185 #define TARGET_FUNCTION_VALUE aarch64_function_value
20187 #undef TARGET_FUNCTION_VALUE_REGNO_P
20188 #define TARGET_FUNCTION_VALUE_REGNO_P aarch64_function_value_regno_p
20190 #undef TARGET_GIMPLE_FOLD_BUILTIN
20191 #define TARGET_GIMPLE_FOLD_BUILTIN aarch64_gimple_fold_builtin
20193 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
20194 #define TARGET_GIMPLIFY_VA_ARG_EXPR aarch64_gimplify_va_arg_expr
20196 #undef TARGET_INIT_BUILTINS
20197 #define TARGET_INIT_BUILTINS aarch64_init_builtins
20199 #undef TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS
20200 #define TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS \
20201 aarch64_ira_change_pseudo_allocno_class
20203 #undef TARGET_LEGITIMATE_ADDRESS_P
20204 #define TARGET_LEGITIMATE_ADDRESS_P aarch64_legitimate_address_hook_p
20206 #undef TARGET_LEGITIMATE_CONSTANT_P
20207 #define TARGET_LEGITIMATE_CONSTANT_P aarch64_legitimate_constant_p
20209 #undef TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT
20210 #define TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT \
20211 aarch64_legitimize_address_displacement
20213 #undef TARGET_LIBGCC_CMP_RETURN_MODE
20214 #define TARGET_LIBGCC_CMP_RETURN_MODE aarch64_libgcc_cmp_return_mode
20216 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
20217 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
20218 aarch64_libgcc_floating_mode_supported_p
20220 #undef TARGET_MANGLE_TYPE
20221 #define TARGET_MANGLE_TYPE aarch64_mangle_type
20223 #undef TARGET_MEMORY_MOVE_COST
20224 #define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
20226 #undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
20227 #define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
20229 #undef TARGET_MUST_PASS_IN_STACK
20230 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
20232 /* This target hook should return true if accesses to volatile bitfields
20233 should use the narrowest mode possible. It should return false if these
20234 accesses should use the bitfield container type. */
20235 #undef TARGET_NARROW_VOLATILE_BITFIELD
20236 #define TARGET_NARROW_VOLATILE_BITFIELD hook_bool_void_false
20238 #undef TARGET_OPTION_OVERRIDE
20239 #define TARGET_OPTION_OVERRIDE aarch64_override_options
20241 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
20242 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE \
20243 aarch64_override_options_after_change
20245 #undef TARGET_OPTION_SAVE
20246 #define TARGET_OPTION_SAVE aarch64_option_save
20248 #undef TARGET_OPTION_RESTORE
20249 #define TARGET_OPTION_RESTORE aarch64_option_restore
20251 #undef TARGET_OPTION_PRINT
20252 #define TARGET_OPTION_PRINT aarch64_option_print
20254 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
20255 #define TARGET_OPTION_VALID_ATTRIBUTE_P aarch64_option_valid_attribute_p
20257 #undef TARGET_SET_CURRENT_FUNCTION
20258 #define TARGET_SET_CURRENT_FUNCTION aarch64_set_current_function
20260 #undef TARGET_PASS_BY_REFERENCE
20261 #define TARGET_PASS_BY_REFERENCE aarch64_pass_by_reference
20263 #undef TARGET_PREFERRED_RELOAD_CLASS
20264 #define TARGET_PREFERRED_RELOAD_CLASS aarch64_preferred_reload_class
20266 #undef TARGET_SCHED_REASSOCIATION_WIDTH
20267 #define TARGET_SCHED_REASSOCIATION_WIDTH aarch64_reassociation_width
20269 #undef TARGET_PROMOTED_TYPE
20270 #define TARGET_PROMOTED_TYPE aarch64_promoted_type
20272 #undef TARGET_SECONDARY_RELOAD
20273 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
20275 #undef TARGET_SHIFT_TRUNCATION_MASK
20276 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
20278 #undef TARGET_SETUP_INCOMING_VARARGS
20279 #define TARGET_SETUP_INCOMING_VARARGS aarch64_setup_incoming_varargs
20281 #undef TARGET_STRUCT_VALUE_RTX
20282 #define TARGET_STRUCT_VALUE_RTX aarch64_struct_value_rtx
20284 #undef TARGET_REGISTER_MOVE_COST
20285 #define TARGET_REGISTER_MOVE_COST aarch64_register_move_cost
20287 #undef TARGET_RETURN_IN_MEMORY
20288 #define TARGET_RETURN_IN_MEMORY aarch64_return_in_memory
20290 #undef TARGET_RETURN_IN_MSB
20291 #define TARGET_RETURN_IN_MSB aarch64_return_in_msb
20293 #undef TARGET_RTX_COSTS
20294 #define TARGET_RTX_COSTS aarch64_rtx_costs_wrapper
20296 #undef TARGET_SCALAR_MODE_SUPPORTED_P
20297 #define TARGET_SCALAR_MODE_SUPPORTED_P aarch64_scalar_mode_supported_p
20299 #undef TARGET_SCHED_ISSUE_RATE
20300 #define TARGET_SCHED_ISSUE_RATE aarch64_sched_issue_rate
20302 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
20303 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
20304 aarch64_sched_first_cycle_multipass_dfa_lookahead
20306 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
20307 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD \
20308 aarch64_first_cycle_multipass_dfa_lookahead_guard
20310 #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
20311 #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS \
20312 aarch64_get_separate_components
20314 #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
20315 #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB \
20316 aarch64_components_for_bb
20318 #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
20319 #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS \
20320 aarch64_disqualify_components
20322 #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
20323 #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
20324 aarch64_emit_prologue_components
20326 #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
20327 #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
20328 aarch64_emit_epilogue_components
20330 #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
20331 #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS \
20332 aarch64_set_handled_components
20334 #undef TARGET_TRAMPOLINE_INIT
20335 #define TARGET_TRAMPOLINE_INIT aarch64_trampoline_init
20337 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
20338 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P aarch64_use_blocks_for_constant_p
20340 #undef TARGET_VECTOR_MODE_SUPPORTED_P
20341 #define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
20343 #undef TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT
20344 #define TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT \
20345 aarch64_builtin_support_vector_misalignment
20347 #undef TARGET_ARRAY_MODE
20348 #define TARGET_ARRAY_MODE aarch64_array_mode
20350 #undef TARGET_ARRAY_MODE_SUPPORTED_P
20351 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
20353 #undef TARGET_VECTORIZE_ADD_STMT_COST
20354 #define TARGET_VECTORIZE_ADD_STMT_COST aarch64_add_stmt_cost
20356 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
20357 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
20358 aarch64_builtin_vectorization_cost
20360 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
20361 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
20363 #undef TARGET_VECTORIZE_BUILTINS
20364 #define TARGET_VECTORIZE_BUILTINS
20366 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
20367 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
20368 aarch64_builtin_vectorized_function
20370 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
20371 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
20372 aarch64_autovectorize_vector_sizes
20374 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
20375 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV \
20376 aarch64_atomic_assign_expand_fenv
20378 /* Section anchor support. */
20380 #undef TARGET_MIN_ANCHOR_OFFSET
20381 #define TARGET_MIN_ANCHOR_OFFSET -256
20383 /* Limit the maximum anchor offset to 4k-1, since that's the limit for a
20384 byte offset; we can do much more for larger data types, but have no way
20385 to determine the size of the access. We assume accesses are aligned. */
20386 #undef TARGET_MAX_ANCHOR_OFFSET
20387 #define TARGET_MAX_ANCHOR_OFFSET 4095
20389 #undef TARGET_VECTOR_ALIGNMENT
20390 #define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
20392 #undef TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
20393 #define TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT \
20394 aarch64_vectorize_preferred_vector_alignment
20395 #undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
20396 #define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
20397 aarch64_simd_vector_alignment_reachable
20399 /* vec_perm support. */
20401 #undef TARGET_VECTORIZE_VEC_PERM_CONST
20402 #define TARGET_VECTORIZE_VEC_PERM_CONST \
20403 aarch64_vectorize_vec_perm_const
20405 #undef TARGET_VECTORIZE_GET_MASK_MODE
20406 #define TARGET_VECTORIZE_GET_MASK_MODE aarch64_get_mask_mode
20407 #undef TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE
20408 #define TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE \
20409 aarch64_empty_mask_is_expensive
20410 #undef TARGET_PREFERRED_ELSE_VALUE
20411 #define TARGET_PREFERRED_ELSE_VALUE \
20412 aarch64_preferred_else_value
20414 #undef TARGET_INIT_LIBFUNCS
20415 #define TARGET_INIT_LIBFUNCS aarch64_init_libfuncs
20417 #undef TARGET_FIXED_CONDITION_CODE_REGS
20418 #define TARGET_FIXED_CONDITION_CODE_REGS aarch64_fixed_condition_code_regs
20420 #undef TARGET_FLAGS_REGNUM
20421 #define TARGET_FLAGS_REGNUM CC_REGNUM
20423 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
20424 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
20426 #undef TARGET_ASAN_SHADOW_OFFSET
20427 #define TARGET_ASAN_SHADOW_OFFSET aarch64_asan_shadow_offset
20429 #undef TARGET_LEGITIMIZE_ADDRESS
20430 #define TARGET_LEGITIMIZE_ADDRESS aarch64_legitimize_address
20432 #undef TARGET_SCHED_CAN_SPECULATE_INSN
20433 #define TARGET_SCHED_CAN_SPECULATE_INSN aarch64_sched_can_speculate_insn
20435 #undef TARGET_CAN_USE_DOLOOP_P
20436 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
20438 #undef TARGET_SCHED_ADJUST_PRIORITY
20439 #define TARGET_SCHED_ADJUST_PRIORITY aarch64_sched_adjust_priority
20441 #undef TARGET_SCHED_MACRO_FUSION_P
20442 #define TARGET_SCHED_MACRO_FUSION_P aarch64_macro_fusion_p
20444 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
20445 #define TARGET_SCHED_MACRO_FUSION_PAIR_P aarch_macro_fusion_pair_p
20447 #undef TARGET_SCHED_FUSION_PRIORITY
20448 #define TARGET_SCHED_FUSION_PRIORITY aarch64_sched_fusion_priority
20450 #undef TARGET_UNSPEC_MAY_TRAP_P
20451 #define TARGET_UNSPEC_MAY_TRAP_P aarch64_unspec_may_trap_p
20453 #undef TARGET_USE_PSEUDO_PIC_REG
20454 #define TARGET_USE_PSEUDO_PIC_REG aarch64_use_pseudo_pic_reg
20456 #undef TARGET_PRINT_OPERAND
20457 #define TARGET_PRINT_OPERAND aarch64_print_operand
20459 #undef TARGET_PRINT_OPERAND_ADDRESS
20460 #define TARGET_PRINT_OPERAND_ADDRESS aarch64_print_operand_address
20462 #undef TARGET_OPTAB_SUPPORTED_P
20463 #define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p
20465 #undef TARGET_OMIT_STRUCT_RETURN_REG
20466 #define TARGET_OMIT_STRUCT_RETURN_REG true
20468 #undef TARGET_DWARF_POLY_INDETERMINATE_VALUE
20469 #define TARGET_DWARF_POLY_INDETERMINATE_VALUE \
20470 aarch64_dwarf_poly_indeterminate_value
20472 /* The architecture reserves bits 0 and 1 so use bit 2 for descriptors. */
20473 #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
20474 #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 4
20476 #undef TARGET_HARD_REGNO_NREGS
20477 #define TARGET_HARD_REGNO_NREGS aarch64_hard_regno_nregs
20478 #undef TARGET_HARD_REGNO_MODE_OK
20479 #define TARGET_HARD_REGNO_MODE_OK aarch64_hard_regno_mode_ok
20481 #undef TARGET_MODES_TIEABLE_P
20482 #define TARGET_MODES_TIEABLE_P aarch64_modes_tieable_p
20484 #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
20485 #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
20486 aarch64_hard_regno_call_part_clobbered
20488 #undef TARGET_REMOVE_EXTRA_CALL_PRESERVED_REGS
20489 #define TARGET_REMOVE_EXTRA_CALL_PRESERVED_REGS \
20490 aarch64_remove_extra_call_preserved_regs
20492 #undef TARGET_RETURN_CALL_WITH_MAX_CLOBBERS
20493 #define TARGET_RETURN_CALL_WITH_MAX_CLOBBERS \
20494 aarch64_return_call_with_max_clobbers
20496 #undef TARGET_CONSTANT_ALIGNMENT
20497 #define TARGET_CONSTANT_ALIGNMENT aarch64_constant_alignment
20499 #undef TARGET_STACK_CLASH_PROTECTION_ALLOCA_PROBE_RANGE
20500 #define TARGET_STACK_CLASH_PROTECTION_ALLOCA_PROBE_RANGE \
20501 aarch64_stack_clash_protection_alloca_probe_range
20503 #undef TARGET_COMPUTE_PRESSURE_CLASSES
20504 #define TARGET_COMPUTE_PRESSURE_CLASSES aarch64_compute_pressure_classes
20506 #undef TARGET_CAN_CHANGE_MODE_CLASS
20507 #define TARGET_CAN_CHANGE_MODE_CLASS aarch64_can_change_mode_class
20509 #undef TARGET_SELECT_EARLY_REMAT_MODES
20510 #define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
20512 #undef TARGET_SPECULATION_SAFE_VALUE
20513 #define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value
20515 #undef TARGET_ESTIMATED_POLY_VALUE
20516 #define TARGET_ESTIMATED_POLY_VALUE aarch64_estimated_poly_value
20518 #undef TARGET_ATTRIBUTE_TABLE
20519 #define TARGET_ATTRIBUTE_TABLE aarch64_attribute_table
20521 #undef TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
20522 #define TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN \
20523 aarch64_simd_clone_compute_vecsize_and_simdlen
20525 #undef TARGET_SIMD_CLONE_ADJUST
20526 #define TARGET_SIMD_CLONE_ADJUST aarch64_simd_clone_adjust
20528 #undef TARGET_SIMD_CLONE_USABLE
20529 #define TARGET_SIMD_CLONE_USABLE aarch64_simd_clone_usable
20531 #undef TARGET_COMP_TYPE_ATTRIBUTES
20532 #define TARGET_COMP_TYPE_ATTRIBUTES aarch64_comp_type_attributes
20534 #undef TARGET_GET_MULTILIB_ABI_NAME
20535 #define TARGET_GET_MULTILIB_ABI_NAME aarch64_get_multilib_abi_name
20537 #if CHECKING_P
20538 #undef TARGET_RUN_TARGET_SELFTESTS
20539 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests
20542 #undef TARGET_ASM_POST_CFI_STARTPROC
20543 #define TARGET_ASM_POST_CFI_STARTPROC aarch64_post_cfi_startproc