| blob | 9c68025e1f19236305ab41b0f7ed0cbfa039d3e1 |
1 /* Machine description for AArch64 architecture.
2 Copyright (C) 2009-2018 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
24 #define INCLUDE_STRING
75 /* This file should be included last. */
78 /* Defined for convenience. */
79 #define POINTER_BYTES (POINTER_SIZE / BITS_PER_UNIT)
81 /* Information about a legitimate vector immediate operand. */
82 struct simd_immediate_info
83 {
94 /* The mode of the elements. */
95 scalar_mode elt_mode;
97 /* The value of each element if all elements are the same, or the
98 first value if the constant is a series. */
99 rtx value;
101 /* The value of the step if the constant is a series, null otherwise. */
102 rtx step;
104 /* The instruction to use to move the immediate into a vector. */
105 insn_type insn;
107 /* The kind of shift modifier to use, and the number of bits to shift.
108 This is (LSL, 0) if no shift is needed. */
109 modifier_type modifier;
111 };
113 /* Construct a floating-point immediate in which each element has mode
114 ELT_MODE_IN and value VALUE_IN. */
115 inline simd_immediate_info
119 {}
121 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
122 and value VALUE_IN. The other parameters are as for the structure
123 fields. */
124 inline simd_immediate_info
131 {}
133 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
134 and where element I is equal to VALUE_IN + I * STEP_IN. */
135 inline simd_immediate_info
139 {}
141 /* The current code model. */
144 /* The number of 64-bit elements in an SVE vector. */
145 poly_uint16 aarch64_sve_vg;
147 #ifdef HAVE_AS_TLS
148 #undef TARGET_HAVE_TLS
149 #define TARGET_HAVE_TLS 1
150 #endif
154 const_tree,
163 const_tree type,
168 aarch64_addr_query_type);
170 /* Major revision number of the ARM Architecture implemented by the target. */
173 /* The processor for which instructions should be scheduled. */
176 /* Mask to specify which instruction scheduling options should be used. */
179 /* Global flag for PC relative loads. */
182 /* Global flag for whether frame pointer is enabled. */
185 /* Support for command line parsing of boolean flags in the tuning
186 structures. */
187 struct aarch64_flag_desc
188 {
191 };
193 #define AARCH64_FUSION_PAIR(name, internal_name) \
194 { name, AARCH64_FUSE_##internal_name },
196 {
201 };
203 #define AARCH64_EXTRA_TUNING_OPTION(name, internal_name) \
204 { name, AARCH64_EXTRA_TUNE_##internal_name },
206 {
211 };
213 /* Tuning parameters. */
216 {
217 {
222 },
229 };
232 {
233 {
238 },
245 };
248 {
249 {
254 },
261 };
264 {
265 {
270 },
277 };
280 {
281 {
286 },
293 };
296 {
297 {
302 },
309 };
312 {
314 /* Avoid the use of slow int<->fp moves for spilling by setting
315 their cost higher than memmov_cost. */
319 };
322 {
324 /* Avoid the use of slow int<->fp moves for spilling by setting
325 their cost higher than memmov_cost. */
329 };
332 {
334 /* Avoid the use of slow int<->fp moves for spilling by setting
335 their cost higher than memmov_cost. */
339 };
342 {
344 /* Avoid the use of slow int<->fp moves for spilling by setting
345 their cost higher than memmov_cost (actual, 4 and 9). */
349 };
352 {
357 };
360 {
362 /* Avoid the use of slow int<->fp moves for spilling by setting
363 their cost higher than memmov_cost. */
367 };
370 {
372 /* Avoid the use of int<->fp moves for spilling. */
376 };
379 {
381 /* Avoid the use of int<->fp moves for spilling. */
385 };
388 {
390 /* Avoid the use of slow int<->fp moves for spilling by setting
391 their cost higher than memmov_cost. */
395 };
397 /* Generic costs for vector insn classes. */
399 {
415 };
417 /* QDF24XX costs for vector insn classes. */
419 {
435 };
437 /* ThunderX costs for vector insn classes. */
439 {
455 };
458 {
474 };
476 /* Generic costs for vector insn classes. */
478 {
494 };
497 {
513 };
515 /* Generic costs for vector insn classes. */
517 {
533 };
535 /* Costs for vector insn classes for Vulcan. */
537 {
553 };
555 /* Generic costs for branch instructions. */
557 {
560 };
562 /* Generic approximation modes. */
564 {
567 AARCH64_APPROX_NONE /* recip_sqrt */
568 };
570 /* Approximation modes for Exynos M1. */
572 {
575 AARCH64_APPROX_ALL /* recip_sqrt */
576 };
578 /* Approximation modes for X-Gene 1. */
580 {
583 AARCH64_APPROX_ALL /* recip_sqrt */
584 };
586 /* Generic prefetch settings (which disable prefetch). */
588 {
596 };
599 {
607 };
610 {
618 };
621 {
629 };
632 {
640 };
643 {
651 };
654 {
662 };
665 {
686 &generic_prefetch_tune
687 };
690 {
712 &generic_prefetch_tune
713 };
716 {
738 &generic_prefetch_tune
739 };
742 {
764 &generic_prefetch_tune
765 };
768 {
790 &generic_prefetch_tune
791 };
794 {
816 &generic_prefetch_tune
817 };
822 {
843 &exynosm1_prefetch_tune
844 };
847 {
868 &thunderxt88_prefetch_tune
869 };
872 {
892 (AARCH64_EXTRA_TUNE_SLOW_UNALIGNED_LDPW
894 &thunderx_prefetch_tune
895 };
898 {
920 &tsv110_prefetch_tune
921 };
924 {
945 &generic_prefetch_tune
946 };
949 {
971 &qdf24xx_prefetch_tune
972 };
974 /* Tuning structure for the Qualcomm Saphira core. Default to falkor values
975 for now. */
977 {
999 &generic_prefetch_tune
1000 };
1003 {
1025 &thunderx2t99_prefetch_tune
1026 };
1028 /* Support for fine-grained override of the tuning structures. */
1029 struct aarch64_tuning_override_function
1030 {
1033 };
1038 static const struct aarch64_tuning_override_function
1039 aarch64_tuning_override_functions[] =
1040 {
1044 };
1046 /* A processor implementing AArch64. */
1047 struct processor
1048 {
1056 };
1058 /* Architectures implementing AArch64. */
1060 {
1061 #define AARCH64_ARCH(NAME, CORE, ARCH_IDENT, ARCH_REV, FLAGS) \
1062 {NAME, CORE, CORE, AARCH64_ARCH_##ARCH_IDENT, ARCH_REV, FLAGS, NULL},
1065 };
1067 /* Processor cores implementing AArch64. */
1069 {
1070 #define AARCH64_CORE(NAME, IDENT, SCHED, ARCH, FLAGS, COSTS, IMP, PART, VARIANT) \
1071 {NAME, IDENT, SCHED, AARCH64_ARCH_##ARCH, \
1072 all_architectures[AARCH64_ARCH_##ARCH].architecture_version, \
1073 FLAGS, &COSTS##_tunings},
1078 };
1081 /* Target specification. These are populated by the -march, -mtune, -mcpu
1082 handling code or by target attributes. */
1087 /* The current tuning set. */
1090 #define AARCH64_CPU_DEFAULT_FLAGS ((selected_cpu) ? selected_cpu->flags : 0)
1092 /* An ISA extension in the co-processor and main instruction set space. */
1093 struct aarch64_option_extension
1094 {
1098 };
1101 {
1105 }
1106 aarch64_cc;
1108 #define AARCH64_INVERSE_CONDITION_CODE(X) ((aarch64_cc) (((int) X) ^ 1))
1110 /* The condition codes of the processor, and the inverse function. */
1112 {
1115 };
1117 /* Generate code to enable conditional branches in functions over 1 MiB. */
1121 {
1138 }
1140 void
1142 {
1147 else
1150 else
1154 else
1157 }
1159 /* Implement TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS.
1160 The register allocator chooses POINTER_AND_FP_REGS if FP_REGS and
1161 GENERAL_REGS have the same cost - even if POINTER_AND_FP_REGS has a much
1162 higher cost. POINTER_AND_FP_REGS is also used if the cost of both FP_REGS
1163 and GENERAL_REGS is lower than the memory cost (in this case the best class
1164 is the lowest cost one). Using POINTER_AND_FP_REGS irrespectively of its
1165 cost results in bad allocations with many redundant int<->FP moves which
1166 are expensive on various cores.
1167 To avoid this we don't allow POINTER_AND_FP_REGS as the allocno class, but
1168 force a decision between FP_REGS and GENERAL_REGS. We use the allocno class
1169 if it isn't POINTER_AND_FP_REGS. Similarly, use the best class if it isn't
1170 POINTER_AND_FP_REGS. Otherwise set the allocno class depending on the mode.
1171 The result of this is that it is no longer inefficient to have a higher
1172 memory move cost than the register move cost.
1173 */
1175 static reg_class_t
1177 reg_class_t best_class)
1178 {
1179 machine_mode mode;
1191 }
1193 static unsigned int
1195 {
1199 }
1201 /* Return the reassociation width of treeop OPC with mode MODE. */
1202 static int
1204 {
1209 /* Avoid reassociating floating point addition so we emit more FMAs. */
1213 }
1215 /* Provide a mapping from gcc register numbers to dwarf register numbers. */
1216 unsigned
1218 {
1230 /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
1231 equivalent DWARF register. */
1233 }
1235 /* Return true if MODE is any of the Advanced SIMD structure modes. */
1236 static bool
1238 {
1241 }
1243 /* Return true if MODE is an SVE predicate mode. */
1244 static bool
1246 {
1252 }
1254 /* Three mutually-exclusive flags describing a vector or predicate type. */
1258 /* Can be used in combination with VEC_ADVSIMD or VEC_SVE_DATA to indicate
1259 a structure of 2, 3 or 4 vectors. */
1261 /* Useful combinations of the above. */
1265 /* Return a set of flags describing the vector properties of mode MODE.
1266 Ignore modes that are not supported by the current target. */
1267 static unsigned int
1269 {
1285 {
1287 {
1294 }
1296 /* This includes V1DF but not V1DI (which doesn't exist). */
1301 }
1304 }
1306 /* Return true if MODE is any of the data vector modes, including
1307 structure modes. */
1308 static bool
1310 {
1312 }
1314 /* Return true if MODE is an SVE data vector mode; either a single vector
1315 or a structure of vectors. */
1316 static bool
1318 {
1320 }
1322 /* Implement target hook TARGET_ARRAY_MODE. */
1323 static opt_machine_mode
1325 {
1332 }
1334 /* Implement target hook TARGET_ARRAY_MODE_SUPPORTED_P. */
1335 static bool
1338 {
1346 }
1348 /* Return the SVE predicate mode to use for elements that have
1349 ELEM_NBYTES bytes, if such a mode exists. */
1351 opt_machine_mode
1353 {
1355 {
1364 }
1366 }
1368 /* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
1370 static opt_machine_mode
1372 {
1374 {
1376 machine_mode pred_mode;
1379 }
1382 }
1384 /* Implement TARGET_PREFERRED_ELSE_VALUE. For binary operations,
1385 prefer to use the first arithmetic operand as the else value if
1386 the else value doesn't matter, since that exactly matches the SVE
1387 destructive merging form. For ternary operations we could either
1388 pick the first operand and use FMAD-like instructions or the last
1389 operand and use FMLA-like instructions; the latter seems more
1390 natural. */
1392 static tree
1394 {
1396 }
1398 /* Implement TARGET_HARD_REGNO_NREGS. */
1400 static unsigned int
1402 {
1403 /* ??? Logically we should only need to provide a value when
1404 HARD_REGNO_MODE_OK says that the combination is valid,
1405 but at the moment we need to handle all modes. Just ignore
1406 any runtime parts for registers that can't store them. */
1409 {
1422 }
1424 }
1426 /* Implement TARGET_HARD_REGNO_MODE_OK. */
1428 static bool
1430 {
1435 /* This must have the same size as _Unwind_Word. */
1446 /* The purpose of comparing with ptr_mode is to support the
1447 global register variable associated with the stack pointer
1448 register via the syntax of asm ("wsp") in ILP32. */
1458 {
1461 else
1463 }
1466 }
1468 /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The callee only saves
1469 the lower 64 bits of a 128-bit register. Tell the compiler the callee
1470 clobbers the top 64 bits when restoring the bottom 64 bits. */
1472 static bool
1474 {
1476 }
1478 /* Implement REGMODE_NATURAL_SIZE. */
1479 poly_uint64
1481 {
1482 /* The natural size for SVE data modes is one SVE data vector,
1483 and similarly for predicates. We can't independently modify
1484 anything smaller than that. */
1485 /* ??? For now, only do this for variable-width SVE registers.
1486 Doing it for constant-sized registers breaks lower-subreg.c. */
1487 /* ??? And once that's fixed, we should probably have similar
1488 code for Advanced SIMD. */
1490 {
1496 }
1498 }
1500 /* Implement HARD_REGNO_CALLER_SAVE_MODE. */
1501 machine_mode
1503 machine_mode mode)
1504 {
1505 /* The predicate mode determines which bits are significant and
1506 which are "don't care". Decreasing the number of lanes would
1507 lose data while increasing the number of lanes would make bits
1508 unnecessarily significant. */
1513 else
1515 }
1517 /* Return true if I's bits are consecutive ones from the MSB. */
1518 bool
1520 {
1522 }
1524 /* Implement TARGET_CONSTANT_ALIGNMENT. Make strings word-aligned so
1525 that strcpy from constants will be faster. */
1527 static HOST_WIDE_INT
1529 {
1533 }
1535 /* Return true if calls to DECL should be treated as
1536 long-calls (ie called via a register). */
1537 static bool
1539 {
1541 }
1543 /* Return true if calls to symbol-ref SYM should be treated as
1544 long-calls (ie called via a register). */
1545 bool
1547 {
1549 }
1551 /* Return true if calls to symbol-ref SYM should not go through
1552 plt stubs. */
1554 bool
1556 {
1560 && decl
1561 && (!flag_plt
1567 }
1569 /* Return true if the offsets to a zero/sign-extract operation
1570 represent an expression that matches an extend operation. The
1571 operands represent the paramters from
1573 (extract:MODE (mult (reg) (MULT_IMM)) (EXTRACT_IMM) (const_int 0)). */
1574 bool
1576 rtx extract_imm)
1577 {
1594 }
1596 /* Emit an insn that's a simple single-set. Both the operands must be
1597 known to be valid. */
1600 {
1602 }
1604 /* X and Y are two things to compare using CODE. Emit the compare insn and
1605 return the rtx for register 0 in the proper mode. */
1606 rtx
1608 {
1614 }
1616 /* Build the SYMBOL_REF for __tls_get_addr. */
1620 rtx
1622 {
1626 }
1628 /* Return the TLS model to use for ADDR. */
1630 static enum tls_model
1632 {
1635 {
1636 poly_int64 addend;
1640 }
1645 }
1647 /* We'll allow lo_sum's in addresses in our legitimate addresses
1648 so that combine would take care of combining addresses where
1649 necessary, but for generation purposes, we'll generate the address
1650 as :
1651 RTL Absolute
1652 tmp = hi (symbol_ref); adrp x1, foo
1653 dest = lo_sum (tmp, symbol_ref); add dest, x1, :lo_12:foo
1654 nop
1656 PIC TLS
1657 adrp x1, :got:foo adrp tmp, :tlsgd:foo
1658 ldr x1, [:got_lo12:foo] add dest, tmp, :tlsgd_lo12:foo
1659 bl __tls_get_addr
1660 nop
1662 Load TLS symbol, depending on TLS mechanism and TLS access model.
1664 Global Dynamic - Traditional TLS:
1665 adrp tmp, :tlsgd:imm
1666 add dest, tmp, #:tlsgd_lo12:imm
1667 bl __tls_get_addr
1669 Global Dynamic - TLS Descriptors:
1670 adrp dest, :tlsdesc:imm
1671 ldr tmp, [dest, #:tlsdesc_lo12:imm]
1672 add dest, dest, #:tlsdesc_lo12:imm
1673 blr tmp
1674 mrs tp, tpidr_el0
1675 add dest, dest, tp
1677 Initial Exec:
1678 mrs tp, tpidr_el0
1679 adrp tmp, :gottprel:imm
1680 ldr dest, [tmp, #:gottprel_lo12:imm]
1681 add dest, dest, tp
1683 Local Exec:
1684 mrs tp, tpidr_el0
1685 add t0, tp, #:tprel_hi12:imm, lsl #12
1686 add t0, t0, #:tprel_lo12_nc:imm
1687 */
1689 static void
1692 {
1694 {
1696 {
1697 /* In ILP32, the mode of dest can be either SImode or DImode. */
1709 }
1716 {
1719 rtx insn;
1720 rtx mem;
1722 /* NOTE: pic_offset_table_rtx can be NULL_RTX, because we can reach
1723 here before rtl expand. Tree IVOPT will generate rtl pattern to
1724 decide rtx costs, in which case pic_offset_table_rtx is not
1725 initialized. For that case no need to generate the first adrp
1726 instruction as the final cost for global variable access is
1727 one instruction. */
1729 {
1730 /* -fpic for -mcmodel=small allow 32K GOT table size (but we are
1731 using the page base as GOT base, the first page may be wasted,
1732 in the worst scenario, there is only 28K space for GOT).
1734 The generate instruction sequence for accessing global variable
1735 is:
1737 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym]
1739 Only one instruction needed. But we must initialize
1740 pic_offset_table_rtx properly. We generate initialize insn for
1741 every global access, and allow CSE to remove all redundant.
1743 The final instruction sequences will look like the following
1744 for multiply global variables access.
1746 adrp pic_offset_table_rtx, _GLOBAL_OFFSET_TABLE_
1748 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym1]
1749 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym2]
1750 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym3]
1751 ... */
1760 }
1763 {
1766 else
1770 }
1771 else
1772 {
1777 }
1779 /* The operand is expected to be MEM. Whenever the related insn
1780 pattern changed, above code which calculate mem should be
1781 updated. */
1787 }
1790 {
1791 /* In ILP32, the mode of dest can be either SImode or DImode,
1792 while the got entry is always of SImode size. The mode of
1793 dest depends on how dest is used: if dest is assigned to a
1794 pointer (e.g. in the memory), it has SImode; it may have
1795 DImode if dest is dereferenced to access the memeory.
1796 This is why we have to handle three different ldr_got_small
1797 patterns here (two patterns for ILP32). */
1799 rtx insn;
1800 rtx mem;
1809 {
1812 else
1816 }
1817 else
1818 {
1823 }
1830 }
1833 {
1841 else
1849 }
1852 {
1855 rtx tp;
1859 /* In ILP32, the got entry is always of SImode size. Unlike
1860 small GOT, the dest is fixed at reg 0. */
1863 else
1874 }
1877 {
1878 /* In ILP32, the mode of dest can be either SImode or DImode,
1879 while the got entry is always of SImode size. The mode of
1880 dest depends on how dest is used: if dest is assigned to a
1881 pointer (e.g. in the memory), it has SImode; it may have
1882 DImode if dest is dereferenced to access the memeory.
1883 This is why we have to handle three different tlsie_small
1884 patterns here (two patterns for ILP32). */
1890 {
1893 else
1894 {
1897 }
1898 }
1899 else
1900 {
1903 }
1909 }
1915 {
1923 {
1946 }
1951 }
1958 {
1963 {
1966 else
1967 {
1970 }
1971 }
1972 else
1973 {
1976 }
1981 }
1985 }
1986 }
1988 /* Emit a move from SRC to DEST. Assume that the move expanders can
1989 handle all moves if !can_create_pseudo_p (). The distinction is
1990 important because, unlike emit_move_insn, the move expanders know
1991 how to force Pmode objects into the constant pool even when the
1992 constant pool address is not itself legitimate. */
1993 static rtx
1995 {
1999 }
2001 /* Apply UNOPTAB to OP and store the result in DEST. */
2003 static void
2005 {
2009 }
2011 /* Apply BINOPTAB to OP0 and OP1 and store the result in DEST. */
2013 static void
2015 {
2017 OPTAB_DIRECT);
2020 }
2022 /* Split a 128-bit move operation into two 64-bit move operations,
2023 taking care to handle partial overlap of register to register
2024 copies. Special cases are needed when moving between GP regs and
2025 FP regs. SRC can be a register, constant or memory; DST a register
2026 or memory. If either operand is memory it must not have any side
2027 effects. */
2028 void
2030 {
2041 {
2045 /* Handle FP <-> GP regs. */
2047 {
2054 }
2056 {
2063 }
2064 }
2071 /* At most one pairing may overlap. */
2073 {
2076 }
2077 else
2078 {
2081 }
2082 }
2084 bool
2086 {
2089 }
2091 /* Split a complex SIMD combine. */
2093 void
2095 {
2106 }
2108 /* Split a complex SIMD move. */
2110 void
2112 {
2119 {
2122 }
2123 }
2125 bool
2128 {
2132 }
2135 static rtx
2137 {
2140 else
2141 {
2145 }
2146 }
2148 /* Return true if we can move VALUE into a register using a single
2149 CNT[BHWD] instruction. */
2151 static bool
2153 {
2155 /* The coefficient must be [1, 16] * {2, 4, 8, 16}. */
2160 }
2162 /* Likewise for rtx X. */
2164 bool
2166 {
2167 poly_int64 value;
2169 }
2171 /* Return the asm string for an instruction with a CNT-like vector size
2172 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2173 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2174 first part of the operands template (the part that comes before the
2175 vector size itself). FACTOR is the number of quadwords.
2176 NELTS_PER_VQ, if nonzero, is the number of elements in each quadword.
2177 If it is zero, we can use any element size. */
2183 {
2187 /* There is some overlap in the ranges of the four CNT instructions.
2188 Here we always use the smallest possible element size, so that the
2189 multiplier is 1 whereever possible. */
2200 else
2205 }
2207 /* Return the asm string for an instruction with a CNT-like vector size
2208 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2209 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2210 first part of the operands template (the part that comes before the
2211 vector size itself). X is the value of the vector size operand,
2212 as a polynomial integer rtx. */
2216 rtx x)
2217 {
2222 }
2224 /* Return true if we can add VALUE to a register using a single ADDVL
2225 or ADDPL instruction. */
2227 static bool
2229 {
2233 /* FACTOR counts VG / 2, so a value of 2 is one predicate width
2234 and a value of 16 is one vector width. */
2237 }
2239 /* Likewise for rtx X. */
2241 bool
2243 {
2244 poly_int64 value;
2247 }
2249 /* Return the asm string for adding ADDVL or ADDPL immediate X to operand 1
2250 and storing the result in operand 0. */
2254 {
2259 /* Use INC or DEC if possible. */
2261 {
2268 }
2273 else
2276 }
2278 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2279 instruction. If it is, store the number of elements in each vector
2280 quadword in *NELTS_PER_VQ_OUT (if nonnull) and store the multiplication
2281 factor in *FACTOR_OUT (if nonnull). */
2283 bool
2286 {
2287 rtx elt;
2288 poly_int64 value;
2296 /* There's no vector INCB. */
2303 /* The coefficient must be [1, 16] * NELTS_PER_VQ. */
2313 }
2315 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2316 instruction. */
2318 bool
2320 {
2322 }
2324 /* Return the asm template for an SVE vector INC or DEC instruction.
2325 OPERANDS gives the operands before the vector count and X is the
2326 value of the vector count operand itself. */
2330 {
2337 nelts_per_vq);
2338 else
2340 nelts_per_vq);
2341 }
2343 static int
2345 scalar_int_mode mode)
2346 {
2355 {
2359 }
2361 /* Check to see if the low 32 bits are either 0xffffXXXX or 0xXXXXffff
2362 (with XXXX non-zero). In that case check to see if the move can be done in
2363 a smaller mode. */
2368 {
2372 /* Check if we have to emit a second instruction by checking to see
2373 if any of the upper 32 bits of the original DI mode value is set. */
2384 }
2387 {
2389 {
2394 else
2397 }
2399 }
2401 /* Remaining cases are all for DImode. */
2410 {
2411 /* Try emitting a bitmask immediate with a movk replacing 16 bits.
2412 For a 64-bit bitmask try whether changing 16 bits to all ones or
2413 zeroes creates a valid bitmask. To check any repeated bitmask,
2414 try using 16 bits from the other 32-bit half of val. */
2417 {
2428 }
2430 {
2432 {
2436 }
2438 }
2439 }
2441 /* Generate 2-4 instructions, skipping 16 bits of all zeroes or ones which
2442 are emitted by the initial mov. If one_match > zero_match, skip set bits,
2443 otherwise skip zero bits. */
2455 {
2461 num_insns ++;
2462 }
2465 }
2467 /* Return whether imm is a 128-bit immediate which is simple enough to
2468 expand inline. */
2469 bool
2471 {
2482 }
2485 /* Return the number of temporary registers that aarch64_add_offset_1
2486 would need to add OFFSET to a register. */
2488 static unsigned int
2490 {
2492 }
2494 /* A subroutine of aarch64_add_offset. Set DEST to SRC + OFFSET for
2495 a non-polynomial OFFSET. MODE is the mode of the addition.
2496 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
2497 be set and CFA adjustments added to the generated instructions.
2499 TEMP1, if nonnull, is a register of mode MODE that can be used as a
2500 temporary if register allocation is already complete. This temporary
2501 register may overlap DEST but must not overlap SRC. If TEMP1 is known
2502 to hold abs (OFFSET), EMIT_MOVE_IMM can be set to false to avoid emitting
2503 the immediate again.
2505 Since this function may be used to adjust the stack pointer, we must
2506 ensure that it cannot cause transient stack deallocation (for example
2507 by first incrementing SP and then decrementing when adjusting by a
2508 large immediate). */
2510 static void
2514 {
2522 {
2524 {
2527 }
2529 }
2531 /* Single instruction adjustment. */
2533 {
2537 }
2539 /* Emit 2 additions/subtractions if the adjustment is less than 24 bits
2540 and either:
2542 a) the offset cannot be loaded by a 16-bit move or
2543 b) there is no spare register into which we can move it. */
2547 {
2556 }
2558 /* Emit a move immediate if required and an addition/subtraction. */
2560 {
2563 }
2568 {
2572 }
2573 }
2575 /* Return the number of temporary registers that aarch64_add_offset
2576 would need to move OFFSET into a register or add OFFSET to a register;
2577 ADD_P is true if we want the latter rather than the former. */
2579 static unsigned int
2581 {
2582 /* This follows the same structure as aarch64_add_offset. */
2591 /* Need one register for the ADDVL/ADDPL result. */
2594 {
2597 /* Need one register for the CNT result and one for the multiplication
2598 factor. If necessary, the second temporary can be reused for the
2599 constant part of the offset. */
2601 /* Need one register for the CNT result (which might then
2602 be shifted). */
2604 }
2606 }
2608 /* If X can be represented as a poly_int64, return the number
2609 of temporaries that are required to add it to a register.
2610 Return -1 otherwise. */
2612 int
2614 {
2615 poly_int64 offset;
2619 }
2621 /* Set DEST to SRC + OFFSET. MODE is the mode of the addition.
2622 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
2623 be set and CFA adjustments added to the generated instructions.
2625 TEMP1, if nonnull, is a register of mode MODE that can be used as a
2626 temporary if register allocation is already complete. This temporary
2627 register may overlap DEST if !FRAME_RELATED_P but must not overlap SRC.
2628 If TEMP1 is known to hold abs (OFFSET), EMIT_MOVE_IMM can be set to
2629 false to avoid emitting the immediate again.
2631 TEMP2, if nonnull, is a second temporary register that doesn't
2632 overlap either DEST or REG.
2634 Since this function may be used to adjust the stack pointer, we must
2635 ensure that it cannot cause transient stack deallocation (for example
2636 by first incrementing SP and then decrementing when adjusting by a
2637 large immediate). */
2639 static void
2643 {
2647 || !frame_related_p
2651 /* Try using ADDVL or ADDPL to add the whole value. */
2653 {
2658 }
2660 /* Coefficient 1 is multiplied by the number of 128-bit blocks in an
2661 SVE vector register, over and above the minimum size of 128 bits.
2662 This is equivalent to half the value returned by CNTD with a
2663 vector shape of ALL. */
2667 /* Try using ADDVL or ADDPL to add the VG-based part. */
2671 {
2674 {
2678 }
2679 else
2680 {
2685 }
2686 }
2687 /* Otherwise use a CNT-based sequence. */
2689 {
2690 /* Use a subtraction if we have a negative factor. */
2693 {
2696 }
2698 /* Calculate CNTD * FACTOR / 2. First try to fold the division
2699 into the multiplication. */
2700 rtx val;
2703 /* Use a right shift by 1. */
2705 else
2709 {
2711 {
2712 /* "CNTB Xn, ALL, MUL #FACTOR" is out of range, so calculate
2713 the value with the minimum multiplier and shift it into
2714 position. */
2718 }
2720 }
2721 else
2722 {
2723 /* Use CNTD, then multiply it by FACTOR. */
2727 /* Go back to using a negative multiplication factor if we have
2728 no register from which to subtract. */
2730 {
2733 }
2737 }
2740 {
2741 /* Multiply by 1 << SHIFT. */
2744 }
2746 {
2747 /* Divide by 2. */
2750 }
2752 /* Calculate SRC +/- CNTD * FACTOR / 2. */
2754 {
2757 }
2759 {
2762 }
2765 {
2768 {
2772 poly_offset)));
2773 }
2777 }
2778 else
2779 {
2783 }
2786 }
2790 }
2792 /* Like aarch64_add_offset, but the offset is given as an rtx rather
2793 than a poly_int64. */
2795 void
2798 {
2801 }
2803 /* Add DELTA to the stack pointer, marking the instructions frame-related.
2804 TEMP1 is available as a temporary if nonnull. EMIT_MOVE_IMM is false
2805 if TEMP1 already contains abs (DELTA). */
2809 {
2812 }
2814 /* Subtract DELTA from the stack pointer, marking the instructions
2815 frame-related if FRAME_RELATED_P. TEMP1 is available as a temporary
2816 if nonnull. */
2820 {
2823 }
2825 /* Set DEST to (vec_series BASE STEP). */
2827 static void
2829 {
2833 /* Each operand can be a register or an immediate in the range [-16, 15]. */
2840 }
2842 /* Try to duplicate SRC into SVE register DEST, given that SRC is an
2843 integer of mode INT_MODE. Return true on success. */
2845 static bool
2847 rtx src)
2848 {
2849 /* If the constant is smaller than 128 bits, we can do the move
2850 using a vector of SRC_MODEs. */
2852 {
2859 }
2861 /* Use LD1RQ[BHWD] to load the 128 bits from memory. */
2866 /* Make sure that the address is legitimate. */
2868 {
2871 }
2880 }
2882 /* Expand a move of general CONST_VECTOR SRC into DEST, given that it
2883 isn't a simple duplicate or series. */
2885 static void
2887 {
2894 {
2895 /* The constant is a repeating seqeuence of at least two elements,
2896 where the repeating elements occupy no more than 128 bits.
2897 Get an integer representation of the replicated value. */
2898 scalar_int_mode int_mode;
2900 /* For now, always use LD1RQ to load the value on big-endian
2901 targets, since the handling of smaller integers includes a
2902 subreg that is semantically an element reverse. */
2904 else
2905 {
2909 }
2914 }
2916 /* Expand each pattern individually. */
2917 rtx_vector_builder builder;
2920 {
2925 }
2927 /* Use permutes to interleave the separate vectors. */
2929 {
2932 {
2937 }
2938 }
2940 }
2942 /* Set DEST to immediate IMM. For SVE vector modes, GEN_VEC_DUPLICATE
2943 is a pattern that can be used to set DEST to a replicated scalar
2944 element. */
2946 void
2949 {
2952 /* Check on what type of symbol it is. */
2953 scalar_int_mode int_mode;
2959 {
2960 rtx mem;
2961 poly_int64 offset;
2962 HOST_WIDE_INT const_offset;
2965 /* If we have (const (plus symbol offset)), separate out the offset
2966 before we start classifying the symbol. */
2969 /* We must always add an offset involving VL separately, rather than
2970 folding it into the relocation. */
2972 {
2975 else
2976 {
2977 /* Do arithmetic on 32-bit values if the result is smaller
2978 than that. */
2980 {
2981 /* It is invalid to do symbol calculations in modes
2982 narrower than SImode. */
2986 }
2988 {
2992 }
2993 else
2996 }
2998 }
3002 {
3006 {
3012 }
3017 /* If we aren't generating PC relative literals, then
3018 we need to expand the literal pool access carefully.
3019 This is something that needs to be done in a number
3020 of places, so could well live as a separate function. */
3022 {
3029 }
3046 {
3052 }
3053 /* FALLTHRU */
3066 }
3067 }
3070 {
3078 {
3079 /* If the constant is out of range of an SVE vector move,
3080 load it from memory if we can, otherwise move it into
3081 a register and use a DUP. */
3087 {
3090 }
3092 }
3096 else
3097 {
3101 }
3104 }
3108 }
3110 /* Emit an SVE predicated move from SRC to DEST. PRED is a predicate
3111 that is known to contain PTRUE. */
3113 void
3115 {
3118 UNSPEC_MERGE_PTRUE)));
3119 }
3121 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one
3122 operand is in memory. In this case we need to use the predicated LD1
3123 and ST1 instead of LDR and STR, both for correctness on big-endian
3124 targets and because LD1 and ST1 support a wider range of addressing modes.
3125 PRED_MODE is the mode of the predicate.
3127 See the comment at the head of aarch64-sve.md for details about the
3128 big-endian handling. */
3130 void
3132 {
3137 {
3141 else
3144 }
3146 }
3148 /* Called only on big-endian targets. See whether an SVE vector move
3149 from SRC to DEST is effectively a REV[BHW] instruction, because at
3150 least one operand is a subreg of an SVE vector that has wider or
3151 narrower elements. Return true and emit the instruction if so.
3153 For example:
3155 (set (reg:VNx8HI R1) (subreg:VNx8HI (reg:VNx16QI R2) 0))
3157 represents a VIEW_CONVERT between the following vectors, viewed
3158 in memory order:
3160 R2: { [0].high, [0].low, [1].high, [1].low, ... }
3161 R1: { [0], [1], [2], [3], ... }
3163 The high part of lane X in R2 should therefore correspond to lane X*2
3164 of R1, but the register representations are:
3166 msb lsb
3167 R2: ...... [1].high [1].low [0].high [0].low
3168 R1: ...... [3] [2] [1] [0]
3170 where the low part of lane X in R2 corresponds to lane X*2 in R1.
3171 We therefore need a reverse operation to swap the high and low values
3172 around.
3174 This is purely an optimization. Without it we would spill the
3175 subreg operand to the stack in one mode and reload it in the
3176 other mode, which has the same effect as the REV. */
3178 bool
3180 {
3187 /* The optimization handles two single SVE REGs with different element
3188 sizes. */
3197 /* Generate *aarch64_sve_mov<mode>_subreg_be. */
3200 UNSPEC_REV_SUBREG);
3203 }
3205 /* Return a copy of X with mode MODE, without changing its other
3206 attributes. Unlike gen_lowpart, this doesn't care whether the
3207 mode change is valid. */
3209 static rtx
3211 {
3218 }
3220 /* Split a *aarch64_sve_mov<mode>_subreg_be pattern with the given
3221 operands. */
3223 void
3225 {
3226 /* Decide which REV operation we need. The mode with narrower elements
3227 determines the mode of the operands and the mode with the wider
3228 elements determines the reverse width. */
3243 else
3247 /* Emit:
3249 (set DEST (unspec [PTRUE (unspec [SRC] UNSPEC_REV<nn>)]
3250 UNSPEC_MERGE_PTRUE))
3252 with the appropriate modes. */
3258 UNSPEC_MERGE_PTRUE);
3260 }
3262 static bool
3264 tree exp ATTRIBUTE_UNUSED)
3265 {
3266 /* Currently, always true. */
3268 }
3270 /* Implement TARGET_PASS_BY_REFERENCE. */
3272 static bool
3274 machine_mode mode,
3275 const_tree type,
3277 {
3278 HOST_WIDE_INT size;
3279 machine_mode dummymode;
3282 /* GET_MODE_SIZE (BLKmode) is useless since it is 0. */
3285 else
3286 /* No frontends can create types with variable-sized modes, so we
3287 shouldn't be asked to pass or return them. */
3290 /* Aggregates are passed by reference based on their size. */
3292 {
3294 }
3296 /* Variable sized arguments are always returned by reference. */
3300 /* Can this be a candidate to be passed in fp/simd register(s)? */
3303 NULL))
3306 /* Arguments which are variable sized or larger than 2 registers are
3307 passed by reference unless they are a homogenous floating point
3308 aggregate. */
3310 }
3312 /* Return TRUE if VALTYPE is padded to its least significant bits. */
3313 static bool
3315 {
3316 machine_mode dummy_mode;
3319 /* Never happens in little-endian mode. */
3323 /* Only composite types smaller than or equal to 16 bytes can
3324 be potentially returned in registers. */
3330 /* But not a composite that is an HFA (Homogeneous Floating-point Aggregate)
3331 or an HVA (Homogeneous Short-Vector Aggregate); such a special composite
3332 is always passed/returned in the least significant bits of fp/simd
3333 register(s). */
3339 }
3341 /* Implement TARGET_FUNCTION_VALUE.
3342 Define how to find the value returned by a function. */
3344 static rtx
3347 {
3348 machine_mode mode;
3351 machine_mode ag_mode;
3358 {
3362 {
3365 }
3366 }
3370 {
3372 {
3375 }
3376 else
3377 {
3379 rtx par;
3383 {
3388 }
3390 }
3391 }
3392 else
3394 }
3396 /* Implements TARGET_FUNCTION_VALUE_REGNO_P.
3397 Return true if REGNO is the number of a hard register in which the values
3398 of called function may come back. */
3400 static bool
3402 {
3403 /* Maximum of 16 bytes can be returned in the general registers. Examples
3404 of 16-byte return values are: 128-bit integers and 16-byte small
3405 structures (excluding homogeneous floating-point aggregates). */
3409 /* Up to four fp/simd registers can return a function value, e.g. a
3410 homogeneous floating-point aggregate having four members. */
3415 }
3417 /* Implement TARGET_RETURN_IN_MEMORY.
3419 If the type T of the result of a function is such that
3420 void func (T arg)
3421 would require that arg be passed as a value in a register (or set of
3422 registers) according to the parameter passing rules, then the result
3423 is returned in the same registers as would be used for such an
3424 argument. */
3426 static bool
3428 {
3429 HOST_WIDE_INT size;
3430 machine_mode ag_mode;
3436 /* Simple scalar types always returned in registers. */
3440 type,
3443 NULL))
3446 /* Types larger than 2 registers returned in memory. */
3449 }
3451 static bool
3454 {
3457 type,
3459 nregs,
3460 NULL);
3461 }
3463 /* Given MODE and TYPE of a function argument, return the alignment in
3464 bits. The idea is to suppress any stronger alignment requested by
3465 the user and opt for the natural alignment (specified in AAPCS64 \S 4.1).
3466 This is a helper function for local use only. */
3468 static unsigned int
3470 {
3491 }
3493 /* Layout a function argument according to the AAPCS64 rules. The rule
3494 numbers refer to the rule numbers in the AAPCS64. */
3496 static void
3498 const_tree type,
3500 {
3504 HOST_WIDE_INT size;
3506 /* We need to do this once per argument. */
3512 /* Size in bytes, rounded to the nearest multiple of 8 bytes. */
3515 else
3516 /* No frontends can create types with variable-sized modes, so we
3517 shouldn't be asked to pass or return them. */
3523 mode,
3524 type,
3527 /* allocate_ncrn may be false-positive, but allocate_nvrn is quite reliable.
3528 The following code thus handles passing by SIMD/FP registers first. */
3532 /* C1 - C5 for floating point, homogenous floating point aggregates (HFA)
3533 and homogenous short-vector aggregates (HVA). */
3535 {
3540 {
3543 {
3546 }
3547 else
3548 {
3549 rtx par;
3553 {
3556 rtx offset = gen_int_mode
3560 }
3562 }
3564 }
3565 else
3566 {
3567 /* C.3 NSRN is set to 8. */
3570 }
3571 }
3576 /* C6 - C9. though the sign and zero extension semantics are
3577 handled elsewhere. This is the case where the argument fits
3578 entirely general registers. */
3580 {
3584 /* C.8 if the argument has an alignment of 16 then the NGRN is
3585 rounded up to the next even number. */
3588 /* The == 16 * BITS_PER_UNIT instead of >= 16 * BITS_PER_UNIT
3589 comparison is there because for > 16 * BITS_PER_UNIT
3590 alignment nregs should be > 2 and therefore it should be
3591 passed by reference rather than value. */
3593 {
3596 }
3598 /* NREGS can be 0 when e.g. an empty structure is to be passed.
3599 A reg is still generated for it, but the caller should be smart
3600 enough not to use it. */
3603 else
3604 {
3605 rtx par;
3610 {
3615 }
3617 }
3621 }
3623 /* C.11 */
3626 /* The argument is passed on stack; record the needed number of words for
3627 this argument and align the total size if necessary. */
3628 on_stack:
3635 }
3637 /* Implement TARGET_FUNCTION_ARG. */
3639 static rtx
3642 {
3651 }
3653 void
3655 const_tree fntype ATTRIBUTE_UNUSED,
3656 rtx libname ATTRIBUTE_UNUSED,
3657 const_tree fndecl ATTRIBUTE_UNUSED,
3659 {
3673 {
3680 }
3682 }
3684 static void
3686 machine_mode mode,
3687 const_tree type,
3689 {
3692 {
3702 }
3703 }
3705 bool
3707 {
3710 }
3712 /* Implement FUNCTION_ARG_BOUNDARY. Every parameter gets at least
3713 PARM_BOUNDARY bits of alignment, but will be given anything up
3714 to STACK_BOUNDARY bits if the type requires it. This makes sure
3715 that both before and after the layout of each argument, the Next
3716 Stacked Argument Address (NSAA) will have a minimum alignment of
3717 8 bytes. */
3719 static unsigned int
3721 {
3724 }
3726 /* Implement TARGET_GET_RAW_RESULT_MODE and TARGET_GET_RAW_ARG_MODE. */
3728 static fixed_size_mode
3730 {
3732 /* Don't use the SVE part of the register for __builtin_apply and
3733 __builtin_return. The SVE registers aren't used by the normal PCS,
3734 so using them there would be a waste of time. The PCS extensions
3735 for SVE types are fundamentally incompatible with the
3736 __builtin_return/__builtin_apply interface. */
3739 }
3741 /* Implement TARGET_FUNCTION_ARG_PADDING.
3743 Small aggregate types are placed in the lowest memory address.
3745 The related parameter passing rules are B.4, C.3, C.5 and C.14. */
3747 static pad_direction
3749 {
3750 /* On little-endian targets, the least significant byte of every stack
3751 argument is passed at the lowest byte address of the stack slot. */
3755 /* Otherwise, integral, floating-point and pointer types are padded downward:
3756 the least significant byte of a stack argument is passed at the highest
3757 byte address of the stack slot. */
3764 /* Everything else padded upward, i.e. data in first byte of stack slot. */
3766 }
3768 /* Similarly, for use by BLOCK_REG_PADDING (MODE, TYPE, FIRST).
3770 It specifies padding for the last (may also be the only)
3771 element of a block move between registers and memory. If
3772 assuming the block is in the memory, padding upward means that
3773 the last element is padded after its highest significant byte,
3774 while in downward padding, the last element is padded at the
3775 its least significant byte side.
3777 Small aggregates and small complex types are always padded
3778 upwards.
3780 We don't need to worry about homogeneous floating-point or
3781 short-vector aggregates; their move is not affected by the
3782 padding direction determined here. Regardless of endianness,
3783 each element of such an aggregate is put in the least
3784 significant bits of a fp/simd register.
3786 Return !BYTES_BIG_ENDIAN if the least significant byte of the
3787 register has useful data, and return the opposite if the most
3788 significant byte does. */
3790 bool
3793 {
3795 /* Small composite types are always padded upward. */
3797 {
3798 HOST_WIDE_INT size;
3801 else
3802 /* No frontends can create types with variable-sized modes, so we
3803 shouldn't be asked to pass or return them. */
3807 }
3809 /* Otherwise, use the default padding. */
3811 }
3813 static scalar_int_mode
3815 {
3817 }
3819 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
3821 /* We use the 12-bit shifted immediate arithmetic instructions so values
3822 must be multiple of (1 << 12), i.e. 4096. */
3823 #define ARITH_FACTOR 4096
3825 #if (PROBE_INTERVAL % ARITH_FACTOR) != 0
3826 #error Cannot use simple address calculation for stack probing
3827 #endif
3829 /* The pair of scratch registers used for stack probing. */
3830 #define PROBE_STACK_FIRST_REG 9
3831 #define PROBE_STACK_SECOND_REG 10
3833 /* Emit code to probe a range of stack addresses from FIRST to FIRST+POLY_SIZE,
3834 inclusive. These are offsets from the current stack pointer. */
3836 static void
3838 {
3839 HOST_WIDE_INT size;
3841 {
3844 }
3848 /* See the same assertion on PROBE_INTERVAL above. */
3851 /* See if we have a constant small number of probes to generate. If so,
3852 that's the easy case. */
3854 {
3861 }
3863 /* The run-time loop is made up of 8 insns in the generic case while the
3864 compile-time loop is made up of 4+2*(n-2) insns for n # of intervals. */
3866 {
3871 stack_pointer_rtx,
3875 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
3876 it exceeds SIZE. If only two probes are needed, this will not
3877 generate any code. Then probe at FIRST + SIZE. */
3879 {
3883 }
3887 {
3892 }
3893 else
3895 }
3897 /* Otherwise, do the same as above, but in a loop. Note that we must be
3898 extra careful with variables wrapping around because we might be at
3899 the very top (or the very bottom) of the address space and we have
3900 to be able to handle this case properly; in particular, we use an
3901 equality test for the loop condition. */
3902 else
3903 {
3906 /* Step 1: round SIZE to the previous multiple of the interval. */
3911 /* Step 2: compute initial and final value of the loop counter. */
3913 /* TEST_ADDR = SP + FIRST. */
3917 /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE. */
3920 {
3924 }
3925 else
3929 /* Step 3: the loop
3931 do
3932 {
3933 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
3934 probe at TEST_ADDR
3935 }
3936 while (TEST_ADDR != LAST_ADDR)
3938 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
3939 until it is equal to ROUNDED_SIZE. */
3944 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
3945 that SIZE is equal to ROUNDED_SIZE. */
3948 {
3952 {
3957 }
3958 else
3960 }
3961 }
3963 /* Make sure nothing is scheduled before we are done. */
3965 }
3967 /* Probe a range of stack addresses from REG1 to REG2 inclusive. These are
3968 absolute addresses. */
3972 {
3979 /* Loop. */
3982 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
3987 /* Probe at TEST_ADDR. */
3990 /* Test if TEST_ADDR == LAST_ADDR. */
3994 /* Branch. */
4000 }
4002 /* Determine whether a frame chain needs to be generated. */
4003 static bool
4005 {
4006 /* Force a frame chain for EH returns so the return address is at FP+8. */
4010 /* A leaf function cannot have calls or write LR. */
4013 /* Don't use a frame chain in leaf functions if leaf frame pointers
4014 are disabled. */
4019 }
4021 /* Mark the registers that need to be saved by the callee and calculate
4022 the size of the callee-saved registers area and frame record (both FP
4023 and LR may be omitted). */
4024 static void
4026 {
4032 #define SLOT_NOT_REQUIRED (-2)
4033 #define SLOT_REQUIRED (-1)
4038 /* First mark all the registers that really need to be saved... */
4045 /* ... that includes the eh data registers (if needed)... */
4051 /* ... and any callee saved register that dataflow says is live. */
4061 {
4064 }
4067 {
4068 /* FP and LR are placed in the linkage record. */
4074 }
4076 /* Now assign stack slots for them. */
4079 {
4086 }
4094 {
4095 /* If there is an alignment gap between integer and fp callee-saves,
4096 allocate the last fp register to it if possible. */
4098 {
4101 }
4110 }
4116 HOST_WIDE_INT varargs_and_saved_regs_size
4124 /* Both these values are already aligned. */
4148 {
4149 /* Simple, small frame with no outgoing arguments:
4150 stp reg1, reg2, [sp, -frame_size]!
4151 stp reg3, reg4, [sp, 16] */
4153 }
4158 max_push_offset)))
4159 {
4160 /* Frame with small outgoing arguments:
4161 sub sp, sp, frame_size
4162 stp reg1, reg2, [sp, outgoing_args_size]
4163 stp reg3, reg4, [sp, outgoing_args_size + 16] */
4167 }
4170 {
4171 /* Frame with large outgoing arguments but a small local area:
4172 stp reg1, reg2, [sp, -hard_fp_offset]!
4173 stp reg3, reg4, [sp, 16]
4174 sub sp, sp, outgoing_args_size */
4178 }
4179 else
4180 {
4181 /* Frame with large local area and outgoing arguments using frame pointer:
4182 sub sp, sp, hard_fp_offset
4183 stp x29, x30, [sp, 0]
4184 add x29, sp, 0
4185 stp reg3, reg4, [sp, 16]
4186 sub sp, sp, outgoing_args_size */
4190 }
4193 }
4195 /* Return true if the register REGNO is saved on entry to
4196 the current function. */
4198 static bool
4200 {
4202 }
4204 /* Return the next register up from REGNO up to LIMIT for the callee
4205 to save. */
4207 static unsigned
4209 {
4211 regno ++;
4213 }
4215 /* Push the register number REGNO of mode MODE to the stack with write-back
4216 adjusting the stack by ADJUSTMENT. */
4218 static void
4220 HOST_WIDE_INT adjustment)
4221 {
4232 }
4234 /* Generate and return an instruction to store the pair of registers
4235 REG and REG2 of mode MODE to location BASE with write-back adjusting
4236 the stack location BASE by ADJUSTMENT. */
4238 static rtx
4240 HOST_WIDE_INT adjustment)
4241 {
4243 {
4254 }
4255 }
4257 /* Push registers numbered REGNO1 and REGNO2 to the stack, adjusting the
4258 stack pointer by ADJUSTMENT. */
4260 static void
4262 {
4277 }
4279 /* Load the pair of register REG, REG2 of mode MODE from stack location BASE,
4280 adjusting it by ADJUSTMENT afterwards. */
4282 static rtx
4284 HOST_WIDE_INT adjustment)
4285 {
4287 {
4296 }
4297 }
4299 /* Pop the two registers numbered REGNO1, REGNO2 from the stack, adjusting it
4300 afterwards by ADJUSTMENT and writing the appropriate REG_CFA_RESTORE notes
4301 into CFI_OPS. */
4303 static void
4306 {
4313 {
4317 }
4318 else
4319 {
4324 }
4325 }
4327 /* Generate and return a store pair instruction of mode MODE to store
4328 register REG1 to MEM1 and register REG2 to MEM2. */
4330 static rtx
4332 rtx reg2)
4333 {
4335 {
4344 }
4345 }
4347 /* Generate and regurn a load pair isntruction of mode MODE to load register
4348 REG1 from MEM1 and register REG2 from MEM2. */
4350 static rtx
4352 rtx mem2)
4353 {
4355 {
4364 }
4365 }
4367 /* Return TRUE if return address signing should be enabled for the current
4368 function, otherwise return FALSE. */
4370 bool
4372 {
4373 /* This function should only be called after frame laid out. */
4376 /* If signing scope is AARCH64_FUNCTION_NON_LEAF, we only sign a leaf function
4377 if it's LR is pushed onto stack. */
4381 }
4383 /* Emit code to save the callee-saved registers from register number START
4384 to LIMIT to the stack at the location starting at offset START_OFFSET,
4385 skipping any write-back candidates if SKIP_WB is true. */
4387 static void
4390 {
4398 {
4400 poly_int64 offset;
4413 offset));
4422 {
4424 rtx mem2;
4428 offset));
4430 reg2));
4432 /* The first part of a frame-related parallel insn is
4433 always assumed to be relevant to the frame
4434 calculations; subsequent parts, are only
4435 frame-related if explicitly marked. */
4438 }
4439 else
4443 }
4444 }
4446 /* Emit code to restore the callee registers of mode MODE from register
4447 number START up to and including LIMIT. Restore from the stack offset
4448 START_OFFSET, skipping any write-back candidates if SKIP_WB is true.
4449 Write the appropriate REG_CFA_RESTORE notes into CFI_OPS. */
4451 static void
4455 {
4459 poly_int64 offset;
4464 {
4485 {
4487 rtx mem2;
4495 }
4496 else
4499 }
4500 }
4502 /* Return true if OFFSET is a signed 4-bit value multiplied by the size
4503 of MODE. */
4507 {
4508 HOST_WIDE_INT multiple;
4511 }
4513 /* Return true if OFFSET is a unsigned 6-bit value multiplied by the size
4514 of MODE. */
4518 {
4519 HOST_WIDE_INT multiple;
4522 }
4524 /* Return true if OFFSET is a signed 7-bit value multiplied by the size
4525 of MODE. */
4527 bool
4529 {
4530 HOST_WIDE_INT multiple;
4533 }
4535 /* Return true if OFFSET is a signed 9-bit value. */
4537 bool
4539 poly_int64 offset)
4540 {
4541 HOST_WIDE_INT const_offset;
4544 }
4546 /* Return true if OFFSET is a signed 9-bit value multiplied by the size
4547 of MODE. */
4551 {
4552 HOST_WIDE_INT multiple;
4555 }
4557 /* Return true if OFFSET is an unsigned 12-bit value multiplied by the size
4558 of MODE. */
4562 {
4563 HOST_WIDE_INT multiple;
4566 }
4568 /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
4570 static sbitmap
4572 {
4576 /* The registers we need saved to the frame. */
4579 {
4584 /* Check that we can access the stack slot of the register with one
4585 direct load with no adjustments needed. */
4588 }
4590 /* Don't mess with the hard frame pointer. */
4596 /* If registers have been chosen to be stored/restored with
4597 writeback don't interfere with them to avoid having to output explicit
4598 stack adjustment instructions. */
4608 }
4610 /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
4612 static sbitmap
4614 {
4622 /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
4628 {
4632 /* If there is a callee-save at an adjacent offset, add it too
4633 to increase the use of LDP/STP. */
4638 {
4642 }
4643 }
4646 }
4648 /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS.
4649 Nothing to do for aarch64. */
4651 static void
4653 {
4654 }
4656 /* Return the next set bit in BMP from START onwards. Return the total number
4657 of bits in BMP if no set bit is found at or after START. */
4659 static unsigned int
4661 {
4672 }
4674 /* Do the work for aarch64_emit_prologue_components and
4675 aarch64_emit_epilogue_components. COMPONENTS is the bitmap of registers
4676 to save/restore, PROLOGUE_P indicates whether to emit the prologue sequence
4677 for these components or the epilogue sequence. That is, it determines
4678 whether we should emit stores or loads and what kind of CFA notes to attach
4679 to the insns. Otherwise the logic for the two sequences is very
4680 similar. */
4682 static void
4684 {
4686 ? HARD_FRAME_POINTER_REGNUM
4694 {
4695 /* AAPCS64 section 5.1.2 requires only the bottom 64 bits to be saved
4696 so DFmode for the vector registers is enough. */
4708 /* No more registers to handle after REGNO.
4709 Emit a single save/restore and exit. */
4711 {
4716 else
4719 }
4722 /* The next register is not of the same class or its offset is not
4723 mergeable with the current one into a pair. */
4728 {
4733 else
4738 }
4740 /* REGNO2 can be saved/restored in a pair with REGNO. */
4752 else
4757 {
4760 }
4761 else
4762 {
4765 }
4768 }
4769 }
4771 /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
4773 static void
4775 {
4777 }
4779 /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
4781 static void
4783 {
4785 }
4787 /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
4789 static void
4791 {
4795 }
4797 /* Add a REG_CFA_EXPRESSION note to INSN to say that register REG
4798 is saved at BASE + OFFSET. */
4800 static void
4803 {
4807 }
4809 /* AArch64 stack frames generated by this compiler look like:
4811 +-------------------------------+
4812 | |
4813 | incoming stack arguments |
4814 | |
4815 +-------------------------------+
4816 | | <-- incoming stack pointer (aligned)
4817 | callee-allocated save area |
4818 | for register varargs |
4819 | |
4820 +-------------------------------+
4821 | local variables | <-- frame_pointer_rtx
4822 | |
4823 +-------------------------------+
4824 | padding0 | \
4825 +-------------------------------+ |
4826 | callee-saved registers | | frame.saved_regs_size
4827 +-------------------------------+ |
4828 | LR' | |
4829 +-------------------------------+ |
4830 | FP' | / <- hard_frame_pointer_rtx (aligned)
4831 +-------------------------------+
4832 | dynamic allocation |
4833 +-------------------------------+
4834 | padding |
4835 +-------------------------------+
4836 | outgoing stack arguments | <-- arg_pointer
4837 | |
4838 +-------------------------------+
4839 | | <-- stack_pointer_rtx (aligned)
4841 Dynamic stack allocations via alloca() decrease stack_pointer_rtx
4842 but leave frame_pointer_rtx and hard_frame_pointer_rtx
4843 unchanged. */
4845 /* Generate the prologue instructions for entry into a function.
4846 Establish the stack frame by decreasing the stack pointer with a
4847 properly calculated size and, if necessary, create a frame record
4848 filled with the values of LR and previous frame pointer. The
4849 current FP is also set up if it is in use. */
4851 void
4853 {
4864 /* Sign return address for functions. */
4866 {
4870 }
4876 {
4878 {
4882 (frame_size
4884 }
4887 }
4898 {
4901 {
4906 }
4911 {
4912 /* Variable-sized frames need to describe the save slot
4913 address using DW_CFA_expression rather than DW_CFA_offset.
4914 This means that, without taking further action, the
4915 locations of the registers that we've already saved would
4916 remain based on the stack pointer even after we redefine
4917 the CFA based on the frame pointer. We therefore need new
4918 DW_CFA_expressions to re-express the save slots with addresses
4919 based on the frame pointer. */
4923 /* Add an explicit CFA definition if this was previously
4924 implicit. */
4926 {
4928 callee_offset);
4931 }
4933 /* Change the save slot expressions for the registers that
4934 we've already saved. */
4939 reg_offset);
4940 }
4942 }
4949 }
4951 /* Return TRUE if we can use a simple_return insn.
4953 This function checks whether the callee saved stack is empty, which
4954 means no restore actions are need. The pro_and_epilogue will use
4955 this to check whether shrink-wrapping opt is feasible. */
4957 bool
4959 {
4967 }
4969 /* Generate the epilogue instructions for returning from a function.
4970 This is almost exactly the reverse of the prolog sequence, except
4971 that we need to insert barriers to avoid scheduling loads that read
4972 from a deallocated stack, and we optimize the unwind records by
4973 emitting them all together if possible. */
4974 void
4976 {
4985 /* A stack clash protection prologue may not have left IP0_REGNUM or
4986 IP1_REGNUM in a usable state. The same is true for allocations
4987 with an SVE component, since we then need both temporary registers
4988 for each allocation. */
4993 /* We need to add memory barrier to prevent read from deallocated stack. */
4994 bool need_barrier_p
4998 /* Emit a barrier to prevent loads from a deallocated stack. */
5002 {
5005 }
5007 /* Restore the stack pointer from the frame pointer if it may not
5008 be the same as the stack pointer. */
5013 /* If writeback is used when restoring callee-saves, the CFA
5014 is restored on the instruction doing the writeback. */
5018 else
5034 {
5035 /* Emit delayed restores and set the CFA to be SP + initial_adjust. */
5041 }
5047 {
5048 /* Emit delayed restores and reset the CFA to be SP. */
5053 }
5055 /* We prefer to emit the combined return/authenticate instruction RETAA,
5056 however there are three cases in which we must instead emit an explicit
5057 authentication instruction.
5059 1) Sibcalls don't return in a normal way, so if we're about to call one
5060 we must authenticate.
5062 2) The RETAA instruction is not available before ARMv8.3-A, so if we are
5063 generating code for !TARGET_ARMV8_3 we can't use it and must
5064 explicitly authenticate.
5066 3) On an eh_return path we make extra stack adjustments to update the
5067 canonical frame address to be the exception handler's CFA. We want
5068 to authenticate using the CFA of the function which calls eh_return.
5069 */
5072 {
5076 }
5078 /* Stack adjustment for exception handler. */
5080 {
5081 /* We need to unwind the stack by the offset computed by
5082 EH_RETURN_STACKADJ_RTX. We have already reset the CFA
5083 to be SP; letting the CFA move during this adjustment
5084 is just as correct as retaining the CFA from the body
5085 of the function. Therefore, do nothing special. */
5087 }
5092 }
5094 /* Implement EH_RETURN_HANDLER_RTX. EH returns need to either return
5095 normally or return to a previous frame after unwinding.
5097 An EH return uses a single shared return sequence. The epilogue is
5098 exactly like a normal epilogue except that it has an extra input
5099 register (EH_RETURN_STACKADJ_RTX) which contains the stack adjustment
5100 that must be applied after the frame has been destroyed. An extra label
5101 is inserted before the epilogue which initializes this register to zero,
5102 and this is the entry point for a normal return.
5104 An actual EH return updates the return address, initializes the stack
5105 adjustment and jumps directly into the epilogue (bypassing the zeroing
5106 of the adjustment). Since the return address is typically saved on the
5107 stack when a function makes a call, the saved LR must be updated outside
5108 the epilogue.
5110 This poses problems as the store is generated well before the epilogue,
5111 so the offset of LR is not known yet. Also optimizations will remove the
5112 store as it appears dead, even after the epilogue is generated (as the
5113 base or offset for loading LR is different in many cases).
5115 To avoid these problems this implementation forces the frame pointer
5116 in eh_return functions so that the location of LR is fixed and known early.
5117 It also marks the store volatile, so no optimization is permitted to
5118 remove the store. */
5119 rtx
5121 {
5125 /* Mark the store volatile, so no optimization is permitted to remove it. */
5128 }
5130 /* Output code to add DELTA to the first argument, and then jump
5131 to FUNCTION. Used for C++ multiple inheritance. */
5132 static void
5134 HOST_WIDE_INT delta,
5135 HOST_WIDE_INT vcall_offset,
5136 tree function)
5137 {
5138 /* The this pointer is always in x0. Note that this differs from
5139 Arm where the this pointer maybe bumped to r1 if r0 is required
5140 to return a pointer to an aggregate. On AArch64 a result value
5141 pointer will be in x8. */
5155 else
5156 {
5161 {
5165 else
5168 }
5172 else
5179 else
5180 {
5182 Pmode);
5184 }
5188 else
5194 }
5196 /* Generate a tail call to the target function. */
5198 {
5201 }
5213 /* Stop pretending to be a post-reload pass. */
5215 }
5217 static bool
5219 {
5224 {
5228 /* Don't recurse into UNSPEC_TLS looking for TLS symbols; these are
5229 TLS offsets, not real symbol references. */
5232 }
5234 }
5237 /* Return true if val can be encoded as a 12-bit unsigned immediate with
5238 a left shift of 0 or 12 bits. */
5239 bool
5241 {
5244 );
5245 }
5248 /* Return true if val is an immediate that can be loaded into a
5249 register by a MOVZ instruction. */
5250 static bool
5252 {
5254 {
5258 }
5259 else
5260 {
5261 /* Ignore sign extension. */
5263 }
5266 }
5268 /* VAL is a value with the inner mode of MODE. Replicate it to fill a
5269 64-bit (DImode) integer. */
5271 static unsigned HOST_WIDE_INT
5273 {
5276 {
5280 }
5282 }
5284 /* Multipliers for repeating bitmasks of width 32, 16, 8, 4, and 2. */
5287 {
5293 };
5296 /* Return true if val is a valid bitmask immediate. */
5298 bool
5300 {
5304 /* Check for a single sequence of one bits and return quickly if so.
5305 The special cases of all ones and all zeroes returns false. */
5312 /* Replicate 32-bit immediates so we can treat them as 64-bit. */
5316 /* Invert if the immediate doesn't start with a zero bit - this means we
5317 only need to search for sequences of one bits. */
5321 /* Find the first set bit and set tmp to val with the first sequence of one
5322 bits removed. Return success if there is a single sequence of ones. */
5329 /* Find the next set bit and compute the difference in bit position. */
5334 /* Check the bit position difference is a power of 2, and that the first
5335 sequence of one bits fits within 'bits' bits. */
5339 /* Check the sequence of one bits is repeated 64/bits times. */
5341 }
5343 /* Create mask of ones, covering the lowest to highest bits set in VAL_IN.
5344 Assumed precondition: VAL_IN Is not zero. */
5346 unsigned HOST_WIDE_INT
5348 {
5355 }
5357 /* Create constant where bits outside of lowest bit set to highest bit set
5358 are set to 1. */
5360 unsigned HOST_WIDE_INT
5362 {
5364 }
5366 /* Return true if VAL_IN is a valid 'and' bitmask immediate. */
5368 bool
5370 {
5371 scalar_int_mode int_mode;
5384 }
5386 /* Return true if val is an immediate that can be loaded into a
5387 register in a single instruction. */
5388 bool
5390 {
5391 scalar_int_mode int_mode;
5398 }
5400 static bool
5402 {
5408 /* There's no way to calculate VL-based values using relocations. */
5416 {
5420 else
5421 /* Avoid generating a 64-bit relocation in ILP32; leave
5422 to aarch64_expand_mov_immediate to handle it properly. */
5424 }
5427 }
5429 /* Implement TARGET_CASE_VALUES_THRESHOLD.
5430 The expansion for a table switch is quite expensive due to the number
5431 of instructions, the table lookup and hard to predict indirect jump.
5432 When optimizing for speed, and -O3 enabled, use the per-core tuning if
5433 set, otherwise use tables for > 16 cases as a tradeoff between size and
5434 performance. When optimizing for size, use the default setting. */
5436 static unsigned int
5438 {
5439 /* Use the specified limit for the number of cases before using jump
5440 tables at higher optimization levels. */
5444 else
5446 }
5448 /* Return true if register REGNO is a valid index register.
5449 STRICT_P is true if REG_OK_STRICT is in effect. */
5451 bool
5453 {
5455 {
5463 }
5465 }
5467 /* Return true if register REGNO is a valid base register for mode MODE.
5468 STRICT_P is true if REG_OK_STRICT is in effect. */
5470 bool
5472 {
5474 {
5482 }
5484 /* The fake registers will be eliminated to either the stack or
5485 hard frame pointer, both of which are usually valid base registers.
5486 Reload deals with the cases where the eliminated form isn't valid. */
5491 }
5493 /* Return true if X is a valid base register for mode MODE.
5494 STRICT_P is true if REG_OK_STRICT is in effect. */
5496 static bool
5498 {
5505 }
5507 /* Return true if address offset is a valid index. If it is, fill in INFO
5508 appropriately. STRICT_P is true if REG_OK_STRICT is in effect. */
5510 static bool
5513 {
5515 rtx index;
5518 /* (reg:P) */
5521 {
5525 }
5526 /* (sign_extend:DI (reg:SI)) */
5531 {
5536 }
5537 /* (mult:DI (sign_extend:DI (reg:SI)) (const_int scale)) */
5544 {
5549 }
5550 /* (ashift:DI (sign_extend:DI (reg:SI)) (const_int shift)) */
5557 {
5562 }
5563 /* (sign_extract:DI (mult:DI (reg:DI) (const_int scale)) 32+shift 0) */
5570 {
5578 }
5579 /* (and:DI (mult:DI (reg:DI) (const_int scale))
5580 (const_int 0xffffffff<<shift)) */
5587 {
5593 }
5594 /* (sign_extract:DI (ashift:DI (reg:DI) (const_int shift)) 32+shift 0) */
5601 {
5609 }
5610 /* (and:DI (ashift:DI (reg:DI) (const_int shift))
5611 (const_int 0xffffffff<<shift)) */
5618 {
5624 }
5625 /* (mult:P (reg:P) (const_int scale)) */
5630 {
5634 }
5635 /* (ashift:P (reg:P) (const_int shift)) */
5640 {
5644 }
5645 else
5654 {
5658 }
5659 else
5660 {
5665 }
5669 {
5674 }
5677 }
5679 /* Return true if MODE is one of the modes for which we
5680 support LDP/STP operations. */
5682 static bool
5684 {
5692 }
5694 /* Return true if REGNO is a virtual pointer register, or an eliminable
5695 "soft" frame register. Like REGNO_PTR_FRAME_P except that we don't
5696 include stack_pointer or hard_frame_pointer. */
5697 static bool
5699 {
5704 }
5706 /* Return true if X is a valid address of type TYPE for machine mode MODE.
5707 If it is, fill in INFO appropriately. STRICT_P is true if
5708 REG_OK_STRICT is in effect. */
5710 bool
5713 aarch64_addr_query_type type)
5714 {
5717 poly_int64 offset;
5719 HOST_WIDE_INT const_size;
5721 /* On BE, we use load/store pair for all large int mode load/stores.
5722 TI/TFmode may also use a load/store pair. */
5731 /* If we are dealing with ADDR_QUERY_LDP_STP_N that means the incoming mode
5732 corresponds to the actual size of the memory being loaded/stored and the
5733 mode of the corresponding addressing mode is half of that. */
5743 /* For SVE, only accept [Rn], [Rn, Rm, LSL #shift] and
5744 [Rn, #offset, MUL VL]. */
5749 /* On LE, for AdvSIMD, don't support anything other than POST_INC or
5750 REG addressing. */
5752 && !BYTES_BIG_ENDIAN
5760 {
5777 {
5784 }
5789 {
5795 /* TImode and TFmode values are allowed in both pairs of X
5796 registers and individual Q registers. The available
5797 address modes are:
5798 X,X: 7-bit signed scaled offset
5799 Q: 9-bit signed offset
5800 We conservatively require an offset representable in either mode.
5801 When performing the check for pairs of X registers i.e. LDP/STP
5802 pass down DImode since that is the natural size of the LDP/STP
5803 instruction memory accesses. */
5809 /* A 7bit offset check because OImode will emit a ldp/stp
5810 instruction (only big endian will get here).
5811 For ldp/stp instructions, the offset is scaled for the size of a
5812 single element of the pair. */
5816 /* Three 9/12 bit offsets checks because CImode will emit three
5817 ldr/str instructions (only big endian will get here). */
5825 /* Two 7bit offsets checks because XImode will emit two ldp/stp
5826 instructions (only big endian will get here). */
5832 /* Make "m" use the LD1 offset range for SVE data modes, so
5833 that pre-RTL optimizers like ivopts will work to that
5834 instead of the wider LDR/STR range. */
5841 {
5842 poly_int64 end_offset = (offset
5849 end_offset)));
5850 }
5860 else
5863 }
5866 {
5867 /* Look for base + (scaled/extended) index register. */
5870 {
5873 }
5876 {
5879 }
5880 }
5901 {
5905 /* TImode and TFmode values are allowed in both pairs of X
5906 registers and individual Q registers. The available
5907 address modes are:
5908 X,X: 7-bit signed scaled offset
5909 Q: 9-bit signed offset
5910 We conservatively require an offset representable in either mode.
5911 */
5921 else
5923 }
5929 /* load literal: pc-relative constant pool entry. Only supported
5930 for SI mode or larger. */
5936 {
5944 }
5953 {
5959 {
5960 /* The symbol and offset must be aligned to the access size. */
5966 {
5970 }
5976 else
5985 }
5986 }
5991 }
5992 }
5994 /* Return true if the address X is valid for a PRFM instruction.
5995 STRICT_P is true if we should do strict checking with
5996 aarch64_classify_address. */
5998 bool
6000 {
6003 /* PRFM accepts the same addresses as DImode... */
6008 /* ... except writeback forms. */
6010 }
6012 bool
6014 {
6015 rtx offset;
6019 }
6021 /* Classify the base of symbolic expression X. */
6023 enum aarch64_symbol_type
6025 {
6026 rtx offset;
6030 }
6033 /* Return TRUE if X is a legitimate address for accessing memory in
6034 mode MODE. */
6035 static bool
6037 {
6041 }
6043 /* Return TRUE if X is a legitimate address of type TYPE for accessing
6044 memory in mode MODE. STRICT_P is true if REG_OK_STRICT is in effect. */
6045 bool
6047 aarch64_addr_query_type type)
6048 {
6052 }
6054 /* Implement TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT. */
6056 static bool
6058 poly_int64 orig_offset,
6059 machine_mode mode)
6060 {
6061 HOST_WIDE_INT size;
6063 {
6066 /* A general SVE offset is A * VQ + B. Remove the A component from
6067 coefficient 0 in order to get the constant B. */
6070 /* Split an out-of-range address displacement into a base and
6071 offset. Use 4KB range for 1- and 2-byte accesses and a 16KB
6072 range otherwise to increase opportunities for sharing the base
6073 address of different sizes. Unaligned accesses use the signed
6074 9-bit range, TImode/TFmode use the intersection of signed
6075 scaled 7-bit and signed 9-bit offset. */
6080 else
6086 /* Split the offset into second_offset and the rest. */
6090 }
6091 else
6092 {
6093 /* Get the mode we should use as the basis of the range. For structure
6094 modes this is the mode of one vector. */
6096 machine_mode step_mode
6099 /* Get the "mul vl" multiplier we'd like to use. */
6103 /* LDR supports a 9-bit range, but the move patterns for
6104 structure modes require all vectors to be in range of the
6105 same base. The simplest way of accomodating that while still
6106 promoting reuse of anchor points between different modes is
6107 to use an 8-bit range unconditionally. */
6109 else
6110 /* Predicates are only handled singly, so we might as well use
6111 the full range. */
6116 /* Convert the "mul vl" multiplier into a byte offset. */
6121 /* Split the offset into second_offset and the rest. */
6125 }
6126 }
6128 /* Return the binary representation of floating point constant VALUE in INTVAL.
6129 If the value cannot be converted, return false without setting INTVAL.
6130 The conversion is done in the given MODE. */
6131 bool
6133 {
6135 /* We make a general exception for 0. */
6137 {
6140 }
6142 scalar_float_mode mode;
6146 /* Only support up to DF mode. */
6158 {
6162 }
6163 else
6168 }
6170 /* Return TRUE if rtx X is an immediate constant that can be moved using a
6171 single MOV(+MOVK) followed by an FMOV. */
6172 bool
6174 {
6179 /* Determine whether it's cheaper to write float constants as
6180 mov/movk pairs over ldr/adrp pairs. */
6186 {
6188 ? SImode
6193 }
6196 }
6198 /* Return TRUE if rtx X is immediate constant 0.0 */
6199 bool
6201 {
6208 }
6210 /* Return TRUE if rtx X is immediate constant that fits in a single
6211 MOVI immediate operation. */
6212 bool
6214 {
6218 machine_mode vmode;
6219 scalar_int_mode imode;
6224 {
6228 /* We make a general exception for 0. */
6233 }
6237 else
6240 /* use a 64 bit mode for everything except for DI/DF mode, where we use
6241 a 128 bit vector mode. */
6248 }
6251 /* Return the fixed registers used for condition codes. */
6253 static bool
6255 {
6259 }
6261 /* This function is used by the call expanders of the machine description.
6262 RESULT is the register in which the result is returned. It's NULL for
6263 "call" and "sibcall".
6264 MEM is the location of the function call.
6265 SIBCALL indicates whether this function call is normal call or sibling call.
6266 It will generate different pattern accordingly. */
6268 void
6270 {
6272 rtvec vec;
6273 machine_mode mode;
6280 /* Decide if we should generate indirect calls by loading the
6281 address of the callee into a register before performing
6282 the branch-and-link. */
6296 else
6303 }
6305 /* Emit call insn with PAT and do aarch64-specific handling. */
6307 void
6309 {
6315 }
6317 machine_mode
6319 {
6320 /* All floating point compares return CCFP if it is an equality
6321 comparison, and CCFPE otherwise. */
6323 {
6325 {
6346 }
6347 }
6349 /* Equality comparisons of short modes against zero can be performed
6350 using the TST instruction with the appropriate bitmask. */
6356 /* Similarly, comparisons of zero_extends from shorter modes can
6357 be performed using an ANDS with an immediate mask. */
6373 /* A compare with a shifted operand. Because of canonicalization,
6374 the comparison will have to be swapped when we emit the assembly
6375 code. */
6383 /* Similarly for a negated operand, but we can only do this for
6384 equalities. */
6391 /* A test for unsigned overflow. */
6398 /* A test for signed overflow. */
6405 /* For everything else, return CCmode. */
6407 }
6409 static int
6412 int
6414 {
6421 }
6423 static int
6425 {
6427 {
6431 {
6445 }
6450 {
6462 }
6467 {
6479 }
6484 {
6490 }
6495 {
6499 }
6504 {
6508 }
6513 {
6517 }
6522 }
6525 }
6527 bool
6529 HOST_WIDE_INT minval,
6530 HOST_WIDE_INT maxval)
6531 {
6532 rtx elt;
6536 }
6538 bool
6540 {
6542 }
6544 /* Return true if VEC is a constant in which every element is in the range
6545 [MINVAL, MAXVAL]. The elements do not need to have the same value. */
6547 static bool
6549 HOST_WIDE_INT minval,
6550 HOST_WIDE_INT maxval)
6551 {
6563 {
6568 }
6570 }
6572 /* N Z C V. */
6573 #define AARCH64_CC_V 1
6574 #define AARCH64_CC_C (1 << 1)
6575 #define AARCH64_CC_Z (1 << 2)
6576 #define AARCH64_CC_N (1 << 3)
6578 /* N Z C V flags for ccmp. Indexed by AARCH64_COND_CODE. */
6580 {
6597 };
6599 /* Print floating-point vector immediate operand X to F, negating it
6600 first if NEGATE is true. Return true on success, false if it isn't
6601 a constant we can handle. */
6603 static bool
6605 {
6606 rtx elt;
6615 /* We only handle the SVE single-bit immediates here. */
6622 else
6626 }
6628 /* Return the equivalent letter for size. */
6629 static char
6631 {
6633 {
6639 }
6640 }
6642 /* Print operand X to file F in a target specific manner according to CODE.
6643 The acceptable formatting commands given by CODE are:
6644 'c': An integer or symbol address without a preceding #
6645 sign.
6646 'C': Take the duplicated element in a vector constant
6647 and print it in hex.
6648 'D': Take the duplicated element in a vector constant
6649 and print it as an unsigned integer, in decimal.
6650 'e': Print the sign/zero-extend size as a character 8->b,
6651 16->h, 32->w.
6652 'p': Prints N such that 2^N == X (X must be power of 2 and
6653 const int).
6654 'P': Print the number of non-zero bits in X (a const_int).
6655 'H': Print the higher numbered register of a pair (TImode)
6656 of regs.
6657 'm': Print a condition (eq, ne, etc).
6658 'M': Same as 'm', but invert condition.
6659 'N': Take the duplicated element in a vector constant
6660 and print the negative of it in decimal.
6661 'b/h/s/d/q': Print a scalar FP/SIMD register name.
6662 'S/T/U/V': Print a FP/SIMD register name for a register list.
6663 The register printed is the FP/SIMD register name
6664 of X + 0/1/2/3 for S/T/U/V.
6665 'R': Print a scalar FP/SIMD register name + 1.
6666 'X': Print bottom 16 bits of integer constant in hex.
6667 'w/x': Print a general register name or the zero register
6668 (32-bit or 64-bit).
6669 '0': Print a normal operand, if it's a general register,
6670 then we assume DImode.
6671 'k': Print NZCV for conditional compare instructions.
6672 'A': Output address constant representing the first
6673 argument of X, specifying a relocation offset
6674 if appropriate.
6675 'L': Output constant address specified by X
6676 with a relocation offset if appropriate.
6677 'G': Prints address of X, specifying a PC relative
6678 relocation mode if appropriate.
6679 'y': Output address of LDP or STP - this is used for
6680 some LDP/STPs which don't use a PARALLEL in their
6681 pattern (so the mode needs to be adjusted).
6682 'z': Output address of a typical LDP or STP. */
6684 static void
6686 {
6687 rtx elt;
6689 {
6692 {
6704 {
6707 }
6708 /* Fall through. */
6712 }
6716 {
6721 {
6724 }
6727 {
6740 }
6741 }
6745 {
6749 {
6752 }
6755 }
6760 {
6763 }
6770 {
6773 }
6776 {
6779 }
6786 {
6788 /* CONST_TRUE_RTX means al/nv (al is the default, don't print it). */
6790 {
6794 }
6797 {
6800 }
6807 }
6812 {
6815 }
6821 ;
6822 else
6823 {
6826 }
6835 {
6836 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6838 }
6847 {
6848 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6850 }
6858 {
6859 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6861 }
6867 {
6870 }
6875 {
6876 /* Print a replicated constant in hex. */
6878 {
6881 }
6884 }
6888 {
6889 /* Print a replicated constant in decimal, treating it as
6890 unsigned. */
6892 {
6895 }
6898 }
6905 {
6908 }
6911 {
6914 }
6917 {
6920 }
6922 /* Fall through */
6926 {
6929 }
6932 {
6935 {
6938 else
6939 {
6940 char suffix
6945 }
6946 }
6947 else
6966 {
6969 }
6970 /* fall through */
6974 {
6977 }
6983 ;
6984 else
6985 {
6988 }
6992 /* Since we define TARGET_SUPPORTS_WIDE_INT we shouldn't ever
6993 be getting CONST_DOUBLEs holding integers. */
6996 {
6999 }
7001 {
7002 #define buf_size 20
7010 #undef buf_size
7011 }
7017 }
7025 {
7052 }
7058 {
7093 }
7099 {
7105 }
7110 {
7111 HOST_WIDE_INT cond_code;
7114 {
7117 }
7122 }
7127 {
7132 {
7135 }
7139 ? ADDR_QUERY_LDP_STP_N
7142 }
7148 }
7149 }
7151 /* Print address 'x' of a memory access with mode 'mode'.
7152 'op' is the context required by aarch64_classify_address. It can either be
7153 MEM for a normal memory access or PARALLEL for LDP/STP. */
7154 static bool
7156 aarch64_addr_query_type type)
7157 {
7161 /* Check all addresses are Pmode - including ILP32. */
7167 {
7172 {
7173 HOST_WIDE_INT vnum
7178 }
7180 {
7181 HOST_WIDE_INT vnum
7186 }
7187 else
7196 else
7205 else
7214 else
7220 /* Writeback is only supported for fixed-width modes. */
7223 {
7246 }
7258 }
7261 }
7263 /* Print address 'x' of a memory access with mode 'mode'. */
7264 static void
7266 {
7269 }
7271 bool
7273 {
7280 /* UNSPEC_TLS entries for a symbol include a LABEL_REF for the
7281 referencing instruction, but they are constant offsets, not
7282 symbols. */
7288 {
7290 {
7296 }
7299 }
7302 }
7304 /* Implement REGNO_REG_CLASS. */
7306 enum reg_class
7308 {
7326 }
7328 /* OFFSET is an address offset for mode MODE, which has SIZE bytes.
7329 If OFFSET is out of range, return an offset of an anchor point
7330 that is in range. Return 0 otherwise. */
7332 static HOST_WIDE_INT
7334 machine_mode mode)
7335 {
7336 /* Does it look like we'll need a 16-byte load/store-pair operation? */
7340 /* For offsets that aren't a multiple of the access size, the limit is
7341 -256...255. */
7343 {
7344 /* BLKmode typically uses LDP of X-registers. */
7348 }
7350 /* Small negative offsets are supported. */
7357 /* Use 12-bit offset by access size. */
7359 }
7361 static rtx
7363 {
7364 /* Try to split X+CONST into Y=X+(CONST & ~mask), Y+(CONST&mask),
7365 where mask is selected by alignment and size of the offset.
7366 We try to pick as large a range for the offset as possible to
7367 maximize the chance of a CSE. However, for aligned addresses
7368 we limit the range to 4k so that structures with different sized
7369 elements are likely to use the same base. We need to be careful
7370 not to split a CONST for some forms of address expression, otherwise
7371 it will generate sub-optimal code. */
7374 {
7380 {
7384 /* Force any scaling into a temp for CSE. */
7388 /* Let the pointer register be in op0. */
7392 /* If the pointer is virtual or frame related, then we know that
7393 virtual register instantiation or register elimination is going
7394 to apply a second constant. We want the two constants folded
7395 together easily. Therefore, emit as (OP0 + CONST) + OP1. */
7397 {
7401 }
7403 /* Otherwise, in order to encourage CSE (and thence loop strength
7404 reduce) scaled addresses, emit as (OP0 + OP1) + CONST. */
7408 }
7410 HOST_WIDE_INT size;
7412 {
7414 mode);
7416 {
7420 }
7421 }
7422 }
7425 }
7427 static reg_class_t
7429 reg_class_t rclass,
7430 machine_mode mode,
7432 {
7433 /* Use aarch64_sve_reload_be for SVE reloads that cannot be handled
7434 directly by the *aarch64_sve_mov<mode>_be move pattern. See the
7435 comment at the head of aarch64-sve.md for more details about the
7436 big-endian handling. */
7442 {
7445 }
7447 /* If we have to disable direct literal pool loads and stores because the
7448 function is too big, then we need a scratch register. */
7453 {
7456 }
7458 /* Without the TARGET_SIMD instructions we cannot move a Q register
7459 to a Q register directly. We need a scratch. */
7463 {
7466 }
7468 /* A TFmode or TImode memory access should be handled via an FP_REGS
7469 because AArch64 has richer addressing modes for LDR/STR instructions
7470 than LDP/STP instructions. */
7479 }
7481 static bool
7483 {
7486 /* If we need a frame pointer, ARG_POINTER_REGNUM and FRAME_POINTER_REGNUM
7487 can only eliminate to HARD_FRAME_POINTER_REGNUM. */
7491 }
7493 poly_int64
7495 {
7497 {
7504 }
7507 {
7511 }
7514 }
7516 /* Implement RETURN_ADDR_RTX. We do not support moving back to a
7517 previous frame. */
7519 rtx
7521 {
7525 }
7528 static void
7530 {
7532 {
7535 }
7536 else
7537 {
7540 }
7545 }
7547 static void
7549 {
7553 /* Don't need to copy the trailing D-words, we fill those in below. */
7565 /* XXX We should really define a "clear_cache" pattern and use
7566 gen_clear_cache(). */
7571 ptr_mode);
7572 }
7574 static unsigned char
7576 {
7577 /* ??? Logically we should only need to provide a value when
7578 HARD_REGNO_MODE_OK says that at least one register in REGCLASS
7579 can hold MODE, but at the moment we need to handle all modes.
7580 Just ignore any runtime parts for registers that can't store them. */
7584 {
7610 }
7612 }
7614 static reg_class_t
7616 {
7621 {
7627 }
7629 /* Register eliminiation can result in a request for
7630 SP+constant->FP_REGS. We cannot support such operations which
7631 use SP as source and an FP_REG as destination, so reject out
7632 right now. */
7634 {
7637 /* Look through a possible SUBREG introduced by ILP32. */
7643 POINTER_REGS));
7645 }
7648 }
7650 void
7652 {
7654 }
7656 static void
7658 {
7661 else
7662 {
7664 /* While priority is known to be in range [0, 65535], so 18 bytes
7665 would be enough, the compiler might not know that. To avoid
7666 -Wformat-truncation false positive, use a larger size. */
7673 }
7674 }
7676 static void
7678 {
7681 else
7682 {
7684 /* While priority is known to be in range [0, 65535], so 18 bytes
7685 would be enough, the compiler might not know that. To avoid
7686 -Wformat-truncation false positive, use a larger size. */
7693 }
7694 }
7698 {
7704 {
7705 {
7708 },
7709 {
7712 },
7713 {
7716 },
7717 /* We assume that DImode is only generated when not optimizing and
7718 that we don't really need 64-bit address offsets. That would
7719 imply an object file with 8GB of code in a single function! */
7720 {
7723 }
7724 };
7733 /* Need to implement table size reduction, by chaning the code below. */
7743 }
7746 /* Return size in bits of an arithmetic operand which is shifted/scaled and
7747 masked such that it is suitable for a UXTB, UXTH, or UXTW extend
7748 operator. */
7750 int
7752 {
7754 {
7757 {
7761 }
7762 }
7764 }
7766 /* Constant pools are per function only when PC relative
7767 literal loads are true or we are in the large memory
7768 model. */
7772 {
7775 }
7777 static bool
7779 {
7780 /* We can't use blocks for constants when we're using a per-function
7781 constant pool. */
7783 }
7785 /* Select appropriate section for constants depending
7786 on where we place literal pools. */
7790 rtx x,
7792 {
7797 }
7799 /* Implement ASM_OUTPUT_POOL_EPILOGUE. */
7800 void
7802 HOST_WIDE_INT offset)
7803 {
7804 /* When using per-function literal pools, we must ensure that any code
7805 section is aligned to the minimal instruction length, lest we get
7806 errors from the assembler re "unaligned instructions". */
7809 }
7811 /* Costs. */
7813 /* Helper function for rtx cost calculation. Strip a shift expression
7814 from X. Returns the inner operand if successful, or the original
7815 expression on failure. */
7816 static rtx
7818 {
7821 /* We accept both ROTATERT and ROTATE: since the RHS must be a constant
7822 we can convert both to ROR during final output. */
7837 }
7839 /* Helper function for rtx cost calculation. Strip an extend
7840 expression from X. Returns the inner operand if successful, or the
7841 original expression on failure. We deal with a number of possible
7842 canonicalization variations here. If STRIP_SHIFT is true, then
7843 we can strip off a shift also. */
7844 static rtx
7846 {
7847 scalar_int_mode mode;
7853 /* Zero and sign extraction of a widened value. */
7861 /* It can also be represented (for zero-extend) as an AND with an
7862 immediate. */
7871 /* Now handle extended register, as this may also have an optional
7872 left shift by 1..4. */
7887 }
7889 /* Return true iff CODE is a shift supported in combination
7890 with arithmetic instructions. */
7892 static bool
7894 {
7896 }
7899 /* Return true iff X is a cheap shift without a sign extend. */
7901 static bool
7903 {
7929 }
7931 /* Helper function for rtx cost calculation. Calculate the cost of
7932 a MULT or ASHIFT, which may be part of a compound PLUS/MINUS rtx.
7933 Return the calculated cost of the expression, recursing manually in to
7934 operands where needed. */
7936 static int
7938 {
7954 /* Integer multiply/fma. */
7956 {
7957 /* The multiply will be canonicalized as a shift, cost it as such. */
7961 {
7965 {
7967 {
7968 /* If the shift is considered cheap,
7969 then don't add any cost. */
7971 ;
7973 /* ARITH + shift-by-register. */
7976 /* ARITH + extended register. We don't have a cost field
7977 for ARITH+EXTEND+SHIFT, so use extend_arith here. */
7979 else
7980 /* ARITH + shift-by-immediate. */
7982 }
7983 else
7984 /* LSL (immediate). */
7987 }
7988 /* Strip extends as we will have costed them in the case above. */
7995 }
7997 /* MNEG or [US]MNEGL. Extract the NEG operand and indicate that it's a
7998 compound and let the below cases handle it. After all, MNEG is a
7999 special-case alias of MSUB. */
8001 {
8004 }
8006 /* Integer multiplies or FMAs have zero/sign extending variants. */
8011 {
8016 {
8018 /* SMADDL/UMADDL/UMSUBL/SMSUBL. */
8020 else
8021 /* MUL/SMULL/UMULL. */
8023 }
8026 }
8028 /* This is either an integer multiply or a MADD. In both cases
8029 we want to recurse and cost the operands. */
8034 {
8036 /* MADD/MSUB. */
8038 else
8039 /* MUL. */
8041 }
8044 }
8045 else
8046 {
8048 {
8049 /* Floating-point FMA/FMUL can also support negations of the
8050 operands, unless the rounding mode is upward or downward in
8051 which case FNMUL is different than FMUL with operand negation. */
8055 {
8060 }
8063 /* FMADD/FNMADD/FNMSUB/FMSUB. */
8065 else
8066 /* FMUL/FNMUL. */
8068 }
8073 }
8074 }
8076 static int
8078 machine_mode mode,
8079 addr_space_t as ATTRIBUTE_UNUSED,
8081 {
8089 {
8091 {
8092 /* This is a CONST or SYMBOL ref which will be split
8093 in a different way depending on the code model in use.
8094 Cost it through the generic infrastructure. */
8096 /* Divide through by the cost of one instruction to
8097 bring it to the same units as the address costs. */
8099 /* The cost is then the cost of preparing the address,
8100 followed by an immediate (possibly 0) offset. */
8102 }
8103 else
8104 {
8105 /* This is most likely a jump table from a case
8106 statement. */
8108 }
8109 }
8112 {
8124 else
8143 }
8147 {
8148 /* For the sake of calculating the cost of the shifted register
8149 component, we can treat same sized modes in the same way. */
8156 else
8157 /* We can't tell, or this is a 128-bit vector. */
8159 }
8162 }
8164 /* Return the cost of a branch. If SPEED_P is true then the compiler is
8165 optimizing for speed. If PREDICTABLE_P is true then the branch is predicted
8166 to be taken. */
8168 int
8170 {
8171 /* When optimizing for speed, use the cost of unpredictable branches. */
8177 else
8179 }
8181 /* Return true if the RTX X in mode MODE is a zero or sign extract
8182 usable in an ADD or SUB (extended register) instruction. */
8183 static bool
8185 {
8186 /* Catch add with a sign extract.
8187 This is add_<optab><mode>_multp2. */
8190 {
8201 op1))
8202 {
8204 }
8205 }
8206 /* The simple case <ARITH>, XD, XN, XM, [us]xt.
8207 No shift. */
8213 }
8215 static bool
8217 {
8219 {
8231 }
8232 }
8234 /* Return true iff X is an rtx that will match an extr instruction
8235 i.e. as described in the *extr<mode>5_insn family of patterns.
8236 OP0 and OP1 will be set to the operands of the shifts involved
8237 on success and will be NULL_RTX otherwise. */
8239 static bool
8241 {
8243 scalar_int_mode mode;
8258 {
8259 /* Canonicalise locally to ashift in op0, lshiftrt in op1. */
8271 {
8275 }
8276 }
8279 }
8281 /* Calculate the cost of calculating (if_then_else (OP0) (OP1) (OP2)),
8282 storing it in *COST. Result is true if the total cost of the operation
8283 has now been calculated. */
8284 static bool
8286 {
8287 rtx inner;
8288 rtx comparator;
8292 {
8296 }
8297 else
8298 {
8302 }
8305 {
8306 /* Conditional branch. */
8309 else
8310 {
8312 {
8314 {
8315 /* TBZ/TBNZ/CBZ/CBNZ. */
8317 /* TBZ/TBNZ. */
8320 else
8321 /* CBZ/CBNZ. */
8325 }
8326 }
8328 {
8329 /* TBZ/TBNZ. */
8332 }
8333 }
8334 }
8336 {
8337 /* CCMP. */
8339 {
8340 /* Increase cost of CCMP reg, 0, imm, CC to prefer CMP reg, 0. */
8344 {
8351 else
8353 }
8355 }
8357 /* It's a conditional operation based on the status flags,
8358 so it must be some flavor of CSEL. */
8360 /* CSNEG, CSINV, and CSINC are handled for free as part of CSEL. */
8366 {
8367 /* CSEL with zero-extension (*cmovdi_insn_uxtw). */
8370 }
8375 }
8377 /* We don't know what this is, cost all operands. */
8379 }
8381 /* Check whether X is a bitfield operation of the form shift + extend that
8382 maps down to a UBFIZ/SBFIZ/UBFX/SBFX instruction. If so, return the
8383 operand to which the bitfield operation is applied. Otherwise return
8384 NULL_RTX. */
8386 static rtx
8388 {
8402 {
8420 }
8423 }
8425 /* Return true if the mask and a shift amount from an RTX of the form
8426 (x << SHFT_AMNT) & MASK are valid to combine into a UBFIZ instruction of
8427 mode MODE. See the *andim_ashift<mode>_bfiz pattern. */
8429 bool
8431 rtx shft_amnt)
8432 {
8437 }
8439 /* Calculate the cost of calculating X, storing it in *COST. Result
8440 is true if the total cost of the operation has now been calculated. */
8441 static bool
8444 {
8449 scalar_int_mode int_mode;
8451 /* By default, assume that everything has equivalent cost to the
8452 cheapest instruction. Any additional costs are applied as a delta
8453 above this default. */
8457 {
8459 /* The cost depends entirely on the operands to SET. */
8465 {
8468 {
8482 }
8491 /* Fall through. */
8493 /* The cost is one per vector-register copied. */
8495 {
8498 }
8499 /* const0_rtx is in general free, but we will use an
8500 instruction to set a register to 0. */
8502 {
8503 /* The cost is 1 per register copied. */
8506 }
8507 else
8508 /* Cost is just the cost of the RHS of the set. */
8514 /* Bit-field insertion. Strip any redundant widening of
8515 the RHS to meet the width of the target. */
8526 {
8527 /* MOV immediate is assumed to always be cheap. */
8529 }
8530 else
8531 {
8532 /* BFM. */
8536 }
8541 /* We can't make sense of this, assume default cost. */
8544 }
8548 /* If an instruction can incorporate a constant within the
8549 instruction, the instruction's expression avoids calling
8550 rtx_cost() on the constant. If rtx_cost() is called on a
8551 constant, then it is usually because the constant must be
8552 moved into a register by one or more instructions.
8554 The exception is constant 0, which can be expressed
8555 as XZR/WZR and is therefore free. The exception to this is
8556 if we have (set (reg) (const0_rtx)) in which case we must cost
8557 the move. However, we can catch that when we cost the SET, so
8558 we don't need to consider that here. */
8561 else
8562 {
8563 /* To an approximation, building any other constant is
8564 proportionally expensive to the number of instructions
8565 required to build that constant. This is true whether we
8566 are compiling for SPEED or otherwise. */
8571 }
8576 /* First determine number of instructions to do the move
8577 as an integer constant. */
8581 {
8587 ? SImode
8593 }
8596 {
8597 /* mov[df,sf]_aarch64. */
8599 /* FMOV (scalar immediate). */
8602 {
8603 /* This will be a load from memory. */
8606 else
8608 }
8609 else
8610 /* Otherwise this is +0.0. We get this using MOVI d0, #0
8611 or MOV v0.s[0], wzr - neither of which are modeled by the
8612 cost tables. Just use the default cost. */
8613 {
8614 }
8615 }
8621 {
8622 /* For loads we want the base cost of a load, plus an
8623 approximation for the additional cost of the addressing
8624 mode. */
8638 }
8646 {
8648 {
8649 /* FNEG. */
8651 }
8653 }
8656 {
8659 {
8660 /* CSETM. */
8663 }
8665 /* Cost this as SUB wzr, X. */
8669 }
8672 {
8673 /* Support (neg(fma...)) as a single instruction only if
8674 sign of zeros is unimportant. This matches the decision
8675 making in aarch64.md. */
8677 {
8678 /* FNMADD. */
8681 }
8683 {
8684 /* FNMUL. */
8687 }
8689 /* FNEG. */
8692 }
8699 {
8702 else
8704 }
8714 {
8718 }
8721 {
8722 /* TODO: A write to the CC flags possibly costs extra, this
8723 needs encoding in the cost tables. */
8726 /* ANDS. */
8728 {
8731 }
8734 {
8735 /* ADDS (and CMN alias). */
8738 }
8741 {
8742 /* SUBS. */
8745 }
8750 {
8751 /* COMPARE of ZERO_EXTRACT form of TST-immediate.
8752 Handle it here directly rather than going to cost_logic
8753 since we know the immediate generated for the TST is valid
8754 so we can avoid creating an intermediate rtx for it only
8755 for costing purposes. */
8762 }
8765 {
8766 /* CMN. */
8773 }
8775 /* CMP.
8777 Compare can freely swap the order of operands, and
8778 canonicalization puts the more complex operation first.
8779 But the integer MINUS logic expects the shift/extend
8780 operation in op1. */
8783 {
8786 }
8788 }
8791 {
8792 /* FCMP. */
8797 {
8799 /* FCMP supports constant 0.0 for no extra cost. */
8801 }
8803 }
8806 {
8807 /* Vector compare. */
8812 {
8813 /* Vector cm (eq|ge|gt|lt|le) supports constant 0.0 for no extra
8814 cost. */
8816 }
8818 }
8822 {
8826 cost_minus:
8829 /* Detect valid immediates. */
8835 {
8837 /* SUB(S) (immediate). */
8840 }
8842 /* Look for SUB (extended register). */
8845 {
8853 }
8857 /* Cost this as an FMA-alike operation. */
8861 {
8864 speed);
8866 }
8871 {
8873 {
8874 /* Vector SUB. */
8876 }
8878 {
8879 /* SUB(S). */
8881 }
8883 {
8884 /* FSUB. */
8886 }
8887 }
8889 }
8892 {
8893 rtx new_op0;
8898 cost_plus:
8901 {
8902 /* CSINC. */
8906 }
8911 {
8915 /* ADD (immediate). */
8918 }
8922 /* Look for ADD (extended register). */
8925 {
8933 }
8935 /* Strip any extend, leave shifts behind as we will
8936 cost them through mult_cost. */
8941 {
8943 speed);
8945 }
8950 {
8952 {
8953 /* Vector ADD. */
8955 }
8957 {
8958 /* ADD. */
8960 }
8962 {
8963 /* FADD. */
8965 }
8966 }
8968 }
8974 {
8977 else
8979 }
8984 {
8988 {
8991 else
8993 }
8995 }
8998 {
9005 }
9006 /* Fall through. */
9009 cost_logic:
9014 {
9018 }
9026 {
9027 /* This is a UBFM/SBFM. */
9032 }
9035 {
9037 {
9038 /* We have a mask + shift version of a UBFIZ
9039 i.e. the *andim_ashift<mode>_bfiz pattern. */
9043 {
9050 }
9052 {
9053 /* We possibly get the immediate for free, this is not
9054 modelled. */
9061 }
9062 }
9063 else
9064 {
9067 /* Handle ORN, EON, or BIC. */
9073 /* If we had a shift on op0 then this is a logical-shift-
9074 by-register/immediate operation. Otherwise, this is just
9075 a logical operation. */
9077 {
9079 {
9080 /* Shift by immediate. */
9083 else
9085 }
9086 else
9088 }
9090 /* In both cases we want to cost both operands. */
9097 }
9098 }
9106 {
9107 /* Vector NOT. */
9110 }
9112 /* MVN-shifted-reg. */
9114 {
9121 }
9122 /* EON can have two forms: (xor (not a) b) but also (not (xor a b)).
9123 Handle the second form here taking care that 'a' in the above can
9124 be a shift. */
9126 {
9135 {
9138 else
9140 }
9143 }
9144 /* MVN. */
9153 /* If a value is written in SI mode, then zero extended to DI
9154 mode, the operation will in general be free as a write to
9155 a 'w' register implicitly zeroes the upper bits of an 'x'
9156 register. However, if this is
9158 (set (reg) (zero_extend (reg)))
9160 we must cost the explicit register move. */
9164 {
9167 /* If OP_COST is non-zero, then the cost of the zero extend
9168 is effectively the cost of the inner operation. Otherwise
9169 we have a MOV instruction and we take the cost from the MOV
9170 itself. This is true independently of whether we are
9171 optimizing for space or time. */
9176 }
9178 {
9179 /* All loads can zero extend to any size for free. */
9182 }
9186 {
9191 }
9194 {
9196 {
9197 /* UMOV. */
9199 }
9200 else
9201 {
9202 /* We generate an AND instead of UXTB/UXTH. */
9204 }
9205 }
9210 {
9211 /* LDRSH. */
9213 {
9220 }
9222 }
9226 {
9231 }
9234 {
9237 else
9239 }
9247 {
9249 {
9251 {
9252 /* Vector shift (immediate). */
9254 }
9255 else
9256 {
9257 /* LSL (immediate), UBMF, UBFIZ and friends. These are all
9258 aliases. */
9260 }
9261 }
9263 /* We can incorporate zero/sign extend for free. */
9270 }
9271 else
9272 {
9274 {
9276 /* Vector shift (register). */
9278 }
9279 else
9280 {
9282 /* LSLV. */
9289 {
9291 /* We already demanded XEXP (op1, 0) to be REG_P, so
9292 don't recurse into it. */
9294 }
9295 }
9297 }
9307 {
9308 /* ASR (immediate) and friends. */
9310 {
9313 else
9315 }
9319 }
9320 else
9321 {
9323 {
9325 /* Vector shift (register). */
9327 }
9328 else
9329 {
9331 /* ASR (register) and friends. */
9338 {
9340 /* We already demanded XEXP (op1, 0) to be REG_P, so
9341 don't recurse into it. */
9343 }
9344 }
9346 }
9352 {
9353 /* LDR. */
9356 }
9359 {
9360 /* ADRP, followed by ADD. */
9364 }
9367 {
9368 /* ADR. */
9371 }
9374 {
9375 /* One extra load instruction, after accessing the GOT. */
9379 }
9384 /* ADRP/ADD (immediate). */
9391 /* UBFX/SBFX. */
9393 {
9396 else
9398 }
9400 /* We can trust that the immediates used will be correct (there
9401 are no by-register forms), so we need only cost op0. */
9407 /* aarch64_rtx_mult_cost always handles recursion to its
9408 operands. */
9412 /* We can expand signed mod by power of 2 using a NEGS, two parallel
9413 ANDs and a CSNEG. Assume here that CSNEG is the same as the cost of
9414 an unconditional negate. This case should only ever be reached through
9415 the set_smod_pow2_cheap check in expmed.c. */
9419 {
9420 /* We expand to 4 instructions. Reset the baseline. */
9428 }
9430 /* Fall-through. */
9433 {
9434 /* Slighly prefer UMOD over SMOD. */
9441 }
9448 {
9452 /* There is no integer SQRT, so only DIV and UDIV can get
9453 here. */
9455 /* Slighly prefer UDIV over SDIV. */
9457 else
9459 }
9485 {
9488 else
9490 }
9492 /* FMSUB, FNMADD, and FNMSUB are free. */
9499 /* aarch64_fnma4_elt_to_64v2df has the NEG as operand 1,
9500 and the by-element operand as operand 0. */
9504 /* Catch vector-by-element operations. The by-element operand can
9505 either be (vec_duplicate (vec_select (x))) or just
9506 (vec_select (x)), depending on whether we are multiplying by
9507 a vector or a scalar.
9509 Canonicalization is not very good in these cases, FMA4 will put the
9510 by-element operand as operand 0, FNMA4 will have it as operand 1. */
9521 /* If the remaining parameters are not registers,
9522 get the cost to put them into registers. */
9536 {
9538 {
9539 /*Vector truncate. */
9541 }
9542 else
9544 }
9549 {
9551 {
9552 /*Vector conversion. */
9554 }
9555 else
9557 }
9563 /* Strip the rounding part. They will all be implemented
9564 by the fcvt* family of instructions anyway. */
9566 {
9575 }
9578 {
9581 else
9583 }
9585 /* We can combine fmul by a power of 2 followed by a fcvt into a single
9586 fixed-point fcvt. */
9591 {
9595 }
9602 {
9603 /* ABS (vector). */
9606 }
9608 {
9611 /* FABD, which is analogous to FADD. */
9613 {
9620 }
9621 /* Simple FABS is analogous to FNEG. */
9624 }
9625 else
9626 {
9627 /* Integer ABS will either be split to
9628 two arithmetic instructions, or will be an ABS
9629 (scalar), which we don't model. */
9633 }
9639 {
9642 else
9643 {
9644 /* FMAXNM/FMINNM/FMAX/FMIN.
9645 TODO: This may not be accurate for all implementations, but
9646 we do not model this in the cost tables. */
9648 }
9649 }
9653 /* The floating point round to integer frint* instructions. */
9655 {
9660 }
9663 {
9668 }
9673 /* Decompose <su>muldi3_highpart. */
9675 mode == DImode
9676 /* (lshiftrt:TI */
9679 /* (mult:TI */
9681 /* (ANY_EXTEND:TI (reg:DI))
9682 (ANY_EXTEND:TI (reg:DI))) */
9689 /* (const_int 64) */
9692 {
9693 /* UMULH/SMULH. */
9701 }
9703 /* Fall through. */
9706 }
9714 }
9716 /* Wrapper around aarch64_rtx_costs, dumps the partial, or total cost
9717 calculated for X. This cost is stored in *COST. Returns true
9718 if the total cost of X was calculated. */
9719 static bool
9722 {
9727 {
9732 }
9735 }
9737 static int
9740 {
9746 /* Caller save and pointer regs are equivalent to GENERAL_REGS. */
9753 /* Moving between GPR and stack cost is the same as GP2GP. */
9758 /* To/From the stack register, we move via the gprs. */
9764 {
9765 /* 128-bit operations on general registers require 2 instructions. */
9773 /* When AdvSIMD instructions are disabled it is not possible to move
9774 a 128-bit value directly between Q registers. This is handled in
9775 secondary reload. A general register is used as a scratch to move
9776 the upper DI value and the lower DI value is moved directly,
9777 hence the cost is the sum of three moves. */
9782 }
9792 }
9794 static int
9796 reg_class_t rclass ATTRIBUTE_UNUSED,
9798 {
9800 }
9802 /* Return true if it is safe and beneficial to use the approximate rsqrt optabs
9803 to optimize 1.0/sqrt. */
9805 static bool
9807 {
9809 && flag_unsafe_math_optimizations
9813 }
9815 /* Function to decide when to use the approximate reciprocal square root
9816 builtin. */
9818 static tree
9820 {
9826 }
9828 /* Emit instruction sequence to compute either the approximate square root
9829 or its approximate reciprocal, depending on the flag RECP, and return
9830 whether the sequence was emitted or not. */
9832 bool
9834 {
9838 {
9841 }
9844 {
9851 || flag_trapping_math
9852 || !flag_unsafe_math_optimizations
9855 }
9856 else
9857 /* Caller assumes we cannot fail. */
9863 /* When calculating the approximate square root, compare the
9864 argument with 0.0 and create a mask. */
9870 /* Estimate the approximate reciprocal square root. */
9874 /* Iterate over the series twice for SF and thrice for DF. */
9877 /* Optionally iterate over the series once less for faster performance
9878 while sacrificing the accuracy. */
9881 iterations--;
9883 /* Iterate over the series to calculate the approximate reciprocal square
9884 root. */
9887 {
9895 }
9898 {
9899 /* Qualify the approximate reciprocal square root when the argument is
9900 0.0 by squashing the intermediary result to 0.0. */
9906 /* Calculate the approximate square root. */
9908 }
9910 /* Finalize the approximation. */
9914 }
9916 /* Emit the instruction sequence to compute the approximation for the division
9917 of NUM by DEN in QUO and return whether the sequence was emitted or not. */
9919 bool
9921 {
9932 || flag_trapping_math
9933 || !flag_unsafe_math_optimizations
9941 /* Estimate the approximate reciprocal. */
9945 /* Iterate over the series twice for SF and thrice for DF. */
9948 /* Optionally iterate over the series once less for faster performance,
9949 while sacrificing the accuracy. */
9951 iterations--;
9953 /* Iterate over the series to calculate the approximate reciprocal. */
9956 {
9961 }
9964 {
9965 /* As the approximate reciprocal of DEN is already calculated, only
9966 calculate the approximate division when NUM is not 1.0. */
9969 }
9971 /* Finalize the approximation. */
9974 }
9976 /* Return the number of instructions that can be issued per cycle. */
9977 static int
9979 {
9981 }
9983 static int
9985 {
9989 }
9992 /* Implement TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD as
9993 autopref_multipass_dfa_lookahead_guard from haifa-sched.c. It only
9994 has an effect if PARAM_SCHED_AUTOPREF_QUEUE_DEPTH > 0. */
9996 static int
9999 {
10001 }
10004 /* Vectorizer cost model target hooks. */
10006 /* Implement targetm.vectorize.builtin_vectorization_cost. */
10007 static int
10009 tree vectype,
10011 {
10020 {
10071 }
10072 }
10074 /* Implement targetm.vectorize.add_stmt_cost. */
10075 static unsigned
10079 {
10084 {
10089 /* Statements in an inner loop relative to the loop being
10090 vectorized are weighted more heavily. The value here is
10091 arbitrary and could potentially be improved with analysis. */
10097 }
10100 }
10104 /* Parse the TO_PARSE string and put the architecture struct that it
10105 selects into RES and the architectural features into ISA_FLAGS.
10106 Return an aarch64_parse_opt_result describing the parse result.
10107 If there is an error parsing, RES and ISA_FLAGS are left unchanged. */
10109 static enum aarch64_parse_opt_result
10112 {
10124 else
10131 /* Loop through the list of supported ARCHes to find a match. */
10133 {
10135 {
10139 {
10140 /* TO_PARSE string contains at least one extension. */
10141 enum aarch64_parse_opt_result ext_res
10146 }
10147 /* Extension parsing was successful. Confirm the result
10148 arch and ISA flags. */
10152 }
10153 }
10155 /* ARCH name not found in list. */
10157 }
10159 /* Parse the TO_PARSE string and put the result tuning in RES and the
10160 architecture flags in ISA_FLAGS. Return an aarch64_parse_opt_result
10161 describing the parse result. If there is an error parsing, RES and
10162 ISA_FLAGS are left unchanged. */
10164 static enum aarch64_parse_opt_result
10167 {
10179 else
10186 /* Loop through the list of supported CPUs to find a match. */
10188 {
10190 {
10195 {
10196 /* TO_PARSE string contains at least one extension. */
10197 enum aarch64_parse_opt_result ext_res
10202 }
10203 /* Extension parsing was successfull. Confirm the result
10204 cpu and ISA flags. */
10208 }
10209 }
10211 /* CPU name not found in list. */
10213 }
10215 /* Parse the TO_PARSE string and put the cpu it selects into RES.
10216 Return an aarch64_parse_opt_result describing the parse result.
10217 If the parsing fails the RES does not change. */
10219 static enum aarch64_parse_opt_result
10221 {
10227 /* Loop through the list of supported CPUs to find a match. */
10229 {
10231 {
10234 }
10235 }
10237 /* CPU name not found in list. */
10239 }
10241 /* Parse TOKEN, which has length LENGTH to see if it is an option
10242 described in FLAG. If it is, return the index bit for that fusion type.
10243 If not, error (printing OPTION_NAME) and return zero. */
10245 static unsigned int
10250 {
10252 {
10256 }
10260 }
10262 /* Parse OPTION which is a comma-separated list of flags to enable.
10263 FLAGS gives the list of flags we understand, INITIAL_STATE gives any
10264 default state we inherit from the CPU tuning structures. OPTION_NAME
10265 gives the top-level option we are parsing in the -moverride string,
10266 for use in error messages. */
10268 static unsigned int
10273 {
10280 {
10283 token_length,
10284 flags,
10285 option_name);
10286 /* If we find "none" (or, for simplicity's sake, an error) anywhere
10287 in the token stream, reset the supported operations. So:
10289 adrp+add.cmp+branch.none.adrp+add
10291 would have the result of turning on only adrp+add fusion. */
10297 }
10299 /* We ended with a comma, print something. */
10301 {
10304 }
10306 /* We still have one more token to parse. */
10309 token_length,
10310 flags,
10311 option_name);
10317 }
10319 /* Support for overriding instruction fusion. */
10321 static void
10324 {
10326 aarch64_fusible_pairs,
10329 }
10331 /* Support for overriding other tuning flags. */
10333 static void
10336 {
10337 tune->extra_tuning_flags
10339 aarch64_tuning_flags,
10342 }
10344 /* Parse TOKEN, which has length LENGTH to see if it is a tuning option
10345 we understand. If it is, extract the option string and handoff to
10346 the appropriate function. */
10348 void
10352 {
10358 {
10361 }
10363 /* Get the length of the option name. */
10365 /* Skip the '=' to get to the option string. */
10366 option_part++;
10369 {
10371 {
10374 }
10375 }
10379 }
10381 /* A checking mechanism for the implementation of the tls size. */
10383 static void
10385 {
10390 {
10392 /* Both the default and maximum TLS size allowed under tiny is 1M which
10393 needs two instructions to address, so we clamp the size to 24. */
10398 /* The maximum TLS size allowed under small is 4G. */
10403 /* The maximum TLS size allowed under large is 16E.
10404 FIXME: 16E should be 64bit, we only support 48bit offset now. */
10410 }
10413 }
10415 /* Parse STRING looking for options in the format:
10416 string :: option:string
10417 option :: name=substring
10418 name :: {a-z}
10419 substring :: defined by option. */
10421 static void
10424 {
10435 {
10437 /* Make this substring look like a string. */
10441 }
10443 /* One last option to parse. */
10446 }
10449 static void
10451 {
10452 /* PR 70044: We have to be careful about being called multiple times for the
10453 same function. This means all changes should be repeatable. */
10455 /* Set aarch64_use_frame_pointer based on -fno-omit-frame-pointer.
10456 Disable the frame pointer flag so the mid-end will not use a frame
10457 pointer in leaf functions in order to support -fomit-leaf-frame-pointer.
10458 Set x_flag_omit_frame_pointer to the special value 2 to differentiate
10459 between -fomit-frame-pointer (1) and -fno-omit-frame-pointer (2). */
10464 /* If not optimizing for size, set the default
10465 alignment to what the target wants. */
10467 {
10474 }
10476 /* We default to no pc-relative literal loads. */
10480 /* If -mpc-relative-literal-loads is set on the command line, this
10481 implies that the user asked for PC relative literal loads. */
10485 /* In the tiny memory model it makes no sense to disallow PC relative
10486 literal pool loads. */
10491 /* When enabling the lower precision Newton series for the square root, also
10492 enable it for the reciprocal square root, since the latter is an
10493 intermediary step for the former. */
10496 }
10498 /* 'Unpack' up the internal tuning structs and update the options
10499 in OPTS. The caller must have set up selected_tune and selected_arch
10500 as all the other target-specific codegen decisions are
10501 derived from them. */
10503 void
10505 {
10508 /* Make a copy of the tuning parameters attached to the core, which
10509 we may later overwrite. */
10517 /* This target defaults to strict volatile bitfields. */
10526 {
10538 }
10540 /* We don't mind passing in global_options_set here as we don't use
10541 the *options_set structs anyway. */
10543 queue_depth,
10547 /* Set up parameters to be used in prefetching algorithm. Do not
10548 override the defaults unless we are tuning for a core we have
10549 researched values for. */
10581 /* Use the alternative scheduling-pressure algorithm by default. */
10586 /* Enable sw prefetching at specified optimization level for
10587 CPUS that have prefetch. Lower optimization level threshold by 1
10588 when profiling is enabled. */
10603 }
10605 /* Print a hint with a suggestion for a core or architecture name that
10606 most closely resembles what the user passed in STR. ARCH is true if
10607 the user is asking for an architecture name. ARCH is false if the user
10608 is asking for a core name. */
10610 static void
10612 {
10618 #ifdef HAVE_LOCAL_CPU_DETECT
10619 /* Add also "native" as possible value. */
10622 #endif
10629 else
10633 }
10635 /* Print a hint with a suggestion for a core name that most closely resembles
10636 what the user passed in STR. */
10640 {
10642 }
10644 /* Print a hint with a suggestion for an architecture name that most closely
10645 resembles what the user passed in STR. */
10649 {
10651 }
10653 /* Validate a command-line -mcpu option. Parse the cpu and extensions (if any)
10654 specified in STR and throw errors if appropriate. Put the results if
10655 they are valid in RES and ISA_FLAGS. Return whether the option is
10656 valid. */
10658 static bool
10661 {
10662 enum aarch64_parse_opt_result parse_res
10669 {
10682 }
10685 }
10687 /* Validate a command-line -march option. Parse the arch and extensions
10688 (if any) specified in STR and throw errors if appropriate. Put the
10689 results, if they are valid, in RES and ISA_FLAGS. Return whether the
10690 option is valid. */
10692 static bool
10695 {
10696 enum aarch64_parse_opt_result parse_res
10703 {
10716 }
10719 }
10721 /* Validate a command-line -mtune option. Parse the cpu
10722 specified in STR and throw errors if appropriate. Put the
10723 result, if it is valid, in RES. Return whether the option is
10724 valid. */
10726 static bool
10728 {
10729 enum aarch64_parse_opt_result parse_res
10736 {
10746 }
10748 }
10750 /* Return the CPU corresponding to the enum CPU.
10751 If it doesn't specify a cpu, return the default. */
10755 {
10759 /* The & 0x3f is to extract the bottom 6 bits that encode the
10760 default cpu as selected by the --with-cpu GCC configure option
10761 in config.gcc.
10762 ???: The whole TARGET_CPU_DEFAULT and AARCH64_CPU_DEFAULT_FLAGS
10763 flags mechanism should be reworked to make it more sane. */
10765 }
10767 /* Return the architecture corresponding to the enum ARCH.
10768 If it doesn't specify a valid architecture, return the default. */
10772 {
10779 }
10781 /* Return the VG value associated with -msve-vector-bits= value VALUE. */
10783 static poly_uint16
10785 {
10786 /* For now generate vector-length agnostic code for -msve-vector-bits=128.
10787 This ensures we can clearly distinguish SVE and Advanced SIMD modes when
10788 deciding which .md file patterns to use and when deciding whether
10789 something is a legitimate address or constant. */
10792 else
10794 }
10796 /* Implement TARGET_OPTION_OVERRIDE. This is called once in the beginning
10797 and is used to parse the -m{cpu,tune,arch} strings and setup the initial
10798 tuning structs. In particular it must set selected_tune and
10799 aarch64_isa_flags that define the available ISA features and tuning
10800 decisions. It must also set selected_arch as this will be used to
10801 output the .arch asm tags for each function. */
10803 static void
10805 {
10818 /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
10819 If either of -march or -mtune is given, they override their
10820 respective component of -mcpu. */
10832 /* If the user did not specify a processor, choose the default
10833 one for them. This will be the CPU set during configuration using
10834 --with-cpu, otherwise it is "generic". */
10836 {
10838 {
10842 }
10843 else
10844 {
10845 /* Get default configure-time CPU. */
10848 }
10852 }
10853 /* If both -mcpu and -march are specified check that they are architecturally
10854 compatible, warn if they're not and prefer the -march ISA flags. */
10856 {
10858 {
10862 }
10867 }
10868 else
10869 {
10870 /* -mcpu but no -march. */
10877 }
10879 /* Set the arch as well as we will need it when outputing
10880 the .arch directive in assembly. */
10882 {
10885 }
10890 #ifndef HAVE_AS_MABI_OPTION
10891 /* The compiler may have been configured with 2.23.* binutils, which does
10892 not have support for ILP32. */
10895 #endif
10897 /* Convert -msve-vector-bits to a VG count. */
10903 /* Make sure we properly set up the explicit options. */
10914 /* Save these options as the default ones in case we push and pop them later
10915 while processing functions with potential target attributes. */
10916 target_option_default_node = target_option_current_node
10918 }
10920 /* Implement targetm.override_options_after_change. */
10922 static void
10924 {
10926 }
10930 {
10934 }
10936 void
10938 {
10940 }
10942 /* A checking mechanism for the implementation of the various code models. */
10943 static void
10945 {
10947 {
10949 {
10954 #ifdef HAVE_AS_SMALL_PIC_RELOCS
10956 ? AARCH64_CMODEL_SMALL_PIC
10958 #else
10960 #endif
10968 }
10969 }
10970 else
10972 }
10974 /* Implement TARGET_OPTION_SAVE. */
10976 static void
10978 {
10980 }
10982 /* Implements TARGET_OPTION_RESTORE. Restore the backend codegen decisions
10983 using the information saved in PTR. */
10985 static void
10987 {
10995 }
10997 /* Implement TARGET_OPTION_PRINT. */
10999 static void
11001 {
11012 }
11016 void
11018 {
11020 }
11022 /* Restore or save the TREE_TARGET_GLOBALS from or to NEW_TREE.
11023 Used by aarch64_set_current_function and aarch64_pragma_target_parse to
11024 make sure optab availability predicates are recomputed when necessary. */
11026 void
11028 {
11033 else
11035 }
11037 /* Implement TARGET_SET_CURRENT_FUNCTION. Unpack the codegen decisions
11038 like tuning and ISA features from the DECL_FUNCTION_SPECIFIC_TARGET
11039 of the function, if such exists. This function may be called multiple
11040 times on a single function so use aarch64_previous_fndecl to avoid
11041 setting up identical state. */
11043 static void
11045 {
11049 tree old_tree = (aarch64_previous_fndecl
11055 /* If current function has no attributes but the previous one did,
11056 use the default node. */
11060 /* If nothing to do, return. #pragma GCC reset or #pragma GCC pop to
11061 the default have been handled by aarch64_save_restore_target_globals from
11062 aarch64_pragma_target_parse. */
11068 /* First set the target options. */
11072 }
11074 /* Enum describing the various ways we can handle attributes.
11075 In many cases we can reuse the generic option handling machinery. */
11077 enum aarch64_attr_opt_type
11078 {
11082 aarch64_attr_custom /* Attribute requires a custom handling function. */
11083 };
11085 /* All the information needed to handle a target attribute.
11086 NAME is the name of the attribute.
11087 ATTR_TYPE specifies the type of behavior of the attribute as described
11088 in the definition of enum aarch64_attr_opt_type.
11089 ALLOW_NEG is true if the attribute supports a "no-" form.
11090 HANDLER is the function that takes the attribute string as an argument
11091 It is needed only when the ATTR_TYPE is aarch64_attr_custom.
11092 OPT_NUM is the enum specifying the option that the attribute modifies.
11093 This is needed for attributes that mirror the behavior of a command-line
11094 option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
11095 aarch64_attr_enum. */
11097 struct aarch64_attribute_info
11098 {
11104 };
11106 /* Handle the ARCH_STR argument to the arch= target attribute. */
11108 static bool
11110 {
11112 enum aarch64_parse_opt_result parse_res
11116 {
11121 }
11124 {
11137 }
11140 }
11142 /* Handle the argument CPU_STR to the cpu= target attribute. */
11144 static bool
11146 {
11148 enum aarch64_parse_opt_result parse_res
11152 {
11160 }
11163 {
11176 }
11179 }
11181 /* Handle the argument STR to the tune= target attribute. */
11183 static bool
11185 {
11187 enum aarch64_parse_opt_result parse_res
11191 {
11196 }
11199 {
11206 }
11209 }
11211 /* Parse an architecture extensions target attribute string specified in STR.
11212 For example "+fp+nosimd". Show any errors if needed. Return TRUE
11213 if successful. Update aarch64_isa_flags to reflect the ISA features
11214 modified. */
11216 static bool
11218 {
11222 /* We allow "+nothing" in the beginning to clear out all architectural
11223 features if the user wants to handpick specific features. */
11225 {
11228 }
11233 {
11236 }
11239 {
11250 }
11253 }
11255 /* The target attributes that we support. On top of these we also support just
11256 ISA extensions, like __attribute__ ((target ("+crc"))), but that case is
11257 handled explicitly in aarch64_process_one_target_attr. */
11260 {
11262 OPT_mgeneral_regs_only },
11264 OPT_mfix_cortex_a53_835769 },
11266 OPT_mfix_cortex_a53_843419 },
11270 OPT_momit_leaf_frame_pointer },
11273 OPT_march_ },
11276 OPT_mtune_ },
11278 OPT_msign_return_address_ },
11280 };
11282 /* Parse ARG_STR which contains the definition of one target attribute.
11283 Show appropriate errors if any or return true if the attribute is valid. */
11285 static bool
11287 {
11293 {
11296 }
11301 /* Skip leading whitespace. */
11303 str_to_check++;
11305 /* We have something like __attribute__ ((target ("+fp+nosimd"))).
11306 It is easier to detect and handle it explicitly here rather than going
11307 through the machinery for the rest of the target attributes in this
11308 function. */
11313 {
11316 }
11319 /* If we found opt=foo then terminate STR_TO_CHECK at the '='
11320 and point ARG to "foo". */
11322 {
11324 arg++;
11325 }
11329 {
11330 /* If the names don't match up, or the user has given an argument
11331 to an attribute that doesn't accept one, or didn't give an argument
11332 to an attribute that expects one, fail to match. */
11341 {
11344 }
11346 /* If the name matches but the attribute does not allow "no-" versions
11347 then we can't match. */
11349 {
11352 }
11355 {
11356 /* Has a custom handler registered.
11357 For example, cpu=, arch=, tune=. */
11364 /* Either set or unset a boolean option. */
11366 {
11374 }
11375 /* Set or unset a bit in the target_flags. aarch64_handle_option
11376 should know what mask to apply given the option number. */
11378 {
11380 /* We only need to specify the option number.
11381 aarch64_handle_option will know which mask to apply. */
11387 }
11388 /* Use the option setting machinery to set an option to an enum. */
11390 {
11397 {
11400 global_dc);
11401 }
11402 else
11403 {
11405 }
11407 }
11410 }
11411 }
11413 /* If we reached here we either have found an attribute and validated
11414 it or didn't match any. If we matched an attribute but its arguments
11415 were malformed we will have returned false already. */
11417 }
11419 /* Count how many times the character C appears in
11420 NULL-terminated string STR. */
11422 static unsigned int
11424 {
11427 {
11429 res++;
11431 str++;
11432 }
11435 }
11437 /* Parse the tree in ARGS that contains the target attribute information
11438 and update the global target options space. */
11440 bool
11442 {
11444 {
11445 do
11446 {
11449 {
11452 }
11457 }
11460 {
11463 }
11470 {
11473 }
11475 /* Used to catch empty spaces between commas i.e.
11476 attribute ((target ("attr1,,attr2"))). */
11479 /* Handle multiple target attributes separated by ','. */
11484 {
11485 num_attrs++;
11487 {
11490 }
11493 }
11496 {
11499 }
11502 }
11504 /* Implement TARGET_OPTION_VALID_ATTRIBUTE_P. This is used to
11505 process attribute ((target ("..."))). */
11507 static bool
11509 {
11512 tree old_optimize;
11516 /* If what we're processing is the current pragma string then the
11517 target option node is already stored in target_option_current_node
11518 by aarch64_pragma_target_parse in aarch64-c.c. Use that to avoid
11519 having to re-parse the string. This is especially useful to keep
11520 arm_neon.h compile times down since that header contains a lot
11521 of intrinsics enclosed in pragmas. */
11523 {
11526 }
11532 /* If the function changed the optimization levels as well as setting
11533 target options, start with the optimizations specified. */
11538 /* Save the current target options to restore at the end. */
11541 /* If fndecl already has some target attributes applied to it, unpack
11542 them so that we add this attribute on top of them, rather than
11543 overwriting them. */
11545 {
11551 }
11552 else
11558 /* Set up any additional state. */
11560 {
11562 /* Initialize SIMD builtins if we haven't already.
11563 Set current_target_pragma to NULL for the duration so that
11564 the builtin initialization code doesn't try to tag the functions
11565 being built with the attributes specified by any current pragma, thus
11566 going into an infinite recursion. */
11568 {
11573 }
11575 }
11576 else
11582 {
11587 }
11595 }
11597 /* Helper for aarch64_can_inline_p. In the case where CALLER and CALLEE are
11598 tri-bool options (yes, no, don't care) and the default value is
11599 DEF, determine whether to reject inlining. */
11601 static bool
11604 {
11605 /* If the callee doesn't care, always allow inlining. */
11609 /* If the caller doesn't care, always allow inlining. */
11613 /* Otherwise, allow inlining if either the callee and caller values
11614 agree, or if the callee is using the default value. */
11616 }
11618 /* Implement TARGET_CAN_INLINE_P. Decide whether it is valid
11619 to inline CALLEE into CALLER based on target-specific info.
11620 Make sure that the caller and callee have compatible architectural
11621 features. Then go through the other possible target attributes
11622 and see if they can block inlining. Try not to reject always_inline
11623 callees unless they are incompatible architecturally. */
11625 static bool
11627 {
11639 /* Callee's ISA flags should be a subset of the caller's. */
11644 /* Allow non-strict aligned functions inlining into strict
11645 aligned ones. */
11655 /* If the architectural features match up and the callee is always_inline
11656 then the other attributes don't matter. */
11668 /* Honour explicit requests to workaround errata. */
11681 /* If the user explicitly specified -momit-leaf-frame-pointer for the
11682 caller and calle and they don't match up, reject inlining. */
11689 /* If the callee has specific tuning overrides, respect them. */
11694 /* If the user specified tuning override strings for the
11695 caller and callee and they don't match up, reject inlining.
11696 We just do a string compare here, we don't analyze the meaning
11697 of the string, as it would be too costly for little gain. */
11705 }
11707 /* Return true if SYMBOL_REF X binds locally. */
11709 static bool
11711 {
11715 }
11717 /* Return true if SYMBOL_REF X is thread local */
11718 static bool
11720 {
11728 }
11730 /* Classify a TLS symbol into one of the TLS kinds. */
11731 enum aarch64_symbol_type
11733 {
11737 {
11744 {
11750 }
11761 else
11770 }
11771 }
11773 /* Return the correct method for accessing X + OFFSET, where X is either
11774 a SYMBOL_REF or LABEL_REF. */
11776 enum aarch64_symbol_type
11778 {
11780 {
11782 {
11797 }
11798 }
11801 {
11806 {
11808 /* When we retrieve symbol + offset address, we have to make sure
11809 the offset does not cause overflow of the final address. But
11810 we have no way of knowing the address of symbol at compile time
11811 so we can't accurately say if the distance between the PC and
11812 symbol + offset is outside the addressible range of +/-1M in the
11813 TINY code model. So we rely on images not being greater than
11814 1M and cap the offset at 1M and anything beyond 1M will have to
11815 be loaded using an alternative mechanism. Furthermore if the
11816 symbol is a weak reference to something that isn't known to
11817 resolve to a symbol in this module, then force to memory. */
11825 /* Same reasoning as the tiny code model, but the offset cap here is
11826 4G. */
11847 /* This is alright even in PIC code as the constant
11848 pool reference is always PC relative and within
11849 the same translation unit. */
11852 else
11857 }
11858 }
11860 /* By default push everything into the constant pool. */
11862 }
11864 bool
11866 {
11868 }
11870 bool
11872 {
11880 }
11882 /* Implement TARGET_LEGITIMATE_CONSTANT_P hook. Return true for constants
11883 that should be rematerialized rather than spilled. */
11885 static bool
11887 {
11888 /* Support CSE and rematerialization of common constants. */
11894 /* Do not allow vector struct mode constants for Advanced SIMD.
11895 We could support 0 and -1 easily, but they need support in
11896 aarch64-simd.md. */
11901 /* Only accept variable-length vector constants if they can be
11902 handled directly.
11904 ??? It would be possible to handle rematerialization of other
11905 constants via secondary reloads. */
11912 /* Accept polynomial constants that can be calculated by using the
11913 destination of a move as the sole temporary. Constants that
11914 require a second temporary cannot be rematerialized (they can't be
11915 forced to memory and also aren't legitimate constants). */
11916 poly_int64 offset;
11920 /* If an offset is being added to something else, we need to allow the
11921 base to be moved into the destination register, meaning that there
11922 are no free temporaries for the offset. */
11927 /* Do not allow const (plus (anchor_symbol, const_int)). */
11931 /* Treat symbols as constants. Avoid TLS symbols as they are complex,
11932 so spilling them is better than rematerialization. */
11936 /* Label references are always constant. */
11941 }
11943 rtx
11945 {
11951 /* Can return in any reg. */
11954 }
11956 /* On AAPCS systems, this is the "struct __va_list". */
11959 /* Implement TARGET_BUILD_BUILTIN_VA_LIST.
11960 Return the type to use as __builtin_va_list.
11962 AAPCS64 \S 7.1.4 requires that va_list be a typedef for a type defined as:
11964 struct __va_list
11965 {
11966 void *__stack;
11967 void *__gr_top;
11968 void *__vr_top;
11969 int __gr_offs;
11970 int __vr_offs;
11971 }; */
11973 static tree
11975 {
11976 tree va_list_name;
11979 /* Create the type. */
11981 /* Give it the required name. */
11983 TYPE_DECL,
11985 va_list_type);
11990 /* Create the fields. */
11993 ptr_type_node);
11996 ptr_type_node);
11999 ptr_type_node);
12002 integer_type_node);
12005 integer_type_node);
12007 /* Tell tree-stdarg pass about our internal offset fields.
12008 NOTE: va_list_gpr/fpr_counter_field are only used for tree comparision
12009 purpose to identify whether the code is updating va_list internal
12010 offset fields through irregular way. */
12032 /* Compute its layout. */
12036 }
12038 /* Implement TARGET_EXPAND_BUILTIN_VA_START. */
12039 static void
12041 {
12045 tree t;
12059 {
12062 }
12071 NULL_TREE);
12073 NULL_TREE);
12075 NULL_TREE);
12077 NULL_TREE);
12079 NULL_TREE);
12081 /* Emit code to initialize STACK, which points to the next varargs stack
12082 argument. CUM->AAPCS_STACK_SIZE gives the number of stack words used
12083 by named arguments. STACK is 8-byte aligned. */
12090 /* Emit code to initialize GRTOP, the top of the GR save area.
12091 virtual_incoming_args_rtx should have been 16 byte aligned. */
12096 /* Emit code to initialize VRTOP, the top of the VR save area.
12097 This address is gr_save_area_bytes below GRTOP, rounded
12098 down to the next 16-byte boundary. */
12108 /* Emit code to initialize GROFF, the offset from GRTOP of the
12109 next GPR argument. */
12114 /* Likewise emit code to initialize VROFF, the offset from FTOP
12115 of the next VR argument. */
12119 }
12121 /* Implement TARGET_GIMPLIFY_VA_ARG_EXPR. */
12123 static tree
12126 {
12127 tree addr;
12133 machine_mode mode;
12160 type,
12164 {
12165 /* No frontends can create types with variable-sized modes, so we
12166 shouldn't be asked to pass or return them. */
12169 /* TYPE passed in fp/simd registers. */
12181 {
12184 }
12187 {
12189 }
12190 }
12191 else
12192 {
12193 /* TYPE passed in general registers. */
12206 {
12208 }
12209 }
12211 /* Get a local temporary for the field value. */
12214 /* Emit code to branch if off >= 0. */
12220 {
12221 /* Emit: offs = (offs + 15) & -16. */
12227 }
12228 else
12231 /* Update ap.__[g|v]r_offs */
12236 /* String up. */
12240 /* [cond2] if (ap.__[g|v]r_offs > 0) */
12245 /* String up: make sure the assignment happens before the use. */
12249 /* Prepare the trees handling the argument that is passed on the stack;
12250 the top level node will store in ON_STACK. */
12253 {
12254 /* if (alignof(type) > 8) (arg = arg + 15) & -16; */
12259 }
12260 else
12262 /* Advance ap.__stack */
12267 /* String up roundup and advance. */
12270 /* String up with arg */
12272 /* Big-endianness related address adjustment. */
12275 {
12279 }
12284 /* Adjustment to OFFSET in the case of BIG_ENDIAN. */
12294 {
12295 /* type ha; // treat as "struct {ftype field[n];}"
12296 ... [computing offs]
12297 for (i = 0; i <nregs; ++i, offs += 16)
12298 ha.field[i] = *((ftype *)(ap.__vr_top + offs));
12299 return ha; */
12303 /* Declare a local variable. */
12307 /* Establish the base type. */
12309 {
12328 {
12332 }
12336 }
12338 /* *(field_ptr_t)&ha = *((field_ptr_t)vr_saved_area */
12346 /* ha.field[i] = *((field_ptr_t)vr_saved_area + i) */
12348 {
12358 }
12362 }
12372 }
12374 /* Implement TARGET_SETUP_INCOMING_VARARGS. */
12376 static void
12380 {
12382 CUMULATIVE_ARGS local_cum;
12386 /* The caller has advanced CUM up to, but not beyond, the last named
12387 argument. Advance a local copy of CUM past the last "real" named
12388 argument, to find out how many registers are left over. */
12392 /* Found out how many registers we need to save.
12393 Honor tree-stdvar analysis results. */
12402 {
12405 }
12408 {
12410 {
12413 /* virtual_incoming_args_rtx should have been 16-byte aligned. */
12421 }
12423 {
12424 /* We can't use move_block_from_reg, because it will use
12425 the wrong mode, storing D regs only. */
12429 /* Set OFF to the offset from virtual_incoming_args_rtx of
12430 the first vector register. The VR save area lies below
12431 the GR one, and is aligned to 16 bytes. */
12438 {
12446 }
12447 }
12448 }
12450 /* We don't save the size into *PRETEND_SIZE because we want to avoid
12451 any complication of having crtl->args.pretend_args_size changed. */
12456 }
12458 static void
12460 {
12463 {
12465 {
12468 }
12469 }
12472 {
12475 }
12477 /* When tracking speculation, we need a couple of call-clobbered registers
12478 to track the speculation state. It would be nice to just use
12479 IP0 and IP1, but currently there are numerous places that just
12480 assume these registers are free for other uses (eg pointer
12481 authentication). */
12483 {
12488 }
12489 }
12491 /* Walk down the type tree of TYPE counting consecutive base elements.
12492 If *MODEP is VOIDmode, then set it to the first valid floating point
12493 type. If a non-floating point type is found, or if a floating point
12494 type that doesn't match a non-VOIDmode *MODEP is found, then return -1,
12495 otherwise return the count in the sub-tree. */
12496 static int
12498 {
12499 machine_mode mode;
12500 HOST_WIDE_INT size;
12503 {
12533 /* Use V2SImode and V4SImode as representatives of all 64-bit
12534 and 128-bit vector types. */
12537 {
12546 }
12551 /* Vector modes are considered to be opaque: two vectors are
12552 equivalent for the purposes of being homogeneous aggregates
12553 if they are the same size. */
12560 {
12564 /* Can't handle incomplete types nor sizes that are not
12565 fixed. */
12572 || !index
12583 /* There must be no padding. */
12589 }
12592 {
12595 tree field;
12597 /* Can't handle incomplete types nor sizes that are not
12598 fixed. */
12604 {
12612 }
12614 /* There must be no padding. */
12620 }
12624 {
12625 /* These aren't very interesting except in a degenerate case. */
12628 tree field;
12630 /* Can't handle incomplete types nor sizes that are not
12631 fixed. */
12637 {
12645 }
12647 /* There must be no padding. */
12653 }
12657 }
12660 }
12662 /* Return TRUE if the type, as described by TYPE and MODE, is a short vector
12663 type as described in AAPCS64 \S 4.1.2.
12665 See the comment above aarch64_composite_type_p for the notes on MODE. */
12667 static bool
12669 machine_mode mode)
12670 {
12680 }
12682 /* Return TRUE if the type, as described by TYPE and MODE, is a composite
12683 type as described in AAPCS64 \S 4.3. This includes aggregate, union and
12684 array types. The C99 floating-point complex types are also considered
12685 as composite types, according to AAPCS64 \S 7.1.1. The complex integer
12686 types, which are GCC extensions and out of the scope of AAPCS64, are
12687 treated as composite types here as well.
12689 Note that MODE itself is not sufficient in determining whether a type
12690 is such a composite type or not. This is because
12691 stor-layout.c:compute_record_mode may have already changed the MODE
12692 (BLKmode) of a RECORD_TYPE TYPE to some other mode. For example, a
12693 structure with only one field may have its MODE set to the mode of the
12694 field. Also an integer mode whose size matches the size of the
12695 RECORD_TYPE type may be used to substitute the original mode
12696 (i.e. BLKmode) in certain circumstances. In other words, MODE cannot be
12697 solely relied on. */
12699 static bool
12701 machine_mode mode)
12702 {
12715 }
12717 /* Return TRUE if an argument, whose type is described by TYPE and MODE,
12718 shall be passed or returned in simd/fp register(s) (providing these
12719 parameter passing registers are available).
12721 Upon successful return, *COUNT returns the number of needed registers,
12722 *BASE_MODE returns the mode of the individual register and when IS_HAF
12723 is not NULL, *IS_HA indicates whether or not the argument is a homogeneous
12724 floating-point aggregate or a homogeneous short-vector aggregate. */
12726 static bool
12728 const_tree type,
12732 {
12740 {
12743 }
12745 {
12749 }
12751 {
12755 {
12758 }
12759 else
12761 }
12762 else
12767 }
12769 /* Implement TARGET_STRUCT_VALUE_RTX. */
12771 static rtx
12774 {
12776 }
12778 /* Implements target hook vector_mode_supported_p. */
12779 static bool
12781 {
12784 }
12786 /* Return appropriate SIMD container
12787 for MODE within a vector of WIDTH bits. */
12788 static machine_mode
12790 {
12793 {
12810 }
12814 {
12817 {
12834 }
12835 else
12837 {
12850 }
12851 }
12853 }
12855 /* Return 128-bit container as the preferred SIMD mode for MODE. */
12856 static machine_mode
12858 {
12861 }
12863 /* Return a list of possible vector sizes for the vectorizer
12864 to iterate over. */
12865 static void
12867 {
12872 }
12874 /* Implement TARGET_MANGLE_TYPE. */
12878 {
12879 /* The AArch64 ABI documents say that "__va_list" has to be
12880 managled as if it is in the "std" namespace. */
12884 /* Half-precision float. */
12888 /* Mangle AArch64-specific internal types. TYPE_NAME is non-NULL_TREE for
12889 builtin types. */
12893 /* Use the default mangling. */
12895 }
12897 /* Find the first rtx_insn before insn that will generate an assembly
12898 instruction. */
12902 {
12906 do
12907 {
12909 }
12913 }
12915 static bool
12917 {
12919 /* A number of these may be AArch32 only. */
12924 };
12927 {
12930 }
12933 }
12935 /* Check if there is a register dependency between a load and the insn
12936 for which we hold recog_data. */
12938 static bool
12940 {
12941 rtx load_reg;
12951 {
12957 }
12959 }
12962 /* When working around the Cortex-A53 erratum 835769,
12963 given rtx_insn INSN, return true if it is a 64-bit multiply-accumulate
12964 instruction and has a preceding memory instruction such that a NOP
12965 should be inserted between them. */
12967 bool
12969 {
12972 rtx body;
12985 /* aarch64_prev_real_insn can call recog_memoized on insns other than INSN.
12986 Restore recog state to INSN to avoid state corruption. */
12994 /* If the previous insn is a memory op and there is no dependency between
12995 it and the DImode madd, emit a NOP between them. If body is NULL then we
12996 have a complex memory operation, probably a load/store pair.
12997 Be conservative for now and emit a NOP. */
13004 }
13007 /* Implement FINAL_PRESCAN_INSN. */
13009 void
13011 {
13014 }
13017 /* Return true if BASE_OR_STEP is a valid immediate operand for an SVE INDEX
13018 instruction. */
13020 bool
13022 {
13025 }
13027 /* Return true if X is a valid immediate for the SVE ADD and SUB
13028 instructions. Negate X first if NEGATE_P is true. */
13030 bool
13032 {
13033 rtx elt;
13047 }
13049 /* Return true if X is a valid immediate operand for an SVE logical
13050 instruction such as AND. */
13052 bool
13054 {
13055 rtx elt;
13061 }
13063 /* Return true if X is a valid immediate for the SVE DUP and CPY
13064 instructions. */
13066 bool
13068 {
13069 rtx elt;
13079 }
13081 /* Return true if X is a valid immediate operand for an SVE CMP instruction.
13082 SIGNED_P says whether the operand is signed rather than unsigned. */
13084 bool
13086 {
13087 rtx elt;
13091 && (signed_p
13094 }
13096 /* Return true if X is a valid immediate operand for an SVE FADD or FSUB
13097 instruction. Negate X first if NEGATE_P is true. */
13099 bool
13101 {
13102 rtx elt;
13103 REAL_VALUE_TYPE r;
13119 }
13121 /* Return true if X is a valid immediate operand for an SVE FMUL
13122 instruction. */
13124 bool
13126 {
13127 rtx elt;
13129 /* GCC will never generate a multiply with an immediate of 2, so there is no
13130 point testing for it (even though it is a valid constant). */
13134 }
13136 /* Return true if replicating VAL32 is a valid 2-byte or 4-byte immediate
13137 for the Advanced SIMD operation described by WHICH and INSN. If INFO
13138 is nonnull, use it to describe valid immediates. */
13139 static bool
13144 {
13145 /* Try a 4-byte immediate with LSL. */
13148 {
13153 }
13155 /* Try a 2-byte immediate with LSL. */
13160 {
13165 }
13167 /* Try a 4-byte immediate with MSL, except for cases that MVN
13168 can handle. */
13171 {
13174 {
13179 }
13180 }
13183 }
13185 /* Return true if replicating VAL64 is a valid immediate for the
13186 Advanced SIMD operation described by WHICH. If INFO is nonnull,
13187 use it to describe valid immediates. */
13188 static bool
13192 {
13198 {
13209 /* Try using a replicated byte. */
13213 {
13217 }
13218 }
13220 /* Try using a bit-to-bytemask. */
13222 {
13225 {
13229 }
13231 {
13235 }
13236 }
13238 }
13240 /* Return true if replicating VAL64 gives a valid immediate for an SVE MOV
13241 instruction. If INFO is nonnull, use it to describe valid immediates. */
13243 static bool
13246 {
13250 {
13254 {
13259 }
13260 }
13263 {
13264 /* DUP with no shift. */
13268 }
13270 {
13271 /* DUP with LSL #8. */
13275 }
13277 {
13278 /* DUPM. */
13282 }
13284 }
13286 /* Return true if OP is a valid SIMD immediate for the operation
13287 described by WHICH. If INFO is nonnull, use it to describe valid
13288 immediates. */
13289 bool
13292 {
13306 {
13315 }
13319 else
13322 /* Handle PFALSE and PTRUE. */
13327 scalar_float_mode elt_float_mode;
13330 {
13334 {
13338 }
13339 }
13347 /* Expand the vector constant out into a byte vector, with the least
13348 significant byte of the register first. */
13352 {
13353 /* The vector is provided in gcc endian-neutral fashion.
13354 For aarch64_be Advanced SIMD, it must be laid out in the vector
13355 register in reverse order. */
13367 {
13370 }
13371 }
13373 /* The immediate must repeat every eight bytes. */
13379 /* Get the repeating 8-byte value as an integer. No endian correction
13380 is needed here because bytes is already in lsb-first order. */
13388 else
13390 }
13392 /* Check whether X is a VEC_SERIES-like constant that starts at 0 and
13393 has a step in the range of INDEX. Return the index expression if so,
13394 otherwise return null. */
13395 rtx
13397 {
13404 }
13406 /* Check of immediate shift constants are within range. */
13407 bool
13409 {
13413 else
13415 }
13417 /* Return the bitmask CONST_INT to select the bits required by a zero extract
13418 operation of width WIDTH at bit position POS. */
13420 rtx
13422 {
13426 unsigned HOST_WIDE_INT mask
13429 }
13431 bool
13433 {
13452 }
13454 /* Return a const_int vector of VAL. */
13455 rtx
13457 {
13460 }
13462 /* Check OP is a legal scalar immediate for the MOVI instruction. */
13464 bool
13466 {
13467 machine_mode vmode;
13472 }
13474 /* Construct and return a PARALLEL RTX vector with elements numbering the
13475 lanes of either the high (HIGH == TRUE) or low (HIGH == FALSE) half of
13476 the vector - from the perspective of the architecture. This does not
13477 line up with GCC's perspective on lane numbers, so we end up with
13478 different masks depending on our target endian-ness. The diagram
13479 below may help. We must draw the distinction when building masks
13480 which select one half of the vector. An instruction selecting
13481 architectural low-lanes for a big-endian target, must be described using
13482 a mask selecting GCC high-lanes.
13484 Big-Endian Little-Endian
13486 GCC 0 1 2 3 3 2 1 0
13487 | x | x | x | x | | x | x | x | x |
13488 Architecture 3 2 1 0 3 2 1 0
13490 Low Mask: { 2, 3 } { 0, 1 }
13491 High Mask: { 0, 1 } { 2, 3 }
13493 MODE Is the mode of the vector and NUNITS is the number of units in it. */
13495 rtx
13497 {
13502 rtx t1;
13507 else
13515 }
13517 /* Check OP for validity as a PARALLEL RTX vector with elements
13518 numbering the lanes of either the high (HIGH == TRUE) or low lanes,
13519 from the perspective of the architecture. See the diagram above
13520 aarch64_simd_vect_par_cnst_half for more details. */
13522 bool
13525 {
13539 {
13546 }
13548 }
13550 /* Bounds-check lanes. Ensure OPERAND lies between LOW (inclusive) and
13551 HIGH (exclusive). */
13552 void
13554 const_tree exp)
13555 {
13556 HOST_WIDE_INT lane;
13561 {
13564 else
13566 }
13567 }
13569 /* Peform endian correction on lane number N, which indexes a vector
13570 of mode MODE, and return the result as an SImode rtx. */
13572 rtx
13574 {
13576 }
13578 /* Return TRUE if OP is a valid vector addressing mode. */
13580 bool
13582 {
13585 }
13587 /* Return true if OP is a valid MEM operand for an SVE LD1R instruction. */
13589 bool
13591 {
13593 scalar_mode mode;
13600 }
13602 /* Return true if OP is a valid MEM operand for an SVE LDR instruction.
13603 The conditions for STR are the same. */
13604 bool
13606 {
13613 }
13615 /* Return true if OP is a valid MEM operand for an SVE_STRUCT mode.
13616 We need to be able to access the individual pieces, so the range
13617 is different from LD[234] and ST[234]. */
13618 bool
13620 {
13627 ADDR_QUERY_ANY)
13635 }
13637 /* Emit a register copy from operand to operand, taking care not to
13638 early-clobber source registers in the process.
13640 COUNT is the number of components into which the copy needs to be
13641 decomposed. */
13642 void
13645 {
13655 else
13659 }
13661 /* Compute and return the length of aarch64_simd_reglist<mode>, where <mode> is
13662 one of VSTRUCT modes: OI, CI, or XI. */
13663 int
13665 {
13666 /* This is only used (and only meaningful) for Advanced SIMD, not SVE. */
13668 }
13670 /* Implement target hook TARGET_VECTOR_ALIGNMENT. The AAPCS64 sets the maximum
13671 alignment of a vector to 128 bits. SVE predicates have an alignment of
13672 16 bits. */
13673 static HOST_WIDE_INT
13675 {
13677 /* ??? Checking the mode isn't ideal, but VECTOR_BOOLEAN_TYPE_P can
13678 be set for non-predicate vectors of booleans. Modes are the most
13679 direct way we have of identifying real SVE predicate types. */
13683 }
13685 /* Implement target hook TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT. */
13686 static HOST_WIDE_INT
13688 {
13690 {
13691 /* If the length of the vector is fixed, try to align to that length,
13692 otherwise don't try to align at all. */
13693 HOST_WIDE_INT result;
13697 }
13699 }
13701 /* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE. */
13702 static bool
13704 {
13708 /* For fixed-length vectors, check that the vectorizer will aim for
13709 full-vector alignment. This isn't true for generic GCC vectors
13710 that are wider than the ABI maximum of 128 bits. */
13716 /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned. */
13718 }
13720 /* Return true if the vector misalignment factor is supported by the
13721 target. */
13722 static bool
13726 {
13728 {
13729 /* Return if movmisalign pattern is not supported for this mode. */
13733 /* Misalignment factor is unknown at compile time. */
13736 }
13738 is_packed);
13739 }
13741 /* If VALS is a vector constant that can be loaded into a register
13742 using DUP, generate instructions to do so and return an RTX to
13743 assign to the register. Otherwise return NULL_RTX. */
13744 static rtx
13746 {
13749 rtx x;
13754 /* We can load this constant by using DUP and a constant in a
13755 single ARM register. This will be cheaper than a vector
13756 load. */
13759 }
13762 /* Generate code to load VALS, which is a PARALLEL containing only
13763 constants (for vec_init) or CONST_VECTOR, efficiently into a
13764 register. Returns an RTX to copy into the register, or NULL_RTX
13765 for a PARALLEL that can not be converted into a CONST_VECTOR. */
13766 static rtx
13768 {
13770 rtx const_dup;
13778 {
13779 /* A CONST_VECTOR must contain only CONST_INTs and
13780 CONST_DOUBLEs, but CONSTANT_P allows more (e.g. SYMBOL_REF).
13781 Only store valid constants in a CONST_VECTOR. */
13784 {
13787 n_const++;
13788 }
13791 }
13792 else
13797 /* Load using MOVI/MVNI. */
13800 /* Loaded using DUP. */
13803 /* Load from constant pool. We can not take advantage of single-cycle
13804 LD1 because we need a PC-relative addressing mode. */
13806 else
13807 /* A PARALLEL containing something not valid inside CONST_VECTOR.
13808 We can not construct an initializer. */
13810 }
13812 /* Expand a vector initialisation sequence, such that TARGET is
13813 initialised to contain VALS. */
13815 void
13817 {
13820 /* The number of vector elements. */
13822 /* The number of vector elements which are not constant. */
13825 /* The first element of vals. */
13829 /* Count the number of variable elements to initialise. */
13831 {
13835 else
13839 }
13841 /* No variable elements, hand off to aarch64_simd_make_constant which knows
13842 how best to handle this. */
13844 {
13847 {
13850 }
13851 }
13853 /* Splat a single non-constant element if we can. */
13855 {
13859 }
13864 /* If there are only variable elements, try to optimize
13865 the insertion using dup for the most common element
13866 followed by insertions. */
13868 /* The algorithm will fill matches[*][0] with the earliest matching element,
13869 and matches[X][1] with the count of duplicate elements (if X is the
13870 earliest element which has duplicates). */
13873 {
13876 {
13878 {
13880 {
13884 }
13885 }
13886 }
13891 {
13894 }
13896 /* Create a duplicate of the most common element, unless all elements
13897 are equally useless to us, in which case just immediately set the
13898 vector register using the first element. */
13901 {
13902 /* For vectors of two 64-bit elements, we can do even better. */
13907 {
13910 /* Combine can pick up this case, but handling it directly
13911 here leaves clearer RTL.
13913 This is load_pair_lanes<mode>, and also gives us a clean-up
13914 for store_pair_lanes<mode>. */
13917 && !STRICT_ALIGNMENT
13922 {
13923 rtx t;
13926 else
13930 }
13931 }
13932 /* The subreg-move sequence below will move into lane zero of the
13933 vector register. For big-endian we want that position to hold
13934 the last element of VALS. */
13938 }
13939 else
13940 {
13943 }
13945 /* Insert the rest. */
13947 {
13953 }
13955 }
13957 /* Initialise a vector which is part-variable. We want to first try
13958 to build those lanes which are constant in the most efficient way we
13959 can. */
13961 {
13964 /* Load constant part of vector. We really don't care what goes into the
13965 parts we will overwrite, but we're more likely to be able to load the
13966 constant efficiently if it has fewer, larger, repeating parts
13967 (see aarch64_simd_valid_immediate). */
13969 {
13975 {
13976 /* Look in the copied vector, as more elements are const. */
13979 {
13982 }
13983 }
13985 }
13987 }
13989 /* Insert the variable lanes directly. */
13991 {
13997 }
13998 }
14000 static unsigned HOST_WIDE_INT
14002 {
14006 }
14008 /* Select a format to encode pointers in exception handling data. */
14009 int
14011 {
14014 {
14020 /* text+got+data < 4Gb. 4-byte signed relocs are sufficient
14021 for everything. */
14025 /* No assumptions here. 8-byte relocs required. */
14028 }
14030 }
14032 /* The last .arch and .tune assembly strings that we printed. */
14036 /* Implement ASM_DECLARE_FUNCTION_NAME. Output the ISA features used
14037 by the function fndecl. */
14039 void
14041 tree fndecl)
14042 {
14048 else
14059 /* Only update the assembler .arch string if it is distinct from the last
14060 such string we printed. */
14063 {
14066 }
14068 /* Print the cpu name we're tuning for in the comments, might be
14069 useful to readers of the generated asm. Do it only when it changes
14070 from function to function and verbose assembly is requested. */
14075 {
14079 }
14081 /* Don't forget the type directive for ELF. */
14084 }
14086 /* Implements TARGET_ASM_FILE_START. Output the assembly header. */
14088 static void
14090 {
14107 }
14109 /* Emit load exclusive. */
14111 static void
14114 {
14116 }
14118 /* Emit store exclusive. */
14120 static void
14123 {
14125 }
14127 /* Mark the previous jump instruction as unlikely. */
14129 static void
14131 {
14134 }
14136 /* Expand a compare and swap pattern. */
14138 void
14140 {
14155 /* Normally the succ memory model must be stronger than fail, but in the
14156 unlikely event of fail being ACQUIRE and succ being RELEASE we need to
14157 promote succ to ACQ_REL so that we don't lose the acquire semantics. */
14164 {
14167 /* For short modes, we're going to perform the comparison in SImode,
14168 so do the zero-extension now. */
14172 /* Fall through. */
14176 /* Force the value into a register if needed. */
14183 }
14188 mod_f));
14189 else
14200 }
14202 /* Test whether the target supports using a atomic load-operate instruction.
14203 CODE is the operation and AFTER is TRUE if the data in memory after the
14204 operation should be returned and FALSE if the data before the operation
14205 should be returned. Returns FALSE if the operation isn't supported by the
14206 architecture. */
14208 bool
14210 {
14215 {
14225 }
14226 }
14228 /* Emit a barrier, that is appropriate for memory model MODEL, at the end of a
14229 sequence implementing an atomic operation. */
14231 static void
14233 {
14240 {
14242 }
14243 }
14245 /* Emit an atomic compare-and-swap operation. RVAL is the destination register
14246 for the data in memory. EXPECTED is the value expected to be in memory.
14247 DESIRED is the value to store to memory. MEM is the memory location. MODEL
14248 is the memory ordering to use. */
14250 void
14253 rtx model)
14254 {
14255 machine_mode mode;
14259 /* Move the expected value into the CAS destination register. */
14262 /* Emit the CAS. */
14265 /* Compare the expected value with the value loaded by the CAS, to establish
14266 whether the swap was made. */
14268 }
14270 /* Split a compare and swap pattern. */
14272 void
14274 {
14276 machine_mode mode;
14281 rtx model_rtx;
14293 /* When OLDVAL is zero and we want the strong version we can emit a tighter
14294 loop:
14295 .label1:
14296 LD[A]XR rval, [mem]
14297 CBNZ rval, .label2
14298 ST[L]XR scratch, newval, [mem]
14299 CBNZ scratch, .label1
14300 .label2:
14301 CMP rval, 0. */
14306 {
14309 }
14312 /* The initial load can be relaxed for a __sync operation since a final
14313 barrier will be emitted to stop code hoisting. */
14317 else
14321 {
14323 {
14324 /* Emit an explicit compare instruction, so that we can correctly
14325 track the condition codes. */
14328 }
14329 else
14335 }
14336 else
14337 {
14343 }
14348 {
14350 {
14351 /* Emit an explicit compare instruction, so that we can correctly
14352 track the condition codes. */
14355 }
14356 else
14362 }
14363 else
14364 {
14368 }
14371 /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL
14372 to set the condition flags. If this is not used it will be removed by
14373 later passes. */
14375 {
14379 }
14380 /* Emit any final barrier needed for a __sync operation. */
14383 }
14385 /* Emit a BIC instruction. */
14387 static void
14389 {
14394 {
14399 }
14402 }
14404 /* Emit an atomic swap. */
14406 static void
14409 {
14411 }
14413 /* Emit an atomic load+operate. CODE is the operation. OUT_DATA is the
14414 location to store the data read from memory. OUT_RESULT is the location to
14415 store the result of the operation. MEM is the memory location to read and
14416 modify. MODEL_RTX is the memory ordering to use. VALUE is the second
14417 operand for the operation. Either OUT_DATA or OUT_RESULT, but not both, can
14418 be NULL. */
14420 void
14423 {
14428 rtx src;
14429 rtx x;
14437 /* Make sure the value is in a register, putting it into a destination
14438 register if it needs to be manipulated. */
14441 {
14444 }
14445 else
14449 /* Preprocess the data for the operation as necessary. If the operation is
14450 a SET then emit a swap instruction and finish. */
14452 {
14458 /* Negate the value and treat it as a PLUS. */
14459 {
14460 rtx neg_src;
14462 /* Resize the value if necessary. */
14471 }
14472 /* Fall-through. */
14486 {
14487 rtx not_src;
14489 /* Resize the value if necessary. */
14498 }
14503 /* The operation can't be done with atomic instructions. */
14505 }
14510 /* If necessary, calculate the data in memory after the update by redoing the
14511 operation from values in registers. */
14516 {
14520 }
14525 {
14541 }
14546 }
14548 /* Split an atomic operation. */
14550 void
14553 {
14559 rtx x;
14561 /* Split the atomic operation into a sequence. */
14569 else
14573 /* The initial load can be relaxed for a __sync operation since a final
14574 barrier will be emitted to stop code hoisting. */
14578 else
14582 {
14596 {
14599 }
14600 /* Fall through. */
14606 }
14612 {
14613 /* Emit an explicit compare instruction, so that we can correctly
14614 track the condition codes. */
14617 }
14618 else
14625 /* Emit any final barrier needed for a __sync operation. */
14628 }
14630 static void
14632 {
14633 /* Half-precision float operations. The compiler handles all operations
14634 with NULL libfuncs by converting to SFmode. */
14636 /* Conversions. */
14640 /* Arithmetic. */
14647 /* Comparisons. */
14655 }
14657 /* Target hook for c_mode_for_suffix. */
14658 static machine_mode
14660 {
14665 }
14667 /* We can only represent floating point constants which will fit in
14668 "quarter-precision" values. These values are characterised by
14669 a sign bit, a 4-bit mantissa and a 3-bit exponent. And are given
14670 by:
14672 (-1)^s * (n/16) * 2^r
14674 Where:
14675 's' is the sign bit.
14676 'n' is an integer in the range 16 <= n <= 31.
14677 'r' is an integer in the range -3 <= r <= 4. */
14679 /* Return true iff X can be represented by a quarter-precision
14680 floating point immediate operand X. Note, we cannot represent 0.0. */
14681 bool
14683 {
14684 /* This represents our current view of how many bits
14685 make up the mantissa. */
14701 /* We cannot represent infinities, NaNs or +/-zero. We won't
14702 know if we have +zero until we analyse the mantissa, but we
14703 can reject the other invalid values. */
14708 /* Extract exponent. */
14712 /* For the mantissa, we expand into two HOST_WIDE_INTS, apart from the
14713 highest (sign) bit, with a fixed binary point at bit point_pos.
14714 m1 holds the low part of the mantissa, m2 the high part.
14715 WARNING: If we ever have a representation using more than 2 * H_W_I - 1
14716 bits for the mantissa, this can fail (low bits will be lost). */
14720 /* If the low part of the mantissa has bits set we cannot represent
14721 the value. */
14724 /* We have rejected the lower HOST_WIDE_INT, so update our
14725 understanding of how many bits lie in the mantissa and
14726 look only at the high HOST_WIDE_INT. */
14730 /* We can only represent values with a mantissa of the form 1.xxxx. */
14735 /* Having filtered unrepresentable values, we may now remove all
14736 but the highest 5 bits. */
14739 /* We cannot represent the value 0.0, so reject it. This is handled
14740 elsewhere. */
14744 /* Then, as bit 4 is always set, we can mask it off, leaving
14745 the mantissa in the range [0, 15]. */
14749 /* GCC internally does not use IEEE754-like encoding (where normalized
14750 significands are in the range [1, 2). GCC uses [0.5, 1) (see real.c).
14751 Our mantissa values are shifted 4 places to the left relative to
14752 normalized IEEE754 so we must modify the exponent returned by REAL_EXP
14753 by 5 places to correct for GCC's representation. */
14757 }
14759 /* Returns the string with the instruction for AdvSIMD MOVI, MVNI, ORR or BIC
14760 immediate with a CONST_VECTOR of MODE and WIDTH. WHICH selects whether to
14761 output MOVI/MVNI, ORR or BIC immediate. */
14765 {
14775 /* This will return true to show const_vector is legal for use as either
14776 a AdvSIMD MOVI instruction (or, implicitly, MVNI), ORR or BIC immediate.
14777 It will also update INFO to show how the immediate should be generated.
14778 WHICH selects whether to check for MOVI/MVNI, ORR or BIC. */
14786 {
14788 /* For FP zero change it to a CONST_INT 0 and use the integer SIMD
14789 move immediate path. */
14792 else
14793 {
14802 else
14806 }
14807 }
14812 {
14822 else
14826 }
14827 else
14828 {
14829 /* For AARCH64_CHECK_BIC and AARCH64_CHECK_ORR. */
14835 else
14839 }
14841 }
14845 {
14847 /* If a floating point number was passed and we desire to use it in an
14848 integer mode do the conversion to integer. */
14850 {
14855 }
14857 machine_mode vmode;
14858 /* use a 64 bit mode for everything except for DI/DF mode, where we use
14859 a 128 bit vector mode. */
14865 }
14867 /* Return the output string to use for moving immediate CONST_VECTOR
14868 into an SVE register. */
14872 {
14883 {
14888 }
14891 {
14894 else
14895 {
14905 }
14906 }
14911 }
14913 /* Return the asm format for a PTRUE instruction whose destination has
14914 mode MODE. SUFFIX is the element size suffix. */
14918 {
14923 else
14926 }
14928 /* Split operands into moves from op[1] + op[2] into op[0]. */
14930 void
14932 {
14943 {
14944 /* No-op move. Can't split to nothing; emit something. */
14947 }
14949 /* Preserve register attributes for variable tracking. */
14954 /* Special case of reversed high/low parts. */
14957 {
14961 }
14963 {
14964 /* Try to avoid unnecessary moves if part of the result
14965 is in the right place already. */
14970 }
14971 else
14972 {
14977 }
14978 }
14980 /* vec_perm support. */
14982 struct expand_vec_perm_d
14983 {
14985 vec_perm_indices perm;
14986 machine_mode vmode;
14990 };
14992 /* Generate a variable permutation. */
14994 static void
14996 {
15007 {
15009 {
15010 /* Expand the argument to a V16QI mode by duplicating it. */
15014 }
15015 else
15016 {
15018 }
15019 }
15020 else
15021 {
15022 rtx pair;
15025 {
15029 }
15030 else
15031 {
15035 }
15036 }
15037 }
15039 /* Expand a vec_perm with the operands given by TARGET, OP0, OP1 and SEL.
15040 NELT is the number of elements in the vector. */
15042 void
15045 {
15048 rtx mask;
15050 /* The TBL instruction does not use a modulo index, so we must take care
15051 of that ourselves. */
15056 /* For big-endian, we also need to reverse the index within the vector
15057 (but not which vector). */
15059 {
15060 /* If one_vector_p, mask is a vector of (nelt - 1)'s already. */
15065 }
15067 }
15069 /* Generate (set TARGET (unspec [OP0 OP1] CODE)). */
15071 static void
15073 {
15077 }
15079 /* Expand an SVE vec_perm with the given operands. */
15081 void
15083 {
15086 /* Enforced by the pattern condition. */
15089 /* Note: vec_perm indices are supposed to wrap when they go beyond the
15090 size of the two value vectors, i.e. the upper bits of the indices
15091 are effectively ignored. SVE TBL instead produces 0 for any
15092 out-of-range indices, so we need to modulo all the vec_perm indices
15093 to ensure they are all in range. */
15096 /* Check if the sel only references the first values vector. */
15099 {
15102 }
15104 /* Check if the two values vectors are the same. */
15106 {
15112 }
15114 /* Run TBL on for each value vector and combine the results. */
15121 {
15126 }
15133 else
15135 }
15137 /* Recognize patterns suitable for the TRN instructions. */
15138 static bool
15140 {
15141 HOST_WIDE_INT odd;
15149 /* Note that these are little-endian tests.
15150 We correct for big-endian later. */
15157 /* Success! */
15163 /* We don't need a big-endian lane correction for SVE; see the comment
15164 at the head of aarch64-sve.md for details. */
15166 {
15169 }
15175 }
15177 /* Recognize patterns suitable for the UZP instructions. */
15178 static bool
15180 {
15181 HOST_WIDE_INT odd;
15188 /* Note that these are little-endian tests.
15189 We correct for big-endian later. */
15195 /* Success! */
15201 /* We don't need a big-endian lane correction for SVE; see the comment
15202 at the head of aarch64-sve.md for details. */
15204 {
15207 }
15213 }
15215 /* Recognize patterns suitable for the ZIP instructions. */
15216 static bool
15218 {
15227 /* Note that these are little-endian tests.
15228 We correct for big-endian later. */
15236 /* Success! */
15242 /* We don't need a big-endian lane correction for SVE; see the comment
15243 at the head of aarch64-sve.md for details. */
15245 {
15248 }
15254 }
15256 /* Recognize patterns for the EXT insn. */
15258 static bool
15260 {
15261 HOST_WIDE_INT location;
15262 rtx offset;
15264 /* The first element always refers to the first vector.
15265 Check if the extracted indices are increasing by one. */
15271 /* Success! */
15275 /* The case where (location == 0) is a no-op for both big- and little-endian,
15276 and is removed by the mid-end at optimization levels -O1 and higher.
15278 We don't need a big-endian lane correction for SVE; see the comment
15279 at the head of aarch64-sve.md for details. */
15281 {
15282 /* After setup, we want the high elements of the first vector (stored
15283 at the LSB end of the register), and the low elements of the second
15284 vector (stored at the MSB end of the register). So swap. */
15286 /* location != 0 (above), so safe to assume (nelt - location) < nelt.
15287 to_constant () is safe since this is restricted to Advanced SIMD
15288 vectors. */
15290 }
15296 UNSPEC_EXT));
15298 }
15300 /* Recognize patterns for the REV{64,32,16} insns, which reverse elements
15301 within each 64-bit, 32-bit or 16-bit granule. */
15303 static bool
15305 {
15306 HOST_WIDE_INT diff;
15308 machine_mode pred_mode;
15317 {
15320 }
15322 {
15325 }
15327 {
15330 }
15331 else
15339 /* Success! */
15345 {
15348 UNSPEC_MERGE_PTRUE);
15349 }
15352 }
15354 /* Recognize patterns for the REV insn, which reverses elements within
15355 a full vector. */
15357 static bool
15359 {
15368 /* Success! */
15375 }
15377 static bool
15379 {
15381 rtx in0;
15382 HOST_WIDE_INT elt;
15384 rtx lane;
15394 /* Success! */
15398 /* The generic preparation in aarch64_expand_vec_perm_const_1
15399 swaps the operand order and the permute indices if it finds
15400 d->perm[0] to be in the second operand. Thus, we can always
15401 use d->op0 and need not do any extra arithmetic to get the
15402 correct lane number. */
15410 }
15412 static bool
15414 {
15418 /* Make sure that the indices are constant. */
15427 /* Generic code will try constant permutation twice. Once with the
15428 original mode and again with the elements lowered to QImode.
15429 So wait and don't do the selector expansion ourselves. */
15433 /* to_constant is safe since this routine is specific to Advanced SIMD
15434 vectors. */
15437 /* If big-endian and two vectors we end up with a weird mixed-endian
15438 mode on NEON. Reverse the index within each word but not the word
15439 itself. to_constant is safe because we checked is_constant above. */
15449 }
15451 /* Try to implement D using an SVE TBL instruction. */
15453 static bool
15455 {
15458 /* Permuting two variable-length vectors could overflow the
15459 index range. */
15470 else
15473 }
15475 static bool
15477 {
15478 /* The pattern matching functions above are written to look for a small
15479 number to begin the sequence (0, 1, N/2). If we begin with an index
15480 from the second operand, we can swap the operands. */
15483 {
15486 }
15492 {
15511 }
15513 }
15515 /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
15517 static bool
15520 {
15523 /* Check whether the mask can be applied to a single vector. */
15528 {
15531 }
15533 {
15536 }
15537 else
15557 }
15559 /* Generate a byte permute mask for a register of mode MODE,
15560 which has NUNITS units. */
15562 rtx
15564 {
15565 /* We have to reverse each vector because we dont have
15566 a permuted load that can reverse-load according to ABI rules. */
15567 rtx mask;
15580 }
15582 /* Return true if X is a valid second operand for the SVE instruction
15583 that implements integer comparison OP_CODE. */
15585 static bool
15587 {
15592 {
15607 }
15608 }
15610 /* Use predicated SVE instructions to implement the equivalent of:
15612 (set TARGET OP)
15614 given that PTRUE is an all-true predicate of the appropriate mode. */
15616 static void
15618 {
15621 UNSPEC_MERGE_PTRUE);
15624 }
15626 /* Likewise, but also clobber the condition codes. */
15628 static void
15630 {
15633 UNSPEC_MERGE_PTRUE);
15636 }
15638 /* Return the UNSPEC_COND_* code for comparison CODE. */
15640 static unsigned int
15642 {
15644 {
15659 }
15660 }
15662 /* Emit:
15664 (set TARGET (unspec [PRED OP0 OP1] UNSPEC_COND_<X>))
15666 where <X> is the operation associated with comparison CODE. This form
15667 of instruction is used when (and (CODE OP0 OP1) PRED) would have different
15668 semantics, such as when PRED might not be all-true and when comparing
15669 inactive lanes could have side effects. */
15671 static void
15674 {
15679 }
15681 /* Expand an SVE integer comparison using the SVE equivalent of:
15683 (set TARGET (CODE OP0 OP1)). */
15685 void
15687 {
15697 }
15699 /* Emit the SVE equivalent of:
15701 (set TMP1 (CODE1 OP0 OP1))
15702 (set TMP2 (CODE2 OP0 OP1))
15703 (set TARGET (ior:PRED_MODE TMP1 TMP2))
15705 PTRUE is an all-true predicate with the same mode as TARGET. */
15707 static void
15710 {
15719 }
15721 /* Emit the SVE equivalent of:
15723 (set TMP (CODE OP0 OP1))
15724 (set TARGET (not TMP))
15726 PTRUE is an all-true predicate with the same mode as TARGET. */
15728 static void
15731 {
15737 }
15739 /* Expand an SVE floating-point comparison using the SVE equivalent of:
15741 (set TARGET (CODE OP0 OP1))
15743 If CAN_INVERT_P is true, the caller can also handle inverted results;
15744 return true if the result is in fact inverted. */
15746 bool
15749 {
15755 {
15757 /* UNORDERED has no immediate form. */
15759 /* fall through */
15766 {
15767 /* There is native support for the comparison. */
15771 }
15774 /* This is a trapping operation (LT or GT). */
15780 {
15781 /* This would trap for signaling NaNs. */
15785 }
15786 /* fall through */
15792 {
15793 /* Work out which elements are ordered. */
15798 /* Test the opposite condition for the ordered elements,
15799 then invert the result. */
15802 else
15805 {
15809 }
15814 }
15818 /* ORDERED has no immediate form. */
15824 }
15826 /* There is native support for the inverse comparison. */
15829 {
15833 }
15836 }
15838 /* Expand an SVE vcond pattern with operands OPS. DATA_MODE is the mode
15839 of the data being selected and CMP_MODE is the mode of the values being
15840 compared. */
15842 void
15845 {
15846 machine_mode pred_mode
15851 {
15855 }
15856 else
15861 }
15863 /* Implement TARGET_MODES_TIEABLE_P. In principle we should always return
15864 true. However due to issues with register allocation it is preferable
15865 to avoid tieing integer scalar and FP scalar modes. Executing integer
15866 operations in general registers is better than treating them as scalar
15867 vector operations. This reduces latency and avoids redundant int<->FP
15868 moves. So tie modes if they are either the same class, or vector modes
15869 with other vector modes, vector structs or any scalar mode. */
15871 static bool
15873 {
15877 /* We specifically want to allow elements of "structure" modes to
15878 be tieable to the structure. This more general condition allows
15879 other rarer situations too. The reason we don't extend this to
15880 predicate modes is that there are no predicate structure modes
15881 nor any specific instructions for extracting part of a predicate
15882 register. */
15887 /* Also allow any scalar modes with vectors. */
15893 }
15895 /* Return a new RTX holding the result of moving POINTER forward by
15896 AMOUNT bytes. */
15898 static rtx
15900 {
15905 }
15907 /* Return a new RTX holding the result of moving POINTER forward by the
15908 size of the mode it points to. */
15910 static rtx
15912 {
15914 }
15916 /* Copy one MODE sized block from SRC to DST, then progress SRC and DST by
15917 MODE bytes. */
15919 static void
15921 machine_mode mode)
15922 {
15925 /* "Cast" the pointers to the correct mode. */
15928 /* Emit the memcpy. */
15931 /* Move the pointers forward. */
15934 }
15936 /* Expand movmem, as if from a __builtin_memcpy. Return true if
15937 we succeed, otherwise return false. */
15939 bool
15941 {
15945 rtx base;
15949 /* When optimizing for size, give a better estimate of the length of a
15950 memcpy call, but use the default otherwise. Moves larger than 8 bytes
15951 will always require an even number of instructions to do now. And each
15952 operation requires both a load+store, so devide the max number by 2. */
15955 /* We can't do anything smart if the amount to copy is not constant. */
15961 /* Try to keep the number of instructions low. For all cases we will do at
15962 most two moves for the residual amount, since we'll always overlap the
15963 remainder. */
15973 /* Convert n to bits to make the rest of the code simpler. */
15976 /* Maximum amount to copy in one go. The AArch64 back-end has integer modes
15977 larger than TImode, but we should not use them for loads/stores here. */
15981 {
15982 /* Find the largest mode in which to do the copy in without over reading
15983 or writing. */
15984 opt_scalar_int_mode mode_iter;
15996 /* Do certain trailing copies as overlapping if it's going to be
15997 cheaper. i.e. less instructions to do so. For instance doing a 15
15998 byte copy it's more efficient to do two overlapping 8 byte copies than
15999 8 + 6 + 1. */
16001 {
16007 }
16008 }
16011 }
16013 /* Split a DImode store of a CONST_INT SRC to MEM DST as two
16014 SImode stores. Handle the case when the constant has identical
16015 bottom and top halves. This is beneficial when the two stores can be
16016 merged into an STP and we avoid synthesising potentially expensive
16017 immediates twice. Return true if such a split is possible. */
16019 bool
16021 {
16030 unsigned int orig_cost
16032 unsigned int lo_cost
16035 /* We want to transform:
16036 MOV x1, 49370
16037 MOVK x1, 0x140, lsl 16
16038 MOVK x1, 0xc0da, lsl 32
16039 MOVK x1, 0x140, lsl 48
16040 STR x1, [x0]
16041 into:
16042 MOV w1, 49370
16043 MOVK w1, 0x140, lsl 16
16044 STP w1, w1, [x0]
16045 So we want to perform this only when we save two instructions
16046 or more. When optimizing for size, however, accept any code size
16047 savings we can. */
16062 /* Don't emit an explicit store pair as this may not be always profitable.
16063 Let the sched-fusion logic decide whether to merge them. */
16068 }
16070 /* Generate RTL for a conditional branch with rtx comparison CODE in
16071 mode CC_MODE. The destination of the unlikely conditional branch
16072 is LABEL_REF. */
16074 void
16076 rtx label_ref)
16077 {
16078 rtx x;
16081 const0_rtx);
16085 pc_rtx);
16087 }
16089 /* Generate DImode scratch registers for 128-bit (TImode) addition.
16091 OP1 represents the TImode destination operand 1
16092 OP2 represents the TImode destination operand 2
16093 LOW_DEST represents the low half (DImode) of TImode operand 0
16094 LOW_IN1 represents the low half (DImode) of TImode operand 1
16095 LOW_IN2 represents the low half (DImode) of TImode operand 2
16096 HIGH_DEST represents the high half (DImode) of TImode operand 0
16097 HIGH_IN1 represents the high half (DImode) of TImode operand 1
16098 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
16100 void
16105 {
16114 }
16116 /* Generate DImode scratch registers for 128-bit (TImode) subtraction.
16118 This function differs from 'arch64_addti_scratch_regs' in that
16119 OP1 can be an immediate constant (zero). We must call
16120 subreg_highpart_offset with DImode and TImode arguments, otherwise
16121 VOIDmode will be used for the const_int which generates an internal
16122 error from subreg_size_highpart_offset which does not expect a size of zero.
16124 OP1 represents the TImode destination operand 1
16125 OP2 represents the TImode destination operand 2
16126 LOW_DEST represents the low half (DImode) of TImode operand 0
16127 LOW_IN1 represents the low half (DImode) of TImode operand 1
16128 LOW_IN2 represents the low half (DImode) of TImode operand 2
16129 HIGH_DEST represents the high half (DImode) of TImode operand 0
16130 HIGH_IN1 represents the high half (DImode) of TImode operand 1
16131 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
16134 void
16139 {
16152 }
16154 /* Generate RTL for 128-bit (TImode) subtraction with overflow.
16156 OP0 represents the TImode destination operand 0
16157 LOW_DEST represents the low half (DImode) of TImode operand 0
16158 LOW_IN1 represents the low half (DImode) of TImode operand 1
16159 LOW_IN2 represents the low half (DImode) of TImode operand 2
16160 HIGH_DEST represents the high half (DImode) of TImode operand 0
16161 HIGH_IN1 represents the high half (DImode) of TImode operand 1
16162 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
16164 void
16167 rtx high_in2)
16168 {
16170 {
16174 }
16175 else
16176 {
16178 {
16182 }
16183 else
16187 high_in2));
16188 }
16193 }
16195 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
16197 static unsigned HOST_WIDE_INT
16199 {
16201 }
16203 static rtx
16206 {
16210 insn_code icode;
16221 {
16249 }
16254 {
16257 }
16266 {
16269 }
16275 }
16277 static rtx
16280 {
16284 insn_code icode;
16296 {
16324 }
16329 {
16332 }
16340 {
16345 }
16356 {
16359 }
16365 }
16367 #undef TARGET_GEN_CCMP_FIRST
16368 #define TARGET_GEN_CCMP_FIRST aarch64_gen_ccmp_first
16370 #undef TARGET_GEN_CCMP_NEXT
16371 #define TARGET_GEN_CCMP_NEXT aarch64_gen_ccmp_next
16373 /* Implement TARGET_SCHED_MACRO_FUSION_P. Return true if target supports
16374 instruction fusion of some sort. */
16376 static bool
16378 {
16380 }
16383 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P. Return true if PREV and CURR
16384 should be kept together during scheduling. */
16386 static bool
16388 {
16389 rtx set_dest;
16392 /* prev and curr are simple SET insns i.e. no flag setting or branching. */
16399 {
16400 /* We are trying to match:
16401 prev (mov) == (set (reg r0) (const_int imm16))
16402 curr (movk) == (set (zero_extract (reg r0)
16403 (const_int 16)
16404 (const_int 16))
16405 (const_int imm16_1)) */
16417 {
16419 }
16420 }
16423 {
16425 /* We're trying to match:
16426 prev (adrp) == (set (reg r1)
16427 (high (symbol_ref ("SYM"))))
16428 curr (add) == (set (reg r0)
16429 (lo_sum (reg r1)
16430 (symbol_ref ("SYM"))))
16431 Note that r0 need not necessarily be the same as r1, especially
16432 during pre-regalloc scheduling. */
16436 {
16444 }
16445 }
16448 {
16450 /* We're trying to match:
16451 prev (movk) == (set (zero_extract (reg r0)
16452 (const_int 16)
16453 (const_int 32))
16454 (const_int imm16_1))
16455 curr (movk) == (set (zero_extract (reg r0)
16456 (const_int 16)
16457 (const_int 48))
16458 (const_int imm16_2)) */
16474 }
16476 {
16477 /* We're trying to match:
16478 prev (adrp) == (set (reg r0)
16479 (high (symbol_ref ("SYM"))))
16480 curr (ldr) == (set (reg r1)
16481 (mem (lo_sum (reg r0)
16482 (symbol_ref ("SYM")))))
16483 or
16484 curr (ldr) == (set (reg r1)
16485 (zero_extend (mem
16486 (lo_sum (reg r0)
16487 (symbol_ref ("SYM")))))) */
16490 {
16503 }
16504 }
16512 {
16516 rtx cc_reg_1;
16521 && prev
16523 {
16524 /* FIXME: this misses some which is considered simple arthematic
16525 instructions for ThunderX. Simple shifts are missed here. */
16531 }
16532 }
16535 && curr_set
16538 {
16539 /* We're trying to match:
16540 prev (alu_insn) == (set (r0) plus ((r0) (r1/imm)))
16541 curr (cbz) == (set (pc) (if_then_else (eq/ne) (r0)
16542 (const_int 0))
16543 (label_ref ("SYM"))
16544 (pc)) */
16551 {
16552 /* Fuse ALU operations followed by conditional branch instruction. */
16554 {
16575 }
16576 }
16577 }
16580 }
16582 /* Return true iff the instruction fusion described by OP is enabled. */
16584 bool
16586 {
16588 }
16590 /* If MEM is in the form of [base+offset], extract the two parts
16591 of address and set to BASE and OFFSET, otherwise return false
16592 after clearing BASE and OFFSET. */
16594 bool
16596 {
16597 rtx addr;
16604 {
16608 }
16612 {
16616 }
16622 }
16624 /* Types for scheduling fusion. */
16625 enum sched_fusion_type
16626 {
16628 SCHED_FUSION_LD_SIGN_EXTEND,
16629 SCHED_FUSION_LD_ZERO_EXTEND,
16630 SCHED_FUSION_LD,
16631 SCHED_FUSION_ST,
16632 SCHED_FUSION_NUM
16633 };
16635 /* If INSN is a load or store of address in the form of [base+offset],
16636 extract the two parts and set to BASE and OFFSET. Return scheduling
16637 fusion type this INSN is. */
16639 static enum sched_fusion_type
16641 {
16659 {
16664 }
16666 {
16671 }
16676 {
16679 }
16680 else
16687 }
16689 /* Implement the TARGET_SCHED_FUSION_PRIORITY hook.
16691 Currently we only support to fuse ldr or str instructions, so FUSION_PRI
16692 and PRI are only calculated for these instructions. For other instruction,
16693 FUSION_PRI and PRI are simply set to MAX_PRI - 1. In the future, other
16694 type instruction fusion can be added by returning different priorities.
16696 It's important that irrelevant instructions get the largest FUSION_PRI. */
16698 static void
16701 {
16711 {
16715 }
16717 /* Set FUSION_PRI according to fusion type and base register. */
16720 /* Calculate PRI. */
16723 /* INSN with smaller offset goes first. */
16727 else
16732 }
16734 /* Implement the TARGET_SCHED_ADJUST_PRIORITY hook.
16735 Adjust priority of sha1h instructions so they are scheduled before
16736 other SHA1 instructions. */
16738 static int
16740 {
16744 {
16749 }
16752 }
16754 /* Given OPERANDS of consecutive load/store, check if we can merge
16755 them into ldp/stp. LOAD is true if they are load instructions.
16756 MODE is the mode of memory operands. */
16758 bool
16760 machine_mode mode)
16761 {
16767 {
16775 }
16776 else
16777 {
16782 }
16784 /* The mems cannot be volatile. */
16788 /* If we have SImode and slow unaligned ldp,
16789 check the alignment to be at least 8 byte. */
16793 && !optimize_size
16797 /* Check if the addresses are in the form of [base+offset]. */
16805 /* Check if the bases are same. */
16809 /* The operands must be of the same size. */
16815 /* We should only be trying this for fixed-sized modes. There is no
16816 SVE LDP/STP instruction. */
16818 /* Check if the offsets are consecutive. */
16822 /* Check if the addresses are clobbered by load. */
16824 {
16828 /* In increasing order, the last load can clobber the address. */
16831 }
16833 /* One of the memory accesses must be a mempair operand.
16834 If it is not the first one, they need to be swapped by the
16835 peephole. */
16842 else
16847 else
16850 /* Check if the registers are of same class. */
16855 }
16857 /* Given OPERANDS of consecutive load/store that can be merged,
16858 swap them if they are not in ascending order. */
16859 void
16861 {
16866 {
16869 }
16870 else
16871 {
16874 }
16883 {
16884 /* Irrespective of whether this is a load or a store,
16885 we do the same swap. */
16888 }
16889 }
16891 /* Taking X and Y to be HOST_WIDE_INT pointers, return the result of a
16892 comparison between the two. */
16893 int
16895 {
16898 }
16900 /* Taking X and Y to be pairs of RTX, one pointing to a MEM rtx and the
16901 other pointing to a REG rtx containing an offset, compare the offsets
16902 of the two pairs.
16904 Return:
16906 1 iff offset (X) > offset (Y)
16907 0 iff offset (X) == offset (Y)
16908 -1 iff offset (X) < offset (Y) */
16909 int
16911 {
16918 else
16923 else
16926 /* Extract the offsets. */
16933 }
16935 /* Given OPERANDS of consecutive load/store, check if we can merge
16936 them into ldp/stp by adjusting the offset. LOAD is true if they
16937 are load instructions. MODE is the mode of memory operands.
16939 Given below consecutive stores:
16941 str w1, [xb, 0x100]
16942 str w1, [xb, 0x104]
16943 str w1, [xb, 0x108]
16944 str w1, [xb, 0x10c]
16946 Though the offsets are out of the range supported by stp, we can
16947 still pair them after adjusting the offset, like:
16949 add scratch, xb, 0x100
16950 stp w1, w1, [scratch]
16951 stp w1, w1, [scratch, 0x8]
16953 The peephole patterns detecting this opportunity should guarantee
16954 the scratch register is avaliable. */
16956 bool
16958 scalar_mode mode)
16959 {
16966 {
16968 {
16973 }
16975 /* Do not attempt to merge the loads if the loads clobber each other. */
16980 }
16981 else
16983 {
16986 }
16988 /* Skip if memory operand is by itself valid for ldp/stp. */
16993 {
16994 /* The mems cannot be volatile. */
16998 /* Check if the addresses are in the form of [base+offset]. */
17002 }
17004 /* Check if the registers are of same class. */
17010 {
17013 }
17014 else
17015 {
17018 }
17020 /* Only the last register in the order in which they occur
17021 may be clobbered by the load. */
17027 /* Check if the bases are same. */
17037 /* Check if the offsets can be put in the right order to do a ldp/stp. */
17039 aarch64_host_wide_int_compare);
17045 /* Check that offsets are within range of each other. The ldp/stp
17046 instructions have 7 bit immediate offsets, so use 0x80. */
17050 /* The offsets must be aligned with respect to each other. */
17054 /* If we have SImode and slow unaligned ldp,
17055 check the alignment to be at least 8 byte. */
17059 && !optimize_size
17064 }
17066 /* Given OPERANDS of consecutive load/store, this function pairs them
17067 into LDP/STP after adjusting the offset. It depends on the fact
17068 that the operands can be sorted so the offsets are correct for STP.
17069 MODE is the mode of memory operands. CODE is the rtl operator
17070 which should be applied to all memory operands, it's SIGN_EXTEND,
17071 ZERO_EXTEND or UNKNOWN. */
17073 bool
17076 {
17083 /* We make changes on a copy as we may still bail out. */
17087 /* Sort the operands. */
17091 {
17096 }
17097 else
17098 {
17104 }
17111 /* Adjust offset so it can fit in LDP/STP instruction. */
17119 /* The base offset is optimally half way between the two STP/LDP offsets. */
17122 else
17123 /* However, due to issues with negative LDP/STP offset generation for
17124 larger modes, for DF, DI and vector modes. we must not use negative
17125 addresses smaller than 9 signed unadjusted bits can store. This
17126 provides the most range in this case. */
17129 /* Adjust the base so that it is aligned with the addresses but still
17130 optimal. */
17132 /* Fix the offset, bearing in mind we want to make it bigger not
17133 smaller. */
17136 /* The negative range of LDP/STP is one larger than the positive range. */
17139 /* Check if base offset is too big or too small. We can attempt to resolve
17140 this issue by setting it to the maximum value and seeing if the offsets
17141 still fit. */
17143 {
17145 /* We must still make sure that the base offset is aligned with respect
17146 to the address. But it may may not be made any bigger. */
17148 }
17150 /* Likewise for the case where the base is too small. */
17152 {
17155 }
17157 /* Offset of the first STP/LDP. */
17160 /* Offset of the second STP/LDP. */
17163 /* The offsets must be within the range of the LDP/STP instructions. */
17182 {
17187 }
17189 {
17194 }
17197 {
17206 }
17207 else
17208 {
17217 }
17219 /* Emit adjusting instruction. */
17221 /* Emit ldp/stp instructions. */
17229 }
17231 /* Implement TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE. Assume for now that
17232 it isn't worth branching around empty masked ops (including masked
17233 stores). */
17235 static bool
17237 {
17239 }
17241 /* Return 1 if pseudo register should be created and used to hold
17242 GOT address for PIC code. */
17244 bool
17246 {
17248 }
17250 /* Implement TARGET_UNSPEC_MAY_TRAP_P. */
17252 static int
17254 {
17256 {
17263 }
17266 }
17269 /* If X is a positive CONST_DOUBLE with a value that is a power of 2
17270 return the log2 of that value. Otherwise return -1. */
17272 int
17274 {
17289 }
17291 /* If X is a vector of equal CONST_DOUBLE values and that value is
17292 Y, return the aarch64_fpconst_pow_of_2 of Y. Otherwise return -1. */
17294 int
17296 {
17314 }
17316 /* Implement TARGET_PROMOTED_TYPE to promote 16-bit floating point types
17317 to float.
17319 __fp16 always promotes through this hook.
17320 _Float16 may promote if TARGET_FLT_EVAL_METHOD is 16, but we do that
17321 through the generic excess precision logic rather than here. */
17323 static tree
17325 {
17331 }
17333 /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
17335 static bool
17337 optimization_type opt_type)
17338 {
17340 {
17346 }
17347 }
17349 /* Implement the TARGET_DWARF_POLY_INDETERMINATE_VALUE hook. */
17351 static unsigned int
17354 {
17355 /* Polynomial invariant 1 == (VG / 2) - 1. */
17360 }
17362 /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
17363 if MODE is HFmode, and punt to the generic implementation otherwise. */
17365 static bool
17367 {
17371 }
17373 /* Implement TARGET_SCALAR_MODE_SUPPORTED_P - return TRUE
17374 if MODE is HFmode, and punt to the generic implementation otherwise. */
17376 static bool
17378 {
17382 }
17384 /* Set the value of FLT_EVAL_METHOD.
17385 ISO/IEC TS 18661-3 defines two values that we'd like to make use of:
17387 0: evaluate all operations and constants, whose semantic type has at
17388 most the range and precision of type float, to the range and
17389 precision of float; evaluate all other operations and constants to
17390 the range and precision of the semantic type;
17392 N, where _FloatN is a supported interchange floating type
17393 evaluate all operations and constants, whose semantic type has at
17394 most the range and precision of _FloatN type, to the range and
17395 precision of the _FloatN type; evaluate all other operations and
17396 constants to the range and precision of the semantic type;
17398 If we have the ARMv8.2-A extensions then we support _Float16 in native
17399 precision, so we should set this to 16. Otherwise, we support the type,
17400 but want to evaluate expressions in float precision, so set this to
17401 0. */
17403 static enum flt_eval_method
17405 {
17407 {
17410 /* We can calculate either in 16-bit range and precision or
17411 32-bit range and precision. Make that decision based on whether
17412 we have native support for the ARMv8.2-A 16-bit floating-point
17413 instructions or not. */
17415 ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
17421 }
17423 }
17425 /* Implement TARGET_SCHED_CAN_SPECULATE_INSN. Return true if INSN can be
17426 scheduled for speculative execution. Reject the long-running division
17427 and square-root instructions. */
17429 static bool
17431 {
17433 {
17451 }
17452 }
17454 /* Implement TARGET_COMPUTE_PRESSURE_CLASSES. */
17456 static int
17458 {
17462 /* PR_REGS isn't a useful pressure class because many predicate pseudo
17463 registers need to go in PR_LO_REGS at some point during their
17464 lifetime. Splitting it into two halves has the effect of making
17465 all predicates count against PR_LO_REGS, so that we try whenever
17466 possible to restrict the number of live predicates to 8. This
17467 greatly reduces the amount of spilling in certain loops. */
17471 }
17473 /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
17475 static bool
17478 {
17480 {
17484 /* Don't allow changes between SVE data modes and non-SVE modes.
17485 See the comment at the head of aarch64-sve.md for details. */
17489 /* Don't allow changes in element size: lane 0 of the new vector
17490 would not then be lane 0 of the old vector. See the comment
17491 above aarch64_maybe_expand_sve_subreg_move for a more detailed
17492 description.
17494 In the worst case, this forces a register to be spilled in
17495 one mode and reloaded in the other, which handles the
17496 endianness correctly. */
17499 }
17501 }
17503 /* Implement TARGET_EARLY_REMAT_MODES. */
17505 static void
17507 {
17508 /* SVE values are not normally live across a call, so it should be
17509 worth doing early rematerialization even in VL-specific mode. */
17511 {
17516 }
17517 }
17519 /* Override the default target speculation_safe_value. */
17520 static rtx
17523 {
17524 /* Maybe we should warn if falling back to hard barriers. They are
17525 likely to be noticably more expensive than the alternative below. */
17536 {
17554 }
17556 }
17558 /* Target-specific selftests. */
17560 #if CHECKING_P
17564 /* Selftest for the RTL loader.
17565 Verify that the RTL loader copes with a dump from
17566 print_rtx_function. This is essentially just a test that class
17567 function_reader can handle a real dump, but it also verifies
17568 that lookup_reg_by_dump_name correctly handles hard regs.
17569 The presence of hard reg names in the dump means that the test is
17570 target-specific, hence it is in this file. */
17572 static void
17574 {
17586 /* Verify crtl->return_rtx. */
17590 }
17592 /* Run all target-specific selftests. */
17594 static void
17596 {
17598 }
17604 #undef TARGET_ADDRESS_COST
17605 #define TARGET_ADDRESS_COST aarch64_address_cost
17607 /* This hook will determines whether unnamed bitfields affect the alignment
17608 of the containing structure. The hook returns true if the structure
17609 should inherit the alignment requirements of an unnamed bitfield's
17610 type. */
17611 #undef TARGET_ALIGN_ANON_BITFIELD
17612 #define TARGET_ALIGN_ANON_BITFIELD hook_bool_void_true
17614 #undef TARGET_ASM_ALIGNED_DI_OP
17617 #undef TARGET_ASM_ALIGNED_HI_OP
17620 #undef TARGET_ASM_ALIGNED_SI_OP
17623 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
17624 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK \
17625 hook_bool_const_tree_hwi_hwi_const_tree_true
17627 #undef TARGET_ASM_FILE_START
17628 #define TARGET_ASM_FILE_START aarch64_start_file
17630 #undef TARGET_ASM_OUTPUT_MI_THUNK
17631 #define TARGET_ASM_OUTPUT_MI_THUNK aarch64_output_mi_thunk
17633 #undef TARGET_ASM_SELECT_RTX_SECTION
17634 #define TARGET_ASM_SELECT_RTX_SECTION aarch64_select_rtx_section
17636 #undef TARGET_ASM_TRAMPOLINE_TEMPLATE
17637 #define TARGET_ASM_TRAMPOLINE_TEMPLATE aarch64_asm_trampoline_template
17639 #undef TARGET_BUILD_BUILTIN_VA_LIST
17640 #define TARGET_BUILD_BUILTIN_VA_LIST aarch64_build_builtin_va_list
17642 #undef TARGET_CALLEE_COPIES
17643 #define TARGET_CALLEE_COPIES hook_bool_CUMULATIVE_ARGS_mode_tree_bool_false
17645 #undef TARGET_CAN_ELIMINATE
17646 #define TARGET_CAN_ELIMINATE aarch64_can_eliminate
17648 #undef TARGET_CAN_INLINE_P
17649 #define TARGET_CAN_INLINE_P aarch64_can_inline_p
17651 #undef TARGET_CANNOT_FORCE_CONST_MEM
17652 #define TARGET_CANNOT_FORCE_CONST_MEM aarch64_cannot_force_const_mem
17654 #undef TARGET_CASE_VALUES_THRESHOLD
17655 #define TARGET_CASE_VALUES_THRESHOLD aarch64_case_values_threshold
17657 #undef TARGET_CONDITIONAL_REGISTER_USAGE
17658 #define TARGET_CONDITIONAL_REGISTER_USAGE aarch64_conditional_register_usage
17660 /* Only the least significant bit is used for initialization guard
17661 variables. */
17662 #undef TARGET_CXX_GUARD_MASK_BIT
17663 #define TARGET_CXX_GUARD_MASK_BIT hook_bool_void_true
17665 #undef TARGET_C_MODE_FOR_SUFFIX
17666 #define TARGET_C_MODE_FOR_SUFFIX aarch64_c_mode_for_suffix
17668 #ifdef TARGET_BIG_ENDIAN_DEFAULT
17669 #undef TARGET_DEFAULT_TARGET_FLAGS
17670 #define TARGET_DEFAULT_TARGET_FLAGS (MASK_BIG_END)
17671 #endif
17673 #undef TARGET_CLASS_MAX_NREGS
17674 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
17676 #undef TARGET_BUILTIN_DECL
17677 #define TARGET_BUILTIN_DECL aarch64_builtin_decl
17679 #undef TARGET_BUILTIN_RECIPROCAL
17680 #define TARGET_BUILTIN_RECIPROCAL aarch64_builtin_reciprocal
17682 #undef TARGET_C_EXCESS_PRECISION
17683 #define TARGET_C_EXCESS_PRECISION aarch64_excess_precision
17685 #undef TARGET_EXPAND_BUILTIN
17686 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
17688 #undef TARGET_EXPAND_BUILTIN_VA_START
17689 #define TARGET_EXPAND_BUILTIN_VA_START aarch64_expand_builtin_va_start
17691 #undef TARGET_FOLD_BUILTIN
17692 #define TARGET_FOLD_BUILTIN aarch64_fold_builtin
17694 #undef TARGET_FUNCTION_ARG
17695 #define TARGET_FUNCTION_ARG aarch64_function_arg
17697 #undef TARGET_FUNCTION_ARG_ADVANCE
17698 #define TARGET_FUNCTION_ARG_ADVANCE aarch64_function_arg_advance
17700 #undef TARGET_FUNCTION_ARG_BOUNDARY
17701 #define TARGET_FUNCTION_ARG_BOUNDARY aarch64_function_arg_boundary
17703 #undef TARGET_FUNCTION_ARG_PADDING
17704 #define TARGET_FUNCTION_ARG_PADDING aarch64_function_arg_padding
17706 #undef TARGET_GET_RAW_RESULT_MODE
17707 #define TARGET_GET_RAW_RESULT_MODE aarch64_get_reg_raw_mode
17708 #undef TARGET_GET_RAW_ARG_MODE
17709 #define TARGET_GET_RAW_ARG_MODE aarch64_get_reg_raw_mode
17711 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
17712 #define TARGET_FUNCTION_OK_FOR_SIBCALL aarch64_function_ok_for_sibcall
17714 #undef TARGET_FUNCTION_VALUE
17715 #define TARGET_FUNCTION_VALUE aarch64_function_value
17717 #undef TARGET_FUNCTION_VALUE_REGNO_P
17718 #define TARGET_FUNCTION_VALUE_REGNO_P aarch64_function_value_regno_p
17720 #undef TARGET_GIMPLE_FOLD_BUILTIN
17721 #define TARGET_GIMPLE_FOLD_BUILTIN aarch64_gimple_fold_builtin
17723 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
17724 #define TARGET_GIMPLIFY_VA_ARG_EXPR aarch64_gimplify_va_arg_expr
17726 #undef TARGET_INIT_BUILTINS
17727 #define TARGET_INIT_BUILTINS aarch64_init_builtins
17729 #undef TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS
17730 #define TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS \
17731 aarch64_ira_change_pseudo_allocno_class
17733 #undef TARGET_LEGITIMATE_ADDRESS_P
17734 #define TARGET_LEGITIMATE_ADDRESS_P aarch64_legitimate_address_hook_p
17736 #undef TARGET_LEGITIMATE_CONSTANT_P
17737 #define TARGET_LEGITIMATE_CONSTANT_P aarch64_legitimate_constant_p
17739 #undef TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT
17740 #define TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT \
17741 aarch64_legitimize_address_displacement
17743 #undef TARGET_LIBGCC_CMP_RETURN_MODE
17744 #define TARGET_LIBGCC_CMP_RETURN_MODE aarch64_libgcc_cmp_return_mode
17746 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
17747 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
17748 aarch64_libgcc_floating_mode_supported_p
17750 #undef TARGET_MANGLE_TYPE
17751 #define TARGET_MANGLE_TYPE aarch64_mangle_type
17753 #undef TARGET_MEMORY_MOVE_COST
17754 #define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
17756 #undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
17757 #define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
17759 #undef TARGET_MUST_PASS_IN_STACK
17760 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
17762 /* This target hook should return true if accesses to volatile bitfields
17763 should use the narrowest mode possible. It should return false if these
17764 accesses should use the bitfield container type. */
17765 #undef TARGET_NARROW_VOLATILE_BITFIELD
17766 #define TARGET_NARROW_VOLATILE_BITFIELD hook_bool_void_false
17768 #undef TARGET_OPTION_OVERRIDE
17769 #define TARGET_OPTION_OVERRIDE aarch64_override_options
17771 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
17772 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE \
17773 aarch64_override_options_after_change
17775 #undef TARGET_OPTION_SAVE
17776 #define TARGET_OPTION_SAVE aarch64_option_save
17778 #undef TARGET_OPTION_RESTORE
17779 #define TARGET_OPTION_RESTORE aarch64_option_restore
17781 #undef TARGET_OPTION_PRINT
17782 #define TARGET_OPTION_PRINT aarch64_option_print
17784 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
17785 #define TARGET_OPTION_VALID_ATTRIBUTE_P aarch64_option_valid_attribute_p
17787 #undef TARGET_SET_CURRENT_FUNCTION
17788 #define TARGET_SET_CURRENT_FUNCTION aarch64_set_current_function
17790 #undef TARGET_PASS_BY_REFERENCE
17791 #define TARGET_PASS_BY_REFERENCE aarch64_pass_by_reference
17793 #undef TARGET_PREFERRED_RELOAD_CLASS
17794 #define TARGET_PREFERRED_RELOAD_CLASS aarch64_preferred_reload_class
17796 #undef TARGET_SCHED_REASSOCIATION_WIDTH
17797 #define TARGET_SCHED_REASSOCIATION_WIDTH aarch64_reassociation_width
17799 #undef TARGET_PROMOTED_TYPE
17800 #define TARGET_PROMOTED_TYPE aarch64_promoted_type
17802 #undef TARGET_SECONDARY_RELOAD
17803 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
17805 #undef TARGET_SHIFT_TRUNCATION_MASK
17806 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
17808 #undef TARGET_SETUP_INCOMING_VARARGS
17809 #define TARGET_SETUP_INCOMING_VARARGS aarch64_setup_incoming_varargs
17811 #undef TARGET_STRUCT_VALUE_RTX
17812 #define TARGET_STRUCT_VALUE_RTX aarch64_struct_value_rtx
17814 #undef TARGET_REGISTER_MOVE_COST
17815 #define TARGET_REGISTER_MOVE_COST aarch64_register_move_cost
17817 #undef TARGET_RETURN_IN_MEMORY
17818 #define TARGET_RETURN_IN_MEMORY aarch64_return_in_memory
17820 #undef TARGET_RETURN_IN_MSB
17821 #define TARGET_RETURN_IN_MSB aarch64_return_in_msb
17823 #undef TARGET_RTX_COSTS
17824 #define TARGET_RTX_COSTS aarch64_rtx_costs_wrapper
17826 #undef TARGET_SCALAR_MODE_SUPPORTED_P
17827 #define TARGET_SCALAR_MODE_SUPPORTED_P aarch64_scalar_mode_supported_p
17829 #undef TARGET_SCHED_ISSUE_RATE
17830 #define TARGET_SCHED_ISSUE_RATE aarch64_sched_issue_rate
17832 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
17833 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
17834 aarch64_sched_first_cycle_multipass_dfa_lookahead
17836 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
17837 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD \
17838 aarch64_first_cycle_multipass_dfa_lookahead_guard
17840 #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
17841 #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS \
17842 aarch64_get_separate_components
17844 #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
17845 #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB \
17846 aarch64_components_for_bb
17848 #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
17849 #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS \
17850 aarch64_disqualify_components
17852 #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
17853 #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
17854 aarch64_emit_prologue_components
17856 #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
17857 #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
17858 aarch64_emit_epilogue_components
17860 #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
17861 #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS \
17862 aarch64_set_handled_components
17864 #undef TARGET_TRAMPOLINE_INIT
17865 #define TARGET_TRAMPOLINE_INIT aarch64_trampoline_init
17867 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
17868 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P aarch64_use_blocks_for_constant_p
17870 #undef TARGET_VECTOR_MODE_SUPPORTED_P
17871 #define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
17873 #undef TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT
17874 #define TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT \
17875 aarch64_builtin_support_vector_misalignment
17877 #undef TARGET_ARRAY_MODE
17878 #define TARGET_ARRAY_MODE aarch64_array_mode
17880 #undef TARGET_ARRAY_MODE_SUPPORTED_P
17881 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
17883 #undef TARGET_VECTORIZE_ADD_STMT_COST
17884 #define TARGET_VECTORIZE_ADD_STMT_COST aarch64_add_stmt_cost
17886 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
17887 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
17888 aarch64_builtin_vectorization_cost
17890 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
17891 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
17893 #undef TARGET_VECTORIZE_BUILTINS
17894 #define TARGET_VECTORIZE_BUILTINS
17896 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
17897 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
17898 aarch64_builtin_vectorized_function
17900 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
17901 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
17902 aarch64_autovectorize_vector_sizes
17904 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
17905 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV \
17906 aarch64_atomic_assign_expand_fenv
17908 /* Section anchor support. */
17910 #undef TARGET_MIN_ANCHOR_OFFSET
17911 #define TARGET_MIN_ANCHOR_OFFSET -256
17913 /* Limit the maximum anchor offset to 4k-1, since that's the limit for a
17914 byte offset; we can do much more for larger data types, but have no way
17915 to determine the size of the access. We assume accesses are aligned. */
17916 #undef TARGET_MAX_ANCHOR_OFFSET
17917 #define TARGET_MAX_ANCHOR_OFFSET 4095
17919 #undef TARGET_VECTOR_ALIGNMENT
17920 #define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
17922 #undef TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
17923 #define TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT \
17924 aarch64_vectorize_preferred_vector_alignment
17925 #undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
17926 #define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
17927 aarch64_simd_vector_alignment_reachable
17929 /* vec_perm support. */
17931 #undef TARGET_VECTORIZE_VEC_PERM_CONST
17932 #define TARGET_VECTORIZE_VEC_PERM_CONST \
17933 aarch64_vectorize_vec_perm_const
17935 #undef TARGET_VECTORIZE_GET_MASK_MODE
17936 #define TARGET_VECTORIZE_GET_MASK_MODE aarch64_get_mask_mode
17937 #undef TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE
17938 #define TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE \
17939 aarch64_empty_mask_is_expensive
17940 #undef TARGET_PREFERRED_ELSE_VALUE
17941 #define TARGET_PREFERRED_ELSE_VALUE \
17942 aarch64_preferred_else_value
17944 #undef TARGET_INIT_LIBFUNCS
17945 #define TARGET_INIT_LIBFUNCS aarch64_init_libfuncs
17947 #undef TARGET_FIXED_CONDITION_CODE_REGS
17948 #define TARGET_FIXED_CONDITION_CODE_REGS aarch64_fixed_condition_code_regs
17950 #undef TARGET_FLAGS_REGNUM
17951 #define TARGET_FLAGS_REGNUM CC_REGNUM
17953 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
17954 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
17956 #undef TARGET_ASAN_SHADOW_OFFSET
17957 #define TARGET_ASAN_SHADOW_OFFSET aarch64_asan_shadow_offset
17959 #undef TARGET_LEGITIMIZE_ADDRESS
17960 #define TARGET_LEGITIMIZE_ADDRESS aarch64_legitimize_address
17962 #undef TARGET_SCHED_CAN_SPECULATE_INSN
17963 #define TARGET_SCHED_CAN_SPECULATE_INSN aarch64_sched_can_speculate_insn
17965 #undef TARGET_CAN_USE_DOLOOP_P
17966 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
17968 #undef TARGET_SCHED_ADJUST_PRIORITY
17969 #define TARGET_SCHED_ADJUST_PRIORITY aarch64_sched_adjust_priority
17971 #undef TARGET_SCHED_MACRO_FUSION_P
17972 #define TARGET_SCHED_MACRO_FUSION_P aarch64_macro_fusion_p
17974 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
17975 #define TARGET_SCHED_MACRO_FUSION_PAIR_P aarch_macro_fusion_pair_p
17977 #undef TARGET_SCHED_FUSION_PRIORITY
17978 #define TARGET_SCHED_FUSION_PRIORITY aarch64_sched_fusion_priority
17980 #undef TARGET_UNSPEC_MAY_TRAP_P
17981 #define TARGET_UNSPEC_MAY_TRAP_P aarch64_unspec_may_trap_p
17983 #undef TARGET_USE_PSEUDO_PIC_REG
17984 #define TARGET_USE_PSEUDO_PIC_REG aarch64_use_pseudo_pic_reg
17986 #undef TARGET_PRINT_OPERAND
17987 #define TARGET_PRINT_OPERAND aarch64_print_operand
17989 #undef TARGET_PRINT_OPERAND_ADDRESS
17990 #define TARGET_PRINT_OPERAND_ADDRESS aarch64_print_operand_address
17992 #undef TARGET_OPTAB_SUPPORTED_P
17993 #define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p
17995 #undef TARGET_OMIT_STRUCT_RETURN_REG
17996 #define TARGET_OMIT_STRUCT_RETURN_REG true
17998 #undef TARGET_DWARF_POLY_INDETERMINATE_VALUE
17999 #define TARGET_DWARF_POLY_INDETERMINATE_VALUE \
18000 aarch64_dwarf_poly_indeterminate_value
18002 /* The architecture reserves bits 0 and 1 so use bit 2 for descriptors. */
18003 #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
18004 #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 4
18006 #undef TARGET_HARD_REGNO_NREGS
18007 #define TARGET_HARD_REGNO_NREGS aarch64_hard_regno_nregs
18008 #undef TARGET_HARD_REGNO_MODE_OK
18009 #define TARGET_HARD_REGNO_MODE_OK aarch64_hard_regno_mode_ok
18011 #undef TARGET_MODES_TIEABLE_P
18012 #define TARGET_MODES_TIEABLE_P aarch64_modes_tieable_p
18014 #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
18015 #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
18016 aarch64_hard_regno_call_part_clobbered
18018 #undef TARGET_CONSTANT_ALIGNMENT
18019 #define TARGET_CONSTANT_ALIGNMENT aarch64_constant_alignment
18021 #undef TARGET_COMPUTE_PRESSURE_CLASSES
18022 #define TARGET_COMPUTE_PRESSURE_CLASSES aarch64_compute_pressure_classes
18024 #undef TARGET_CAN_CHANGE_MODE_CLASS
18025 #define TARGET_CAN_CHANGE_MODE_CLASS aarch64_can_change_mode_class
18027 #undef TARGET_SELECT_EARLY_REMAT_MODES
18028 #define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
18030 #undef TARGET_SPECULATION_SAFE_VALUE
18031 #define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value
18033 #if CHECKING_P
18034 #undef TARGET_RUN_TARGET_SELFTESTS
18035 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests