| blob | c5ed870ef57a458ae5f8a393cfa20c58c446271e |
1 /* Machine description for AArch64 architecture.
2 Copyright (C) 2009-2018 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
24 #define INCLUDE_STRING
75 /* This file should be included last. */
78 /* Defined for convenience. */
79 #define POINTER_BYTES (POINTER_SIZE / BITS_PER_UNIT)
81 /* Classifies an address.
83 ADDRESS_REG_IMM
84 A simple base register plus immediate offset.
86 ADDRESS_REG_WB
87 A base register indexed by immediate offset with writeback.
89 ADDRESS_REG_REG
90 A base register indexed by (optionally scaled) register.
92 ADDRESS_REG_UXTW
93 A base register indexed by (optionally scaled) zero-extended register.
95 ADDRESS_REG_SXTW
96 A base register indexed by (optionally scaled) sign-extended register.
98 ADDRESS_LO_SUM
99 A LO_SUM rtx with a base register and "LO12" symbol relocation.
101 ADDRESS_SYMBOLIC:
102 A constant symbolic address, in pc-relative literal pool. */
105 ADDRESS_REG_IMM,
106 ADDRESS_REG_WB,
107 ADDRESS_REG_REG,
108 ADDRESS_REG_UXTW,
109 ADDRESS_REG_SXTW,
110 ADDRESS_LO_SUM,
111 ADDRESS_SYMBOLIC
112 };
116 rtx base;
117 rtx offset;
118 poly_int64 const_offset;
121 };
123 /* Information about a legitimate vector immediate operand. */
124 struct simd_immediate_info
125 {
136 /* The mode of the elements. */
137 scalar_mode elt_mode;
139 /* The value of each element if all elements are the same, or the
140 first value if the constant is a series. */
141 rtx value;
143 /* The value of the step if the constant is a series, null otherwise. */
144 rtx step;
146 /* The instruction to use to move the immediate into a vector. */
147 insn_type insn;
149 /* The kind of shift modifier to use, and the number of bits to shift.
150 This is (LSL, 0) if no shift is needed. */
151 modifier_type modifier;
153 };
155 /* Construct a floating-point immediate in which each element has mode
156 ELT_MODE_IN and value VALUE_IN. */
157 inline simd_immediate_info
161 {}
163 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
164 and value VALUE_IN. The other parameters are as for the structure
165 fields. */
166 inline simd_immediate_info
173 {}
175 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
176 and where element I is equal to VALUE_IN + I * STEP_IN. */
177 inline simd_immediate_info
181 {}
183 /* The current code model. */
186 /* The number of 64-bit elements in an SVE vector. */
187 poly_uint16 aarch64_sve_vg;
189 #ifdef HAVE_AS_TLS
190 #undef TARGET_HAVE_TLS
191 #define TARGET_HAVE_TLS 1
192 #endif
196 const_tree,
205 const_tree type,
211 /* Major revision number of the ARM Architecture implemented by the target. */
214 /* The processor for which instructions should be scheduled. */
217 /* Mask to specify which instruction scheduling options should be used. */
220 /* Global flag for PC relative loads. */
223 /* Support for command line parsing of boolean flags in the tuning
224 structures. */
225 struct aarch64_flag_desc
226 {
229 };
231 #define AARCH64_FUSION_PAIR(name, internal_name) \
232 { name, AARCH64_FUSE_##internal_name },
234 {
239 };
241 #define AARCH64_EXTRA_TUNING_OPTION(name, internal_name) \
242 { name, AARCH64_EXTRA_TUNE_##internal_name },
244 {
249 };
251 /* Tuning parameters. */
254 {
255 {
260 },
267 };
270 {
271 {
276 },
283 };
286 {
287 {
292 },
299 };
302 {
303 {
308 },
315 };
318 {
320 /* Avoid the use of slow int<->fp moves for spilling by setting
321 their cost higher than memmov_cost. */
325 };
328 {
330 /* Avoid the use of slow int<->fp moves for spilling by setting
331 their cost higher than memmov_cost. */
335 };
338 {
340 /* Avoid the use of slow int<->fp moves for spilling by setting
341 their cost higher than memmov_cost. */
345 };
348 {
350 /* Avoid the use of slow int<->fp moves for spilling by setting
351 their cost higher than memmov_cost (actual, 4 and 9). */
355 };
358 {
363 };
366 {
368 /* Avoid the use of slow int<->fp moves for spilling by setting
369 their cost higher than memmov_cost. */
373 };
376 {
378 /* Avoid the use of int<->fp moves for spilling. */
382 };
385 {
387 /* Avoid the use of int<->fp moves for spilling. */
391 };
393 /* Generic costs for vector insn classes. */
395 {
411 };
413 /* ThunderX costs for vector insn classes. */
415 {
431 };
433 /* Generic costs for vector insn classes. */
435 {
451 };
454 {
470 };
472 /* Generic costs for vector insn classes. */
474 {
490 };
492 /* Costs for vector insn classes for Vulcan. */
494 {
510 };
512 /* Generic costs for branch instructions. */
514 {
517 };
519 /* Generic approximation modes. */
521 {
524 AARCH64_APPROX_NONE /* recip_sqrt */
525 };
527 /* Approximation modes for Exynos M1. */
529 {
532 AARCH64_APPROX_ALL /* recip_sqrt */
533 };
535 /* Approximation modes for X-Gene 1. */
537 {
540 AARCH64_APPROX_ALL /* recip_sqrt */
541 };
543 /* Generic prefetch settings (which disable prefetch). */
545 {
551 };
554 {
560 };
563 {
569 };
572 {
578 };
581 {
587 };
590 {
596 };
599 {
620 &generic_prefetch_tune
621 };
624 {
646 &generic_prefetch_tune
647 };
650 {
672 &generic_prefetch_tune
673 };
676 {
698 &generic_prefetch_tune
699 };
702 {
724 &generic_prefetch_tune
725 };
728 {
750 &generic_prefetch_tune
751 };
756 {
777 &exynosm1_prefetch_tune
778 };
781 {
802 &thunderxt88_prefetch_tune
803 };
806 {
826 (AARCH64_EXTRA_TUNE_SLOW_UNALIGNED_LDPW
828 &thunderx_prefetch_tune
829 };
832 {
853 &generic_prefetch_tune
854 };
857 {
879 &qdf24xx_prefetch_tune
880 };
882 /* Tuning structure for the Qualcomm Saphira core. Default to falkor values
883 for now. */
885 {
907 &generic_prefetch_tune
908 };
911 {
933 &thunderx2t99_prefetch_tune
934 };
936 /* Support for fine-grained override of the tuning structures. */
937 struct aarch64_tuning_override_function
938 {
941 };
946 static const struct aarch64_tuning_override_function
947 aarch64_tuning_override_functions[] =
948 {
952 };
954 /* A processor implementing AArch64. */
955 struct processor
956 {
964 };
966 /* Architectures implementing AArch64. */
968 {
969 #define AARCH64_ARCH(NAME, CORE, ARCH_IDENT, ARCH_REV, FLAGS) \
970 {NAME, CORE, CORE, AARCH64_ARCH_##ARCH_IDENT, ARCH_REV, FLAGS, NULL},
973 };
975 /* Processor cores implementing AArch64. */
977 {
978 #define AARCH64_CORE(NAME, IDENT, SCHED, ARCH, FLAGS, COSTS, IMP, PART, VARIANT) \
979 {NAME, IDENT, SCHED, AARCH64_ARCH_##ARCH, \
980 all_architectures[AARCH64_ARCH_##ARCH].architecture_version, \
981 FLAGS, &COSTS##_tunings},
986 };
989 /* Target specification. These are populated by the -march, -mtune, -mcpu
990 handling code or by target attributes. */
995 /* The current tuning set. */
998 #define AARCH64_CPU_DEFAULT_FLAGS ((selected_cpu) ? selected_cpu->flags : 0)
1000 /* An ISA extension in the co-processor and main instruction set space. */
1001 struct aarch64_option_extension
1002 {
1006 };
1009 {
1013 }
1014 aarch64_cc;
1016 #define AARCH64_INVERSE_CONDITION_CODE(X) ((aarch64_cc) (((int) X) ^ 1))
1018 /* The condition codes of the processor, and the inverse function. */
1020 {
1023 };
1025 /* Generate code to enable conditional branches in functions over 1 MiB. */
1029 {
1046 }
1048 void
1050 {
1054 else
1056 }
1058 /* Implement TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS.
1059 The register allocator chooses ALL_REGS if FP_REGS and GENERAL_REGS have
1060 the same cost even if ALL_REGS has a much larger cost. ALL_REGS is also
1061 used if the cost of both FP_REGS and GENERAL_REGS is lower than the memory
1062 cost (in this case the best class is the lowest cost one). Using ALL_REGS
1063 irrespectively of its cost results in bad allocations with many redundant
1064 int<->FP moves which are expensive on various cores.
1065 To avoid this we don't allow ALL_REGS as the allocno class, but force a
1066 decision between FP_REGS and GENERAL_REGS. We use the allocno class if it
1067 isn't ALL_REGS. Similarly, use the best class if it isn't ALL_REGS.
1068 Otherwise set the allocno class depending on the mode.
1069 The result of this is that it is no longer inefficient to have a higher
1070 memory move cost than the register move cost.
1071 */
1073 static reg_class_t
1075 reg_class_t best_class)
1076 {
1077 machine_mode mode;
1087 }
1089 static unsigned int
1091 {
1095 }
1097 static int
1099 machine_mode mode)
1100 {
1108 }
1110 /* Provide a mapping from gcc register numbers to dwarf register numbers. */
1111 unsigned
1113 {
1125 /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
1126 equivalent DWARF register. */
1128 }
1130 /* Return true if MODE is any of the Advanced SIMD structure modes. */
1131 static bool
1133 {
1136 }
1138 /* Return true if MODE is an SVE predicate mode. */
1139 static bool
1141 {
1147 }
1149 /* Three mutually-exclusive flags describing a vector or predicate type. */
1153 /* Can be used in combination with VEC_ADVSIMD or VEC_SVE_DATA to indicate
1154 a structure of 2, 3 or 4 vectors. */
1156 /* Useful combinations of the above. */
1160 /* Return a set of flags describing the vector properties of mode MODE.
1161 Ignore modes that are not supported by the current target. */
1162 static unsigned int
1164 {
1180 {
1185 /* This includes V1DF but not V1DI (which doesn't exist). */
1190 }
1193 }
1195 /* Return true if MODE is any of the data vector modes, including
1196 structure modes. */
1197 static bool
1199 {
1201 }
1203 /* Return true if MODE is an SVE data vector mode; either a single vector
1204 or a structure of vectors. */
1205 static bool
1207 {
1209 }
1211 /* Implement target hook TARGET_ARRAY_MODE_SUPPORTED_P. */
1212 static bool
1215 {
1223 }
1225 /* Return the SVE predicate mode to use for elements that have
1226 ELEM_NBYTES bytes, if such a mode exists. */
1228 opt_machine_mode
1230 {
1232 {
1241 }
1243 }
1245 /* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
1247 static opt_machine_mode
1249 {
1251 {
1253 machine_mode pred_mode;
1256 }
1259 }
1261 /* Implement TARGET_HARD_REGNO_NREGS. */
1263 static unsigned int
1265 {
1266 /* ??? Logically we should only need to provide a value when
1267 HARD_REGNO_MODE_OK says that the combination is valid,
1268 but at the moment we need to handle all modes. Just ignore
1269 any runtime parts for registers that can't store them. */
1272 {
1285 }
1287 }
1289 /* Implement TARGET_HARD_REGNO_MODE_OK. */
1291 static bool
1293 {
1298 /* This must have the same size as _Unwind_Word. */
1309 /* The purpose of comparing with ptr_mode is to support the
1310 global register variable associated with the stack pointer
1311 register via the syntax of asm ("wsp") in ILP32. */
1321 {
1324 else
1326 }
1329 }
1331 /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The callee only saves
1332 the lower 64 bits of a 128-bit register. Tell the compiler the callee
1333 clobbers the top 64 bits when restoring the bottom 64 bits. */
1335 static bool
1337 {
1339 }
1341 /* Implement REGMODE_NATURAL_SIZE. */
1342 poly_uint64
1344 {
1345 /* The natural size for SVE data modes is one SVE data vector,
1346 and similarly for predicates. We can't independently modify
1347 anything smaller than that. */
1348 /* ??? For now, only do this for variable-width SVE registers.
1349 Doing it for constant-sized registers breaks lower-subreg.c. */
1350 /* ??? And once that's fixed, we should probably have similar
1351 code for Advanced SIMD. */
1353 {
1359 }
1361 }
1363 /* Implement HARD_REGNO_CALLER_SAVE_MODE. */
1364 machine_mode
1366 machine_mode mode)
1367 {
1368 /* The predicate mode determines which bits are significant and
1369 which are "don't care". Decreasing the number of lanes would
1370 lose data while increasing the number of lanes would make bits
1371 unnecessarily significant. */
1376 else
1378 }
1380 /* Implement TARGET_CONSTANT_ALIGNMENT. Make strings word-aligned so
1381 that strcpy from constants will be faster. */
1383 static HOST_WIDE_INT
1385 {
1389 }
1391 /* Return true if calls to DECL should be treated as
1392 long-calls (ie called via a register). */
1393 static bool
1395 {
1397 }
1399 /* Return true if calls to symbol-ref SYM should be treated as
1400 long-calls (ie called via a register). */
1401 bool
1403 {
1405 }
1407 /* Return true if calls to symbol-ref SYM should not go through
1408 plt stubs. */
1410 bool
1412 {
1416 && decl
1417 && (!flag_plt
1423 }
1425 /* Return true if the offsets to a zero/sign-extract operation
1426 represent an expression that matches an extend operation. The
1427 operands represent the paramters from
1429 (extract:MODE (mult (reg) (MULT_IMM)) (EXTRACT_IMM) (const_int 0)). */
1430 bool
1432 rtx extract_imm)
1433 {
1450 }
1452 /* Emit an insn that's a simple single-set. Both the operands must be
1453 known to be valid. */
1456 {
1458 }
1460 /* X and Y are two things to compare using CODE. Emit the compare insn and
1461 return the rtx for register 0 in the proper mode. */
1462 rtx
1464 {
1470 }
1472 /* Build the SYMBOL_REF for __tls_get_addr. */
1476 rtx
1478 {
1482 }
1484 /* Return the TLS model to use for ADDR. */
1486 static enum tls_model
1488 {
1491 {
1492 poly_int64 addend;
1496 }
1501 }
1503 /* We'll allow lo_sum's in addresses in our legitimate addresses
1504 so that combine would take care of combining addresses where
1505 necessary, but for generation purposes, we'll generate the address
1506 as :
1507 RTL Absolute
1508 tmp = hi (symbol_ref); adrp x1, foo
1509 dest = lo_sum (tmp, symbol_ref); add dest, x1, :lo_12:foo
1510 nop
1512 PIC TLS
1513 adrp x1, :got:foo adrp tmp, :tlsgd:foo
1514 ldr x1, [:got_lo12:foo] add dest, tmp, :tlsgd_lo12:foo
1515 bl __tls_get_addr
1516 nop
1518 Load TLS symbol, depending on TLS mechanism and TLS access model.
1520 Global Dynamic - Traditional TLS:
1521 adrp tmp, :tlsgd:imm
1522 add dest, tmp, #:tlsgd_lo12:imm
1523 bl __tls_get_addr
1525 Global Dynamic - TLS Descriptors:
1526 adrp dest, :tlsdesc:imm
1527 ldr tmp, [dest, #:tlsdesc_lo12:imm]
1528 add dest, dest, #:tlsdesc_lo12:imm
1529 blr tmp
1530 mrs tp, tpidr_el0
1531 add dest, dest, tp
1533 Initial Exec:
1534 mrs tp, tpidr_el0
1535 adrp tmp, :gottprel:imm
1536 ldr dest, [tmp, #:gottprel_lo12:imm]
1537 add dest, dest, tp
1539 Local Exec:
1540 mrs tp, tpidr_el0
1541 add t0, tp, #:tprel_hi12:imm, lsl #12
1542 add t0, t0, #:tprel_lo12_nc:imm
1543 */
1545 static void
1548 {
1550 {
1552 {
1553 /* In ILP32, the mode of dest can be either SImode or DImode. */
1565 }
1572 {
1575 rtx insn;
1576 rtx mem;
1578 /* NOTE: pic_offset_table_rtx can be NULL_RTX, because we can reach
1579 here before rtl expand. Tree IVOPT will generate rtl pattern to
1580 decide rtx costs, in which case pic_offset_table_rtx is not
1581 initialized. For that case no need to generate the first adrp
1582 instruction as the final cost for global variable access is
1583 one instruction. */
1585 {
1586 /* -fpic for -mcmodel=small allow 32K GOT table size (but we are
1587 using the page base as GOT base, the first page may be wasted,
1588 in the worst scenario, there is only 28K space for GOT).
1590 The generate instruction sequence for accessing global variable
1591 is:
1593 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym]
1595 Only one instruction needed. But we must initialize
1596 pic_offset_table_rtx properly. We generate initialize insn for
1597 every global access, and allow CSE to remove all redundant.
1599 The final instruction sequences will look like the following
1600 for multiply global variables access.
1602 adrp pic_offset_table_rtx, _GLOBAL_OFFSET_TABLE_
1604 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym1]
1605 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym2]
1606 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym3]
1607 ... */
1616 }
1619 {
1622 else
1626 }
1627 else
1628 {
1633 }
1635 /* The operand is expected to be MEM. Whenever the related insn
1636 pattern changed, above code which calculate mem should be
1637 updated. */
1643 }
1646 {
1647 /* In ILP32, the mode of dest can be either SImode or DImode,
1648 while the got entry is always of SImode size. The mode of
1649 dest depends on how dest is used: if dest is assigned to a
1650 pointer (e.g. in the memory), it has SImode; it may have
1651 DImode if dest is dereferenced to access the memeory.
1652 This is why we have to handle three different ldr_got_small
1653 patterns here (two patterns for ILP32). */
1655 rtx insn;
1656 rtx mem;
1665 {
1668 else
1672 }
1673 else
1674 {
1679 }
1686 }
1689 {
1697 else
1705 }
1708 {
1711 rtx tp;
1715 /* In ILP32, the got entry is always of SImode size. Unlike
1716 small GOT, the dest is fixed at reg 0. */
1719 else
1730 }
1733 {
1734 /* In ILP32, the mode of dest can be either SImode or DImode,
1735 while the got entry is always of SImode size. The mode of
1736 dest depends on how dest is used: if dest is assigned to a
1737 pointer (e.g. in the memory), it has SImode; it may have
1738 DImode if dest is dereferenced to access the memeory.
1739 This is why we have to handle three different tlsie_small
1740 patterns here (two patterns for ILP32). */
1746 {
1749 else
1750 {
1753 }
1754 }
1755 else
1756 {
1759 }
1765 }
1771 {
1779 {
1802 }
1807 }
1814 {
1819 {
1822 else
1823 {
1826 }
1827 }
1828 else
1829 {
1832 }
1837 }
1841 }
1842 }
1844 /* Emit a move from SRC to DEST. Assume that the move expanders can
1845 handle all moves if !can_create_pseudo_p (). The distinction is
1846 important because, unlike emit_move_insn, the move expanders know
1847 how to force Pmode objects into the constant pool even when the
1848 constant pool address is not itself legitimate. */
1849 static rtx
1851 {
1855 }
1857 /* Split a 128-bit move operation into two 64-bit move operations,
1858 taking care to handle partial overlap of register to register
1859 copies. Special cases are needed when moving between GP regs and
1860 FP regs. SRC can be a register, constant or memory; DST a register
1861 or memory. If either operand is memory it must not have any side
1862 effects. */
1863 void
1865 {
1876 {
1880 /* Handle FP <-> GP regs. */
1882 {
1887 {
1890 }
1891 else
1892 {
1895 }
1897 }
1899 {
1904 {
1907 }
1908 else
1909 {
1912 }
1914 }
1915 }
1922 /* At most one pairing may overlap. */
1924 {
1927 }
1928 else
1929 {
1932 }
1933 }
1935 bool
1937 {
1940 }
1942 /* Split a complex SIMD combine. */
1944 void
1946 {
1958 {
1982 }
1986 }
1988 /* Split a complex SIMD move. */
1990 void
1992 {
1999 {
2005 {
2029 }
2033 }
2034 }
2036 bool
2039 {
2043 }
2046 static rtx
2048 {
2051 else
2052 {
2056 }
2057 }
2059 /* Return true if we can move VALUE into a register using a single
2060 CNT[BHWD] instruction. */
2062 static bool
2064 {
2066 /* The coefficient must be [1, 16] * {2, 4, 8, 16}. */
2071 }
2073 /* Likewise for rtx X. */
2075 bool
2077 {
2078 poly_int64 value;
2080 }
2082 /* Return the asm string for an instruction with a CNT-like vector size
2083 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2084 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2085 first part of the operands template (the part that comes before the
2086 vector size itself). FACTOR is the number of quadwords.
2087 NELTS_PER_VQ, if nonzero, is the number of elements in each quadword.
2088 If it is zero, we can use any element size. */
2094 {
2098 /* There is some overlap in the ranges of the four CNT instructions.
2099 Here we always use the smallest possible element size, so that the
2100 multiplier is 1 whereever possible. */
2111 else
2116 }
2118 /* Return the asm string for an instruction with a CNT-like vector size
2119 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2120 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2121 first part of the operands template (the part that comes before the
2122 vector size itself). X is the value of the vector size operand,
2123 as a polynomial integer rtx. */
2127 rtx x)
2128 {
2133 }
2135 /* Return true if we can add VALUE to a register using a single ADDVL
2136 or ADDPL instruction. */
2138 static bool
2140 {
2144 /* FACTOR counts VG / 2, so a value of 2 is one predicate width
2145 and a value of 16 is one vector width. */
2148 }
2150 /* Likewise for rtx X. */
2152 bool
2154 {
2155 poly_int64 value;
2158 }
2160 /* Return the asm string for adding ADDVL or ADDPL immediate X to operand 1
2161 and storing the result in operand 0. */
2165 {
2170 /* Use INC or DEC if possible. */
2172 {
2179 }
2184 else
2187 }
2189 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2190 instruction. If it is, store the number of elements in each vector
2191 quadword in *NELTS_PER_VQ_OUT (if nonnull) and store the multiplication
2192 factor in *FACTOR_OUT (if nonnull). */
2194 bool
2197 {
2198 rtx elt;
2199 poly_int64 value;
2207 /* There's no vector INCB. */
2214 /* The coefficient must be [1, 16] * NELTS_PER_VQ. */
2224 }
2226 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2227 instruction. */
2229 bool
2231 {
2233 }
2235 /* Return the asm template for an SVE vector INC or DEC instruction.
2236 OPERANDS gives the operands before the vector count and X is the
2237 value of the vector count operand itself. */
2241 {
2248 nelts_per_vq);
2249 else
2251 nelts_per_vq);
2252 }
2254 static int
2256 scalar_int_mode mode)
2257 {
2266 {
2270 }
2272 /* Check to see if the low 32 bits are either 0xffffXXXX or 0xXXXXffff
2273 (with XXXX non-zero). In that case check to see if the move can be done in
2274 a smaller mode. */
2279 {
2283 /* Check if we have to emit a second instruction by checking to see
2284 if any of the upper 32 bits of the original DI mode value is set. */
2295 }
2298 {
2300 {
2305 else
2308 }
2310 }
2312 /* Remaining cases are all for DImode. */
2321 {
2322 /* Try emitting a bitmask immediate with a movk replacing 16 bits.
2323 For a 64-bit bitmask try whether changing 16 bits to all ones or
2324 zeroes creates a valid bitmask. To check any repeated bitmask,
2325 try using 16 bits from the other 32-bit half of val. */
2328 {
2339 }
2341 {
2343 {
2347 }
2349 }
2350 }
2352 /* Generate 2-4 instructions, skipping 16 bits of all zeroes or ones which
2353 are emitted by the initial mov. If one_match > zero_match, skip set bits,
2354 otherwise skip zero bits. */
2366 {
2372 num_insns ++;
2373 }
2376 }
2378 /* Return the number of temporary registers that aarch64_add_offset_1
2379 would need to add OFFSET to a register. */
2381 static unsigned int
2383 {
2385 }
2387 /* A subroutine of aarch64_add_offset. Set DEST to SRC + OFFSET for
2388 a non-polynomial OFFSET. MODE is the mode of the addition.
2389 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
2390 be set and CFA adjustments added to the generated instructions.
2392 TEMP1, if nonnull, is a register of mode MODE that can be used as a
2393 temporary if register allocation is already complete. This temporary
2394 register may overlap DEST but must not overlap SRC. If TEMP1 is known
2395 to hold abs (OFFSET), EMIT_MOVE_IMM can be set to false to avoid emitting
2396 the immediate again.
2398 Since this function may be used to adjust the stack pointer, we must
2399 ensure that it cannot cause transient stack deallocation (for example
2400 by first incrementing SP and then decrementing when adjusting by a
2401 large immediate). */
2403 static void
2407 {
2415 {
2417 {
2420 }
2422 }
2424 /* Single instruction adjustment. */
2426 {
2430 }
2432 /* Emit 2 additions/subtractions if the adjustment is less than 24 bits
2433 and either:
2435 a) the offset cannot be loaded by a 16-bit move or
2436 b) there is no spare register into which we can move it. */
2440 {
2449 }
2451 /* Emit a move immediate if required and an addition/subtraction. */
2453 {
2456 }
2461 {
2465 }
2466 }
2468 /* Return the number of temporary registers that aarch64_add_offset
2469 would need to move OFFSET into a register or add OFFSET to a register;
2470 ADD_P is true if we want the latter rather than the former. */
2472 static unsigned int
2474 {
2475 /* This follows the same structure as aarch64_add_offset. */
2484 /* Need one register for the ADDVL/ADDPL result. */
2487 {
2490 /* Need one register for the CNT result and one for the multiplication
2491 factor. If necessary, the second temporary can be reused for the
2492 constant part of the offset. */
2494 /* Need one register for the CNT result (which might then
2495 be shifted). */
2497 }
2499 }
2501 /* If X can be represented as a poly_int64, return the number
2502 of temporaries that are required to add it to a register.
2503 Return -1 otherwise. */
2505 int
2507 {
2508 poly_int64 offset;
2512 }
2514 /* Set DEST to SRC + OFFSET. MODE is the mode of the addition.
2515 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
2516 be set and CFA adjustments added to the generated instructions.
2518 TEMP1, if nonnull, is a register of mode MODE that can be used as a
2519 temporary if register allocation is already complete. This temporary
2520 register may overlap DEST if !FRAME_RELATED_P but must not overlap SRC.
2521 If TEMP1 is known to hold abs (OFFSET), EMIT_MOVE_IMM can be set to
2522 false to avoid emitting the immediate again.
2524 TEMP2, if nonnull, is a second temporary register that doesn't
2525 overlap either DEST or REG.
2527 Since this function may be used to adjust the stack pointer, we must
2528 ensure that it cannot cause transient stack deallocation (for example
2529 by first incrementing SP and then decrementing when adjusting by a
2530 large immediate). */
2532 static void
2536 {
2540 || !frame_related_p
2544 /* Try using ADDVL or ADDPL to add the whole value. */
2546 {
2551 }
2553 /* Coefficient 1 is multiplied by the number of 128-bit blocks in an
2554 SVE vector register, over and above the minimum size of 128 bits.
2555 This is equivalent to half the value returned by CNTD with a
2556 vector shape of ALL. */
2560 /* Try using ADDVL or ADDPL to add the VG-based part. */
2564 {
2567 {
2571 }
2572 else
2573 {
2578 }
2579 }
2580 /* Otherwise use a CNT-based sequence. */
2582 {
2583 /* Use a subtraction if we have a negative factor. */
2586 {
2589 }
2591 /* Calculate CNTD * FACTOR / 2. First try to fold the division
2592 into the multiplication. */
2593 rtx val;
2596 /* Use a right shift by 1. */
2598 else
2602 {
2604 {
2605 /* "CNTB Xn, ALL, MUL #FACTOR" is out of range, so calculate
2606 the value with the minimum multiplier and shift it into
2607 position. */
2611 }
2613 }
2614 else
2615 {
2616 /* Use CNTD, then multiply it by FACTOR. */
2620 /* Go back to using a negative multiplication factor if we have
2621 no register from which to subtract. */
2623 {
2626 }
2630 }
2633 {
2634 /* Multiply by 1 << SHIFT. */
2637 }
2639 {
2640 /* Divide by 2. */
2643 }
2645 /* Calculate SRC +/- CNTD * FACTOR / 2. */
2647 {
2650 }
2652 {
2655 }
2658 {
2661 {
2665 poly_offset)));
2666 }
2670 }
2671 else
2672 {
2676 }
2679 }
2683 }
2685 /* Like aarch64_add_offset, but the offset is given as an rtx rather
2686 than a poly_int64. */
2688 void
2691 {
2694 }
2696 /* Add DELTA to the stack pointer, marking the instructions frame-related.
2697 TEMP1 is available as a temporary if nonnull. EMIT_MOVE_IMM is false
2698 if TEMP1 already contains abs (DELTA). */
2702 {
2705 }
2707 /* Subtract DELTA from the stack pointer, marking the instructions
2708 frame-related if FRAME_RELATED_P. TEMP1 is available as a temporary
2709 if nonnull. */
2713 {
2716 }
2718 /* Set DEST to (vec_series BASE STEP). */
2720 static void
2722 {
2726 /* Each operand can be a register or an immediate in the range [-16, 15]. */
2733 }
2735 /* Try to duplicate SRC into SVE register DEST, given that SRC is an
2736 integer of mode INT_MODE. Return true on success. */
2738 static bool
2740 rtx src)
2741 {
2742 /* If the constant is smaller than 128 bits, we can do the move
2743 using a vector of SRC_MODEs. */
2745 {
2752 }
2754 /* The bytes are loaded in little-endian order, so do a byteswap on
2755 big-endian targets. */
2757 {
2761 }
2763 /* Use LD1RQ to load the 128 bits from memory. */
2768 /* Make sure that the address is legitimate. */
2770 {
2773 }
2778 }
2780 /* Expand a move of general CONST_VECTOR SRC into DEST, given that it
2781 isn't a simple duplicate or series. */
2783 static void
2785 {
2792 {
2793 /* The constant is a repeating seqeuence of at least two elements,
2794 where the repeating elements occupy no more than 128 bits.
2795 Get an integer representation of the replicated value. */
2804 }
2806 /* Expand each pattern individually. */
2807 rtx_vector_builder builder;
2810 {
2815 }
2817 /* Use permutes to interleave the separate vectors. */
2819 {
2822 {
2827 }
2828 }
2830 }
2832 /* Set DEST to immediate IMM. For SVE vector modes, GEN_VEC_DUPLICATE
2833 is a pattern that can be used to set DEST to a replicated scalar
2834 element. */
2836 void
2839 {
2842 /* Check on what type of symbol it is. */
2843 scalar_int_mode int_mode;
2849 {
2850 rtx mem;
2851 poly_int64 offset;
2852 HOST_WIDE_INT const_offset;
2855 /* If we have (const (plus symbol offset)), separate out the offset
2856 before we start classifying the symbol. */
2859 /* We must always add an offset involving VL separately, rather than
2860 folding it into the relocation. */
2862 {
2865 else
2866 {
2867 /* Do arithmetic on 32-bit values if the result is smaller
2868 than that. */
2870 {
2871 /* It is invalid to do symbol calculations in modes
2872 narrower than SImode. */
2876 }
2878 {
2882 }
2883 else
2886 }
2888 }
2892 {
2896 {
2902 }
2907 /* If we aren't generating PC relative literals, then
2908 we need to expand the literal pool access carefully.
2909 This is something that needs to be done in a number
2910 of places, so could well live as a separate function. */
2912 {
2919 }
2936 {
2942 }
2943 /* FALLTHRU */
2956 }
2957 }
2960 {
2968 {
2969 /* If the constant is out of range of an SVE vector move,
2970 load it from memory if we can, otherwise move it into
2971 a register and use a DUP. */
2977 {
2980 }
2982 }
2986 else
2987 {
2991 }
2994 }
2998 }
3000 /* Emit an SVE predicated move from SRC to DEST. PRED is a predicate
3001 that is known to contain PTRUE. */
3003 void
3005 {
3008 UNSPEC_MERGE_PTRUE)));
3009 }
3011 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one
3012 operand is in memory. In this case we need to use the predicated LD1
3013 and ST1 instead of LDR and STR, both for correctness on big-endian
3014 targets and because LD1 and ST1 support a wider range of addressing modes.
3015 PRED_MODE is the mode of the predicate.
3017 See the comment at the head of aarch64-sve.md for details about the
3018 big-endian handling. */
3020 void
3022 {
3027 {
3031 else
3034 }
3036 }
3038 static bool
3040 tree exp ATTRIBUTE_UNUSED)
3041 {
3042 /* Currently, always true. */
3044 }
3046 /* Implement TARGET_PASS_BY_REFERENCE. */
3048 static bool
3050 machine_mode mode,
3051 const_tree type,
3053 {
3054 HOST_WIDE_INT size;
3055 machine_mode dummymode;
3058 /* GET_MODE_SIZE (BLKmode) is useless since it is 0. */
3061 else
3062 /* No frontends can create types with variable-sized modes, so we
3063 shouldn't be asked to pass or return them. */
3066 /* Aggregates are passed by reference based on their size. */
3068 {
3070 }
3072 /* Variable sized arguments are always returned by reference. */
3076 /* Can this be a candidate to be passed in fp/simd register(s)? */
3079 NULL))
3082 /* Arguments which are variable sized or larger than 2 registers are
3083 passed by reference unless they are a homogenous floating point
3084 aggregate. */
3086 }
3088 /* Return TRUE if VALTYPE is padded to its least significant bits. */
3089 static bool
3091 {
3092 machine_mode dummy_mode;
3095 /* Never happens in little-endian mode. */
3099 /* Only composite types smaller than or equal to 16 bytes can
3100 be potentially returned in registers. */
3106 /* But not a composite that is an HFA (Homogeneous Floating-point Aggregate)
3107 or an HVA (Homogeneous Short-Vector Aggregate); such a special composite
3108 is always passed/returned in the least significant bits of fp/simd
3109 register(s). */
3115 }
3117 /* Implement TARGET_FUNCTION_VALUE.
3118 Define how to find the value returned by a function. */
3120 static rtx
3123 {
3124 machine_mode mode;
3127 machine_mode ag_mode;
3134 {
3138 {
3141 }
3142 }
3146 {
3148 {
3151 }
3152 else
3153 {
3155 rtx par;
3159 {
3164 }
3166 }
3167 }
3168 else
3170 }
3172 /* Implements TARGET_FUNCTION_VALUE_REGNO_P.
3173 Return true if REGNO is the number of a hard register in which the values
3174 of called function may come back. */
3176 static bool
3178 {
3179 /* Maximum of 16 bytes can be returned in the general registers. Examples
3180 of 16-byte return values are: 128-bit integers and 16-byte small
3181 structures (excluding homogeneous floating-point aggregates). */
3185 /* Up to four fp/simd registers can return a function value, e.g. a
3186 homogeneous floating-point aggregate having four members. */
3191 }
3193 /* Implement TARGET_RETURN_IN_MEMORY.
3195 If the type T of the result of a function is such that
3196 void func (T arg)
3197 would require that arg be passed as a value in a register (or set of
3198 registers) according to the parameter passing rules, then the result
3199 is returned in the same registers as would be used for such an
3200 argument. */
3202 static bool
3204 {
3205 HOST_WIDE_INT size;
3206 machine_mode ag_mode;
3212 /* Simple scalar types always returned in registers. */
3216 type,
3219 NULL))
3222 /* Types larger than 2 registers returned in memory. */
3225 }
3227 static bool
3230 {
3233 type,
3235 nregs,
3236 NULL);
3237 }
3239 /* Given MODE and TYPE of a function argument, return the alignment in
3240 bits. The idea is to suppress any stronger alignment requested by
3241 the user and opt for the natural alignment (specified in AAPCS64 \S 4.1).
3242 This is a helper function for local use only. */
3244 static unsigned int
3246 {
3267 }
3269 /* Layout a function argument according to the AAPCS64 rules. The rule
3270 numbers refer to the rule numbers in the AAPCS64. */
3272 static void
3274 const_tree type,
3276 {
3280 HOST_WIDE_INT size;
3282 /* We need to do this once per argument. */
3288 /* Size in bytes, rounded to the nearest multiple of 8 bytes. */
3291 else
3292 /* No frontends can create types with variable-sized modes, so we
3293 shouldn't be asked to pass or return them. */
3299 mode,
3300 type,
3303 /* allocate_ncrn may be false-positive, but allocate_nvrn is quite reliable.
3304 The following code thus handles passing by SIMD/FP registers first. */
3308 /* C1 - C5 for floating point, homogenous floating point aggregates (HFA)
3309 and homogenous short-vector aggregates (HVA). */
3311 {
3316 {
3319 {
3322 }
3323 else
3324 {
3325 rtx par;
3329 {
3332 rtx offset = gen_int_mode
3336 }
3338 }
3340 }
3341 else
3342 {
3343 /* C.3 NSRN is set to 8. */
3346 }
3347 }
3352 /* C6 - C9. though the sign and zero extension semantics are
3353 handled elsewhere. This is the case where the argument fits
3354 entirely general registers. */
3356 {
3360 /* C.8 if the argument has an alignment of 16 then the NGRN is
3361 rounded up to the next even number. */
3364 /* The == 16 * BITS_PER_UNIT instead of >= 16 * BITS_PER_UNIT
3365 comparison is there because for > 16 * BITS_PER_UNIT
3366 alignment nregs should be > 2 and therefore it should be
3367 passed by reference rather than value. */
3369 {
3372 }
3374 /* NREGS can be 0 when e.g. an empty structure is to be passed.
3375 A reg is still generated for it, but the caller should be smart
3376 enough not to use it. */
3379 else
3380 {
3381 rtx par;
3386 {
3391 }
3393 }
3397 }
3399 /* C.11 */
3402 /* The argument is passed on stack; record the needed number of words for
3403 this argument and align the total size if necessary. */
3404 on_stack:
3411 }
3413 /* Implement TARGET_FUNCTION_ARG. */
3415 static rtx
3418 {
3427 }
3429 void
3431 const_tree fntype ATTRIBUTE_UNUSED,
3432 rtx libname ATTRIBUTE_UNUSED,
3433 const_tree fndecl ATTRIBUTE_UNUSED,
3435 {
3449 {
3456 }
3458 }
3460 static void
3462 machine_mode mode,
3463 const_tree type,
3465 {
3468 {
3478 }
3479 }
3481 bool
3483 {
3486 }
3488 /* Implement FUNCTION_ARG_BOUNDARY. Every parameter gets at least
3489 PARM_BOUNDARY bits of alignment, but will be given anything up
3490 to STACK_BOUNDARY bits if the type requires it. This makes sure
3491 that both before and after the layout of each argument, the Next
3492 Stacked Argument Address (NSAA) will have a minimum alignment of
3493 8 bytes. */
3495 static unsigned int
3497 {
3500 }
3502 /* Implement TARGET_GET_RAW_RESULT_MODE and TARGET_GET_RAW_ARG_MODE. */
3504 static fixed_size_mode
3506 {
3508 /* Don't use the SVE part of the register for __builtin_apply and
3509 __builtin_return. The SVE registers aren't used by the normal PCS,
3510 so using them there would be a waste of time. The PCS extensions
3511 for SVE types are fundamentally incompatible with the
3512 __builtin_return/__builtin_apply interface. */
3515 }
3517 /* Implement TARGET_FUNCTION_ARG_PADDING.
3519 Small aggregate types are placed in the lowest memory address.
3521 The related parameter passing rules are B.4, C.3, C.5 and C.14. */
3523 static pad_direction
3525 {
3526 /* On little-endian targets, the least significant byte of every stack
3527 argument is passed at the lowest byte address of the stack slot. */
3531 /* Otherwise, integral, floating-point and pointer types are padded downward:
3532 the least significant byte of a stack argument is passed at the highest
3533 byte address of the stack slot. */
3540 /* Everything else padded upward, i.e. data in first byte of stack slot. */
3542 }
3544 /* Similarly, for use by BLOCK_REG_PADDING (MODE, TYPE, FIRST).
3546 It specifies padding for the last (may also be the only)
3547 element of a block move between registers and memory. If
3548 assuming the block is in the memory, padding upward means that
3549 the last element is padded after its highest significant byte,
3550 while in downward padding, the last element is padded at the
3551 its least significant byte side.
3553 Small aggregates and small complex types are always padded
3554 upwards.
3556 We don't need to worry about homogeneous floating-point or
3557 short-vector aggregates; their move is not affected by the
3558 padding direction determined here. Regardless of endianness,
3559 each element of such an aggregate is put in the least
3560 significant bits of a fp/simd register.
3562 Return !BYTES_BIG_ENDIAN if the least significant byte of the
3563 register has useful data, and return the opposite if the most
3564 significant byte does. */
3566 bool
3569 {
3571 /* Small composite types are always padded upward. */
3573 {
3574 HOST_WIDE_INT size;
3577 else
3578 /* No frontends can create types with variable-sized modes, so we
3579 shouldn't be asked to pass or return them. */
3583 }
3585 /* Otherwise, use the default padding. */
3587 }
3589 static scalar_int_mode
3591 {
3593 }
3595 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
3597 /* We use the 12-bit shifted immediate arithmetic instructions so values
3598 must be multiple of (1 << 12), i.e. 4096. */
3599 #define ARITH_FACTOR 4096
3601 #if (PROBE_INTERVAL % ARITH_FACTOR) != 0
3602 #error Cannot use simple address calculation for stack probing
3603 #endif
3605 /* The pair of scratch registers used for stack probing. */
3606 #define PROBE_STACK_FIRST_REG 9
3607 #define PROBE_STACK_SECOND_REG 10
3609 /* Emit code to probe a range of stack addresses from FIRST to FIRST+POLY_SIZE,
3610 inclusive. These are offsets from the current stack pointer. */
3612 static void
3614 {
3615 HOST_WIDE_INT size;
3617 {
3620 }
3624 /* See the same assertion on PROBE_INTERVAL above. */
3627 /* See if we have a constant small number of probes to generate. If so,
3628 that's the easy case. */
3630 {
3637 }
3639 /* The run-time loop is made up of 8 insns in the generic case while the
3640 compile-time loop is made up of 4+2*(n-2) insns for n # of intervals. */
3642 {
3647 stack_pointer_rtx,
3651 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
3652 it exceeds SIZE. If only two probes are needed, this will not
3653 generate any code. Then probe at FIRST + SIZE. */
3655 {
3659 }
3663 {
3668 }
3669 else
3671 }
3673 /* Otherwise, do the same as above, but in a loop. Note that we must be
3674 extra careful with variables wrapping around because we might be at
3675 the very top (or the very bottom) of the address space and we have
3676 to be able to handle this case properly; in particular, we use an
3677 equality test for the loop condition. */
3678 else
3679 {
3682 /* Step 1: round SIZE to the previous multiple of the interval. */
3687 /* Step 2: compute initial and final value of the loop counter. */
3689 /* TEST_ADDR = SP + FIRST. */
3693 /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE. */
3696 {
3700 }
3701 else
3702 {
3705 }
3707 /* Step 3: the loop
3709 do
3710 {
3711 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
3712 probe at TEST_ADDR
3713 }
3714 while (TEST_ADDR != LAST_ADDR)
3716 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
3717 until it is equal to ROUNDED_SIZE. */
3722 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
3723 that SIZE is equal to ROUNDED_SIZE. */
3726 {
3730 {
3735 }
3736 else
3738 }
3739 }
3741 /* Make sure nothing is scheduled before we are done. */
3743 }
3745 /* Probe a range of stack addresses from REG1 to REG2 inclusive. These are
3746 absolute addresses. */
3750 {
3757 /* Loop. */
3760 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
3765 /* Probe at TEST_ADDR. */
3768 /* Test if TEST_ADDR == LAST_ADDR. */
3772 /* Branch. */
3778 }
3780 /* Mark the registers that need to be saved by the callee and calculate
3781 the size of the callee-saved registers area and frame record (both FP
3782 and LR may be omitted). */
3783 static void
3785 {
3792 /* Force a frame chain for EH returns so the return address is at FP+8. */
3796 /* Emit a frame chain if the frame pointer is enabled.
3797 If -momit-leaf-frame-pointer is used, do not use a frame chain
3798 in leaf functions which do not use LR. */
3804 #define SLOT_NOT_REQUIRED (-2)
3805 #define SLOT_REQUIRED (-1)
3810 /* First mark all the registers that really need to be saved... */
3817 /* ... that includes the eh data registers (if needed)... */
3823 /* ... and any callee saved register that dataflow says is live. */
3833 {
3836 }
3839 {
3840 /* FP and LR are placed in the linkage record. */
3846 }
3848 /* Now assign stack slots for them. */
3851 {
3858 }
3866 {
3867 /* If there is an alignment gap between integer and fp callee-saves,
3868 allocate the last fp register to it if possible. */
3870 {
3873 }
3882 }
3888 HOST_WIDE_INT varargs_and_saved_regs_size
3896 /* Both these values are already aligned. */
3920 {
3921 /* Simple, small frame with no outgoing arguments:
3922 stp reg1, reg2, [sp, -frame_size]!
3923 stp reg3, reg4, [sp, 16] */
3925 }
3930 max_push_offset)))
3931 {
3932 /* Frame with small outgoing arguments:
3933 sub sp, sp, frame_size
3934 stp reg1, reg2, [sp, outgoing_args_size]
3935 stp reg3, reg4, [sp, outgoing_args_size + 16] */
3939 }
3942 {
3943 /* Frame with large outgoing arguments but a small local area:
3944 stp reg1, reg2, [sp, -hard_fp_offset]!
3945 stp reg3, reg4, [sp, 16]
3946 sub sp, sp, outgoing_args_size */
3950 }
3951 else
3952 {
3953 /* Frame with large local area and outgoing arguments using frame pointer:
3954 sub sp, sp, hard_fp_offset
3955 stp x29, x30, [sp, 0]
3956 add x29, sp, 0
3957 stp reg3, reg4, [sp, 16]
3958 sub sp, sp, outgoing_args_size */
3962 }
3965 }
3967 /* Return true if the register REGNO is saved on entry to
3968 the current function. */
3970 static bool
3972 {
3974 }
3976 /* Return the next register up from REGNO up to LIMIT for the callee
3977 to save. */
3979 static unsigned
3981 {
3983 regno ++;
3985 }
3987 /* Push the register number REGNO of mode MODE to the stack with write-back
3988 adjusting the stack by ADJUSTMENT. */
3990 static void
3992 HOST_WIDE_INT adjustment)
3993 {
4004 }
4006 /* Generate and return an instruction to store the pair of registers
4007 REG and REG2 of mode MODE to location BASE with write-back adjusting
4008 the stack location BASE by ADJUSTMENT. */
4010 static rtx
4012 HOST_WIDE_INT adjustment)
4013 {
4015 {
4026 }
4027 }
4029 /* Push registers numbered REGNO1 and REGNO2 to the stack, adjusting the
4030 stack pointer by ADJUSTMENT. */
4032 static void
4034 {
4049 }
4051 /* Load the pair of register REG, REG2 of mode MODE from stack location BASE,
4052 adjusting it by ADJUSTMENT afterwards. */
4054 static rtx
4056 HOST_WIDE_INT adjustment)
4057 {
4059 {
4068 }
4069 }
4071 /* Pop the two registers numbered REGNO1, REGNO2 from the stack, adjusting it
4072 afterwards by ADJUSTMENT and writing the appropriate REG_CFA_RESTORE notes
4073 into CFI_OPS. */
4075 static void
4078 {
4085 {
4089 }
4090 else
4091 {
4096 }
4097 }
4099 /* Generate and return a store pair instruction of mode MODE to store
4100 register REG1 to MEM1 and register REG2 to MEM2. */
4102 static rtx
4104 rtx reg2)
4105 {
4107 {
4116 }
4117 }
4119 /* Generate and regurn a load pair isntruction of mode MODE to load register
4120 REG1 from MEM1 and register REG2 from MEM2. */
4122 static rtx
4124 rtx mem2)
4125 {
4127 {
4136 }
4137 }
4139 /* Return TRUE if return address signing should be enabled for the current
4140 function, otherwise return FALSE. */
4142 bool
4144 {
4145 /* This function should only be called after frame laid out. */
4148 /* If signing scope is AARCH64_FUNCTION_NON_LEAF, we only sign a leaf function
4149 if it's LR is pushed onto stack. */
4153 }
4155 /* Emit code to save the callee-saved registers from register number START
4156 to LIMIT to the stack at the location starting at offset START_OFFSET,
4157 skipping any write-back candidates if SKIP_WB is true. */
4159 static void
4162 {
4170 {
4172 poly_int64 offset;
4185 offset));
4194 {
4196 rtx mem2;
4200 offset));
4202 reg2));
4204 /* The first part of a frame-related parallel insn is
4205 always assumed to be relevant to the frame
4206 calculations; subsequent parts, are only
4207 frame-related if explicitly marked. */
4210 }
4211 else
4215 }
4216 }
4218 /* Emit code to restore the callee registers of mode MODE from register
4219 number START up to and including LIMIT. Restore from the stack offset
4220 START_OFFSET, skipping any write-back candidates if SKIP_WB is true.
4221 Write the appropriate REG_CFA_RESTORE notes into CFI_OPS. */
4223 static void
4227 {
4231 poly_int64 offset;
4236 {
4257 {
4259 rtx mem2;
4267 }
4268 else
4271 }
4272 }
4274 /* Return true if OFFSET is a signed 4-bit value multiplied by the size
4275 of MODE. */
4279 {
4280 HOST_WIDE_INT multiple;
4283 }
4285 /* Return true if OFFSET is a unsigned 6-bit value multiplied by the size
4286 of MODE. */
4290 {
4291 HOST_WIDE_INT multiple;
4294 }
4296 /* Return true if OFFSET is a signed 7-bit value multiplied by the size
4297 of MODE. */
4299 bool
4301 {
4302 HOST_WIDE_INT multiple;
4305 }
4307 /* Return true if OFFSET is a signed 9-bit value. */
4311 poly_int64 offset)
4312 {
4313 HOST_WIDE_INT const_offset;
4316 }
4318 /* Return true if OFFSET is a signed 9-bit value multiplied by the size
4319 of MODE. */
4323 {
4324 HOST_WIDE_INT multiple;
4327 }
4329 /* Return true if OFFSET is an unsigned 12-bit value multiplied by the size
4330 of MODE. */
4334 {
4335 HOST_WIDE_INT multiple;
4338 }
4340 /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
4342 static sbitmap
4344 {
4350 /* The registers we need saved to the frame. */
4353 {
4358 /* Check that we can access the stack slot of the register with one
4359 direct load with no adjustments needed. */
4362 }
4364 /* Don't mess with the hard frame pointer. */
4370 /* If aarch64_layout_frame has chosen registers to store/restore with
4371 writeback don't interfere with them to avoid having to output explicit
4372 stack adjustment instructions. */
4382 }
4384 /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
4386 static sbitmap
4388 {
4396 /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
4405 }
4407 /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS.
4408 Nothing to do for aarch64. */
4410 static void
4412 {
4413 }
4415 /* Return the next set bit in BMP from START onwards. Return the total number
4416 of bits in BMP if no set bit is found at or after START. */
4418 static unsigned int
4420 {
4431 }
4433 /* Do the work for aarch64_emit_prologue_components and
4434 aarch64_emit_epilogue_components. COMPONENTS is the bitmap of registers
4435 to save/restore, PROLOGUE_P indicates whether to emit the prologue sequence
4436 for these components or the epilogue sequence. That is, it determines
4437 whether we should emit stores or loads and what kind of CFA notes to attach
4438 to the insns. Otherwise the logic for the two sequences is very
4439 similar. */
4441 static void
4443 {
4445 ? HARD_FRAME_POINTER_REGNUM
4453 {
4454 /* AAPCS64 section 5.1.2 requires only the bottom 64 bits to be saved
4455 so DFmode for the vector registers is enough. */
4467 /* No more registers to handle after REGNO.
4468 Emit a single save/restore and exit. */
4470 {
4475 else
4478 }
4481 /* The next register is not of the same class or its offset is not
4482 mergeable with the current one into a pair. */
4487 {
4492 else
4497 }
4499 /* REGNO2 can be saved/restored in a pair with REGNO. */
4511 else
4516 {
4519 }
4520 else
4521 {
4524 }
4527 }
4528 }
4530 /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
4532 static void
4534 {
4536 }
4538 /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
4540 static void
4542 {
4544 }
4546 /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
4548 static void
4550 {
4554 }
4556 /* Add a REG_CFA_EXPRESSION note to INSN to say that register REG
4557 is saved at BASE + OFFSET. */
4559 static void
4562 {
4566 }
4568 /* AArch64 stack frames generated by this compiler look like:
4570 +-------------------------------+
4571 | |
4572 | incoming stack arguments |
4573 | |
4574 +-------------------------------+
4575 | | <-- incoming stack pointer (aligned)
4576 | callee-allocated save area |
4577 | for register varargs |
4578 | |
4579 +-------------------------------+
4580 | local variables | <-- frame_pointer_rtx
4581 | |
4582 +-------------------------------+
4583 | padding0 | \
4584 +-------------------------------+ |
4585 | callee-saved registers | | frame.saved_regs_size
4586 +-------------------------------+ |
4587 | LR' | |
4588 +-------------------------------+ |
4589 | FP' | / <- hard_frame_pointer_rtx (aligned)
4590 +-------------------------------+
4591 | dynamic allocation |
4592 +-------------------------------+
4593 | padding |
4594 +-------------------------------+
4595 | outgoing stack arguments | <-- arg_pointer
4596 | |
4597 +-------------------------------+
4598 | | <-- stack_pointer_rtx (aligned)
4600 Dynamic stack allocations via alloca() decrease stack_pointer_rtx
4601 but leave frame_pointer_rtx and hard_frame_pointer_rtx
4602 unchanged. */
4604 /* Generate the prologue instructions for entry into a function.
4605 Establish the stack frame by decreasing the stack pointer with a
4606 properly calculated size and, if necessary, create a frame record
4607 filled with the values of LR and previous frame pointer. The
4608 current FP is also set up if it is in use. */
4610 void
4612 {
4625 /* Sign return address for functions. */
4627 {
4631 }
4637 {
4639 {
4643 (frame_size
4645 }
4648 }
4659 {
4662 {
4667 }
4672 {
4673 /* Variable-sized frames need to describe the save slot
4674 address using DW_CFA_expression rather than DW_CFA_offset.
4675 This means that, without taking further action, the
4676 locations of the registers that we've already saved would
4677 remain based on the stack pointer even after we redefine
4678 the CFA based on the frame pointer. We therefore need new
4679 DW_CFA_expressions to re-express the save slots with addresses
4680 based on the frame pointer. */
4684 /* Add an explicit CFA definition if this was previously
4685 implicit. */
4687 {
4689 callee_offset);
4692 }
4694 /* Change the save slot expressions for the registers that
4695 we've already saved. */
4700 reg_offset);
4701 }
4703 }
4710 }
4712 /* Return TRUE if we can use a simple_return insn.
4714 This function checks whether the callee saved stack is empty, which
4715 means no restore actions are need. The pro_and_epilogue will use
4716 this to check whether shrink-wrapping opt is feasible. */
4718 bool
4720 {
4730 }
4732 /* Generate the epilogue instructions for returning from a function.
4733 This is almost exactly the reverse of the prolog sequence, except
4734 that we need to insert barriers to avoid scheduling loads that read
4735 from a deallocated stack, and we optimize the unwind records by
4736 emitting them all together if possible. */
4737 void
4739 {
4750 /* A stack clash protection prologue may not have left IP0_REGNUM or
4751 IP1_REGNUM in a usable state. The same is true for allocations
4752 with an SVE component, since we then need both temporary registers
4753 for each allocation. */
4758 /* We need to add memory barrier to prevent read from deallocated stack. */
4759 bool need_barrier_p
4763 /* Emit a barrier to prevent loads from a deallocated stack. */
4767 {
4770 }
4772 /* Restore the stack pointer from the frame pointer if it may not
4773 be the same as the stack pointer. */
4778 /* If writeback is used when restoring callee-saves, the CFA
4779 is restored on the instruction doing the writeback. */
4783 else
4799 {
4800 /* Emit delayed restores and set the CFA to be SP + initial_adjust. */
4806 }
4812 {
4813 /* Emit delayed restores and reset the CFA to be SP. */
4818 }
4820 /* We prefer to emit the combined return/authenticate instruction RETAA,
4821 however there are three cases in which we must instead emit an explicit
4822 authentication instruction.
4824 1) Sibcalls don't return in a normal way, so if we're about to call one
4825 we must authenticate.
4827 2) The RETAA instruction is not available before ARMv8.3-A, so if we are
4828 generating code for !TARGET_ARMV8_3 we can't use it and must
4829 explicitly authenticate.
4831 3) On an eh_return path we make extra stack adjustments to update the
4832 canonical frame address to be the exception handler's CFA. We want
4833 to authenticate using the CFA of the function which calls eh_return.
4834 */
4837 {
4841 }
4843 /* Stack adjustment for exception handler. */
4845 {
4846 /* We need to unwind the stack by the offset computed by
4847 EH_RETURN_STACKADJ_RTX. We have already reset the CFA
4848 to be SP; letting the CFA move during this adjustment
4849 is just as correct as retaining the CFA from the body
4850 of the function. Therefore, do nothing special. */
4852 }
4857 }
4859 /* Implement EH_RETURN_HANDLER_RTX. EH returns need to either return
4860 normally or return to a previous frame after unwinding.
4862 An EH return uses a single shared return sequence. The epilogue is
4863 exactly like a normal epilogue except that it has an extra input
4864 register (EH_RETURN_STACKADJ_RTX) which contains the stack adjustment
4865 that must be applied after the frame has been destroyed. An extra label
4866 is inserted before the epilogue which initializes this register to zero,
4867 and this is the entry point for a normal return.
4869 An actual EH return updates the return address, initializes the stack
4870 adjustment and jumps directly into the epilogue (bypassing the zeroing
4871 of the adjustment). Since the return address is typically saved on the
4872 stack when a function makes a call, the saved LR must be updated outside
4873 the epilogue.
4875 This poses problems as the store is generated well before the epilogue,
4876 so the offset of LR is not known yet. Also optimizations will remove the
4877 store as it appears dead, even after the epilogue is generated (as the
4878 base or offset for loading LR is different in many cases).
4880 To avoid these problems this implementation forces the frame pointer
4881 in eh_return functions so that the location of LR is fixed and known early.
4882 It also marks the store volatile, so no optimization is permitted to
4883 remove the store. */
4884 rtx
4886 {
4890 /* Mark the store volatile, so no optimization is permitted to remove it. */
4893 }
4895 /* Output code to add DELTA to the first argument, and then jump
4896 to FUNCTION. Used for C++ multiple inheritance. */
4897 static void
4899 HOST_WIDE_INT delta,
4900 HOST_WIDE_INT vcall_offset,
4901 tree function)
4902 {
4903 /* The this pointer is always in x0. Note that this differs from
4904 Arm where the this pointer maybe bumped to r1 if r0 is required
4905 to return a pointer to an aggregate. On AArch64 a result value
4906 pointer will be in x8. */
4920 else
4921 {
4926 {
4930 else
4933 }
4937 else
4944 else
4945 {
4947 Pmode);
4949 }
4953 else
4959 }
4961 /* Generate a tail call to the target function. */
4963 {
4966 }
4978 /* Stop pretending to be a post-reload pass. */
4980 }
4982 static bool
4984 {
4989 {
4993 /* Don't recurse into UNSPEC_TLS looking for TLS symbols; these are
4994 TLS offsets, not real symbol references. */
4997 }
4999 }
5002 /* Return true if val can be encoded as a 12-bit unsigned immediate with
5003 a left shift of 0 or 12 bits. */
5004 bool
5006 {
5009 );
5010 }
5013 /* Return true if val is an immediate that can be loaded into a
5014 register by a MOVZ instruction. */
5015 static bool
5017 {
5019 {
5023 }
5024 else
5025 {
5026 /* Ignore sign extension. */
5028 }
5031 }
5033 /* VAL is a value with the inner mode of MODE. Replicate it to fill a
5034 64-bit (DImode) integer. */
5036 static unsigned HOST_WIDE_INT
5038 {
5041 {
5045 }
5047 }
5049 /* Multipliers for repeating bitmasks of width 32, 16, 8, 4, and 2. */
5052 {
5058 };
5061 /* Return true if val is a valid bitmask immediate. */
5063 bool
5065 {
5069 /* Check for a single sequence of one bits and return quickly if so.
5070 The special cases of all ones and all zeroes returns false. */
5077 /* Replicate 32-bit immediates so we can treat them as 64-bit. */
5081 /* Invert if the immediate doesn't start with a zero bit - this means we
5082 only need to search for sequences of one bits. */
5086 /* Find the first set bit and set tmp to val with the first sequence of one
5087 bits removed. Return success if there is a single sequence of ones. */
5094 /* Find the next set bit and compute the difference in bit position. */
5099 /* Check the bit position difference is a power of 2, and that the first
5100 sequence of one bits fits within 'bits' bits. */
5104 /* Check the sequence of one bits is repeated 64/bits times. */
5106 }
5108 /* Create mask of ones, covering the lowest to highest bits set in VAL_IN.
5109 Assumed precondition: VAL_IN Is not zero. */
5111 unsigned HOST_WIDE_INT
5113 {
5120 }
5122 /* Create constant where bits outside of lowest bit set to highest bit set
5123 are set to 1. */
5125 unsigned HOST_WIDE_INT
5127 {
5129 }
5131 /* Return true if VAL_IN is a valid 'and' bitmask immediate. */
5133 bool
5135 {
5136 scalar_int_mode int_mode;
5149 }
5151 /* Return true if val is an immediate that can be loaded into a
5152 register in a single instruction. */
5153 bool
5155 {
5156 scalar_int_mode int_mode;
5163 }
5165 static bool
5167 {
5173 /* There's no way to calculate VL-based values using relocations. */
5181 {
5185 else
5186 /* Avoid generating a 64-bit relocation in ILP32; leave
5187 to aarch64_expand_mov_immediate to handle it properly. */
5189 }
5192 }
5194 /* Implement TARGET_CASE_VALUES_THRESHOLD.
5195 The expansion for a table switch is quite expensive due to the number
5196 of instructions, the table lookup and hard to predict indirect jump.
5197 When optimizing for speed, and -O3 enabled, use the per-core tuning if
5198 set, otherwise use tables for > 16 cases as a tradeoff between size and
5199 performance. When optimizing for size, use the default setting. */
5201 static unsigned int
5203 {
5204 /* Use the specified limit for the number of cases before using jump
5205 tables at higher optimization levels. */
5209 else
5211 }
5213 /* Return true if register REGNO is a valid index register.
5214 STRICT_P is true if REG_OK_STRICT is in effect. */
5216 bool
5218 {
5220 {
5228 }
5230 }
5232 /* Return true if register REGNO is a valid base register for mode MODE.
5233 STRICT_P is true if REG_OK_STRICT is in effect. */
5235 bool
5237 {
5239 {
5247 }
5249 /* The fake registers will be eliminated to either the stack or
5250 hard frame pointer, both of which are usually valid base registers.
5251 Reload deals with the cases where the eliminated form isn't valid. */
5256 }
5258 /* Return true if X is a valid base register for mode MODE.
5259 STRICT_P is true if REG_OK_STRICT is in effect. */
5261 static bool
5263 {
5270 }
5272 /* Return true if address offset is a valid index. If it is, fill in INFO
5273 appropriately. STRICT_P is true if REG_OK_STRICT is in effect. */
5275 static bool
5278 {
5280 rtx index;
5283 /* (reg:P) */
5286 {
5290 }
5291 /* (sign_extend:DI (reg:SI)) */
5296 {
5301 }
5302 /* (mult:DI (sign_extend:DI (reg:SI)) (const_int scale)) */
5309 {
5314 }
5315 /* (ashift:DI (sign_extend:DI (reg:SI)) (const_int shift)) */
5322 {
5327 }
5328 /* (sign_extract:DI (mult:DI (reg:DI) (const_int scale)) 32+shift 0) */
5335 {
5343 }
5344 /* (and:DI (mult:DI (reg:DI) (const_int scale))
5345 (const_int 0xffffffff<<shift)) */
5352 {
5358 }
5359 /* (sign_extract:DI (ashift:DI (reg:DI) (const_int shift)) 32+shift 0) */
5366 {
5374 }
5375 /* (and:DI (ashift:DI (reg:DI) (const_int shift))
5376 (const_int 0xffffffff<<shift)) */
5383 {
5389 }
5390 /* (mult:P (reg:P) (const_int scale)) */
5395 {
5399 }
5400 /* (ashift:P (reg:P) (const_int shift)) */
5405 {
5409 }
5410 else
5419 {
5423 }
5424 else
5425 {
5430 }
5434 {
5439 }
5442 }
5444 /* Return true if MODE is one of the modes for which we
5445 support LDP/STP operations. */
5447 static bool
5449 {
5454 }
5456 /* Return true if REGNO is a virtual pointer register, or an eliminable
5457 "soft" frame register. Like REGNO_PTR_FRAME_P except that we don't
5458 include stack_pointer or hard_frame_pointer. */
5459 static bool
5461 {
5466 }
5468 /* Return true if X is a valid address of type TYPE for machine mode MODE.
5469 If it is, fill in INFO appropriately. STRICT_P is true if
5470 REG_OK_STRICT is in effect. */
5472 static bool
5476 {
5479 poly_int64 offset;
5481 HOST_WIDE_INT const_size;
5483 /* On BE, we use load/store pair for all large int mode load/stores.
5484 TI/TFmode may also use a load/store pair. */
5497 /* For SVE, only accept [Rn], [Rn, Rm, LSL #shift] and
5498 [Rn, #offset, MUL VL]. */
5503 /* On LE, for AdvSIMD, don't support anything other than POST_INC or
5504 REG addressing. */
5506 && !BYTES_BIG_ENDIAN
5514 {
5531 {
5538 }
5543 {
5549 /* TImode and TFmode values are allowed in both pairs of X
5550 registers and individual Q registers. The available
5551 address modes are:
5552 X,X: 7-bit signed scaled offset
5553 Q: 9-bit signed offset
5554 We conservatively require an offset representable in either mode.
5555 When performing the check for pairs of X registers i.e. LDP/STP
5556 pass down DImode since that is the natural size of the LDP/STP
5557 instruction memory accesses. */
5563 /* A 7bit offset check because OImode will emit a ldp/stp
5564 instruction (only big endian will get here).
5565 For ldp/stp instructions, the offset is scaled for the size of a
5566 single element of the pair. */
5570 /* Three 9/12 bit offsets checks because CImode will emit three
5571 ldr/str instructions (only big endian will get here). */
5578 /* Two 7bit offsets checks because XImode will emit two ldp/stp
5579 instructions (only big endian will get here). */
5585 /* Make "m" use the LD1 offset range for SVE data modes, so
5586 that pre-RTL optimizers like ivopts will work to that
5587 instead of the wider LDR/STR range. */
5600 else
5603 }
5606 {
5607 /* Look for base + (scaled/extended) index register. */
5610 {
5613 }
5616 {
5619 }
5620 }
5641 {
5645 /* TImode and TFmode values are allowed in both pairs of X
5646 registers and individual Q registers. The available
5647 address modes are:
5648 X,X: 7-bit signed scaled offset
5649 Q: 9-bit signed offset
5650 We conservatively require an offset representable in either mode.
5651 */
5660 else
5662 }
5668 /* load literal: pc-relative constant pool entry. Only supported
5669 for SI mode or larger. */
5675 {
5683 }
5692 {
5698 {
5699 /* The symbol and offset must be aligned to the access size. */
5705 {
5709 }
5715 else
5724 }
5725 }
5730 }
5731 }
5733 /* Return true if the address X is valid for a PRFM instruction.
5734 STRICT_P is true if we should do strict checking with
5735 aarch64_classify_address. */
5737 bool
5739 {
5742 /* PRFM accepts the same addresses as DImode... */
5747 /* ... except writeback forms. */
5749 }
5751 bool
5753 {
5754 rtx offset;
5758 }
5760 /* Classify the base of symbolic expression X. */
5762 enum aarch64_symbol_type
5764 {
5765 rtx offset;
5769 }
5772 /* Return TRUE if X is a legitimate address for accessing memory in
5773 mode MODE. */
5774 static bool
5776 {
5780 }
5782 /* Return TRUE if X is a legitimate address of type TYPE for accessing
5783 memory in mode MODE. STRICT_P is true if REG_OK_STRICT is in effect. */
5784 bool
5786 aarch64_addr_query_type type)
5787 {
5791 }
5793 /* Split an out-of-range address displacement into a base and offset.
5794 Use 4KB range for 1- and 2-byte accesses and a 16KB range otherwise
5795 to increase opportunities for sharing the base address of different sizes.
5796 Unaligned accesses use the signed 9-bit range, TImode/TFmode use
5797 the intersection of signed scaled 7-bit and signed 9-bit offset. */
5798 static bool
5800 {
5801 HOST_WIDE_INT size;
5803 {
5805 HOST_WIDE_INT base;
5811 else
5817 }
5819 }
5821 /* Return the binary representation of floating point constant VALUE in INTVAL.
5822 If the value cannot be converted, return false without setting INTVAL.
5823 The conversion is done in the given MODE. */
5824 bool
5826 {
5828 /* We make a general exception for 0. */
5830 {
5833 }
5835 scalar_float_mode mode;
5839 /* Only support up to DF mode. */
5851 {
5855 }
5856 else
5861 }
5863 /* Return TRUE if rtx X is an immediate constant that can be moved using a
5864 single MOV(+MOVK) followed by an FMOV. */
5865 bool
5867 {
5872 /* Determine whether it's cheaper to write float constants as
5873 mov/movk pairs over ldr/adrp pairs. */
5879 {
5881 ? SImode
5886 }
5889 }
5891 /* Return TRUE if rtx X is immediate constant 0.0 */
5892 bool
5894 {
5901 }
5903 /* Return TRUE if rtx X is immediate constant that fits in a single
5904 MOVI immediate operation. */
5905 bool
5907 {
5911 machine_mode vmode;
5912 scalar_int_mode imode;
5917 {
5921 /* We make a general exception for 0. */
5926 }
5930 else
5933 /* use a 64 bit mode for everything except for DI/DF mode, where we use
5934 a 128 bit vector mode. */
5941 }
5944 /* Return the fixed registers used for condition codes. */
5946 static bool
5948 {
5952 }
5954 /* This function is used by the call expanders of the machine description.
5955 RESULT is the register in which the result is returned. It's NULL for
5956 "call" and "sibcall".
5957 MEM is the location of the function call.
5958 SIBCALL indicates whether this function call is normal call or sibling call.
5959 It will generate different pattern accordingly. */
5961 void
5963 {
5965 rtvec vec;
5966 machine_mode mode;
5973 /* Decide if we should generate indirect calls by loading the
5974 address of the callee into a register before performing
5975 the branch-and-link. */
5989 else
5996 }
5998 /* Emit call insn with PAT and do aarch64-specific handling. */
6000 void
6002 {
6008 }
6010 machine_mode
6012 {
6013 /* All floating point compares return CCFP if it is an equality
6014 comparison, and CCFPE otherwise. */
6016 {
6018 {
6039 }
6040 }
6042 /* Equality comparisons of short modes against zero can be performed
6043 using the TST instruction with the appropriate bitmask. */
6049 /* Similarly, comparisons of zero_extends from shorter modes can
6050 be performed using an ANDS with an immediate mask. */
6066 /* A compare with a shifted operand. Because of canonicalization,
6067 the comparison will have to be swapped when we emit the assembly
6068 code. */
6076 /* Similarly for a negated operand, but we can only do this for
6077 equalities. */
6084 /* A test for unsigned overflow. */
6091 /* For everything else, return CCmode. */
6093 }
6095 static int
6098 int
6100 {
6107 }
6109 static int
6111 {
6113 {
6117 {
6131 }
6136 {
6148 }
6153 {
6165 }
6170 {
6176 }
6181 {
6185 }
6190 {
6194 }
6199 }
6202 }
6204 bool
6206 HOST_WIDE_INT minval,
6207 HOST_WIDE_INT maxval)
6208 {
6209 rtx elt;
6213 }
6215 bool
6217 {
6219 }
6221 /* Return true if VEC is a constant in which every element is in the range
6222 [MINVAL, MAXVAL]. The elements do not need to have the same value. */
6224 static bool
6226 HOST_WIDE_INT minval,
6227 HOST_WIDE_INT maxval)
6228 {
6240 {
6245 }
6247 }
6249 /* N Z C V. */
6250 #define AARCH64_CC_V 1
6251 #define AARCH64_CC_C (1 << 1)
6252 #define AARCH64_CC_Z (1 << 2)
6253 #define AARCH64_CC_N (1 << 3)
6255 /* N Z C V flags for ccmp. Indexed by AARCH64_COND_CODE. */
6257 {
6274 };
6276 /* Print floating-point vector immediate operand X to F, negating it
6277 first if NEGATE is true. Return true on success, false if it isn't
6278 a constant we can handle. */
6280 static bool
6282 {
6283 rtx elt;
6292 /* We only handle the SVE single-bit immediates here. */
6299 else
6303 }
6305 /* Print operand X to file F in a target specific manner according to CODE.
6306 The acceptable formatting commands given by CODE are:
6307 'c': An integer or symbol address without a preceding #
6308 sign.
6309 'C': Take the duplicated element in a vector constant
6310 and print it in hex.
6311 'D': Take the duplicated element in a vector constant
6312 and print it as an unsigned integer, in decimal.
6313 'e': Print the sign/zero-extend size as a character 8->b,
6314 16->h, 32->w.
6315 'p': Prints N such that 2^N == X (X must be power of 2 and
6316 const int).
6317 'P': Print the number of non-zero bits in X (a const_int).
6318 'H': Print the higher numbered register of a pair (TImode)
6319 of regs.
6320 'm': Print a condition (eq, ne, etc).
6321 'M': Same as 'm', but invert condition.
6322 'N': Take the duplicated element in a vector constant
6323 and print the negative of it in decimal.
6324 'b/h/s/d/q': Print a scalar FP/SIMD register name.
6325 'S/T/U/V': Print a FP/SIMD register name for a register list.
6326 The register printed is the FP/SIMD register name
6327 of X + 0/1/2/3 for S/T/U/V.
6328 'R': Print a scalar FP/SIMD register name + 1.
6329 'X': Print bottom 16 bits of integer constant in hex.
6330 'w/x': Print a general register name or the zero register
6331 (32-bit or 64-bit).
6332 '0': Print a normal operand, if it's a general register,
6333 then we assume DImode.
6334 'k': Print NZCV for conditional compare instructions.
6335 'A': Output address constant representing the first
6336 argument of X, specifying a relocation offset
6337 if appropriate.
6338 'L': Output constant address specified by X
6339 with a relocation offset if appropriate.
6340 'G': Prints address of X, specifying a PC relative
6341 relocation mode if appropriate.
6342 'y': Output address of LDP or STP - this is used for
6343 some LDP/STPs which don't use a PARALLEL in their
6344 pattern (so the mode needs to be adjusted).
6345 'z': Output address of a typical LDP or STP. */
6347 static void
6349 {
6350 rtx elt;
6352 {
6355 {
6367 {
6370 }
6371 /* Fall through. */
6375 }
6379 {
6384 {
6387 }
6390 {
6403 }
6404 }
6408 {
6412 {
6415 }
6418 }
6423 {
6426 }
6433 {
6436 }
6443 {
6445 /* CONST_TRUE_RTX means al/nv (al is the default, don't print it). */
6447 {
6451 }
6454 {
6457 }
6464 }
6469 {
6472 }
6478 ;
6479 else
6480 {
6483 }
6492 {
6493 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6495 }
6504 {
6505 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6507 }
6515 {
6516 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6518 }
6524 {
6527 }
6532 {
6533 /* Print a replicated constant in hex. */
6535 {
6538 }
6541 }
6545 {
6546 /* Print a replicated constant in decimal, treating it as
6547 unsigned. */
6549 {
6552 }
6555 }
6562 {
6565 }
6568 {
6571 }
6574 {
6577 }
6579 /* Fall through */
6583 {
6586 }
6589 {
6593 else
6612 {
6615 }
6616 /* fall through */
6620 {
6623 }
6629 ;
6630 else
6631 {
6634 }
6638 /* Since we define TARGET_SUPPORTS_WIDE_INT we shouldn't ever
6639 be getting CONST_DOUBLEs holding integers. */
6642 {
6645 }
6647 {
6648 #define buf_size 20
6656 #undef buf_size
6657 }
6663 }
6671 {
6698 }
6704 {
6739 }
6745 {
6751 }
6756 {
6757 HOST_WIDE_INT cond_code;
6760 {
6763 }
6768 }
6773 {
6778 {
6781 }
6784 /* LDP/STP which uses a single double-width memory operand.
6785 Adjust the mode to appear like a typical LDP/STP.
6786 Currently this is supported for 16-byte accesses only. */
6791 }
6797 }
6798 }
6800 /* Print address 'x' of a memory access with mode 'mode'.
6801 'op' is the context required by aarch64_classify_address. It can either be
6802 MEM for a normal memory access or PARALLEL for LDP/STP. */
6803 static bool
6805 aarch64_addr_query_type type)
6806 {
6810 /* Check all addresses are Pmode - including ILP32. */
6815 {
6820 {
6821 HOST_WIDE_INT vnum
6826 }
6828 {
6829 HOST_WIDE_INT vnum
6834 }
6835 else
6844 else
6853 else
6862 else
6868 /* Writeback is only supported for fixed-width modes. */
6871 {
6894 }
6906 }
6909 }
6911 /* Print address 'x' of a LDP/STP with mode 'mode'. */
6912 static bool
6914 {
6916 }
6918 /* Print address 'x' of a memory access with mode 'mode'. */
6919 static void
6921 {
6924 }
6926 bool
6928 {
6935 /* UNSPEC_TLS entries for a symbol include a LABEL_REF for the
6936 referencing instruction, but they are constant offsets, not
6937 symbols. */
6943 {
6945 {
6951 }
6954 }
6957 }
6959 /* Implement REGNO_REG_CLASS. */
6961 enum reg_class
6963 {
6981 }
6983 /* OFFSET is an address offset for mode MODE, which has SIZE bytes.
6984 If OFFSET is out of range, return an offset of an anchor point
6985 that is in range. Return 0 otherwise. */
6987 static HOST_WIDE_INT
6989 machine_mode mode)
6990 {
6991 /* Does it look like we'll need a 16-byte load/store-pair operation? */
6995 /* For offsets that aren't a multiple of the access size, the limit is
6996 -256...255. */
6998 {
6999 /* BLKmode typically uses LDP of X-registers. */
7003 }
7005 /* Small negative offsets are supported. */
7012 /* Use 12-bit offset by access size. */
7014 }
7016 static rtx
7018 {
7019 /* Try to split X+CONST into Y=X+(CONST & ~mask), Y+(CONST&mask),
7020 where mask is selected by alignment and size of the offset.
7021 We try to pick as large a range for the offset as possible to
7022 maximize the chance of a CSE. However, for aligned addresses
7023 we limit the range to 4k so that structures with different sized
7024 elements are likely to use the same base. We need to be careful
7025 not to split a CONST for some forms of address expression, otherwise
7026 it will generate sub-optimal code. */
7029 {
7035 {
7039 /* Force any scaling into a temp for CSE. */
7043 /* Let the pointer register be in op0. */
7047 /* If the pointer is virtual or frame related, then we know that
7048 virtual register instantiation or register elimination is going
7049 to apply a second constant. We want the two constants folded
7050 together easily. Therefore, emit as (OP0 + CONST) + OP1. */
7052 {
7056 }
7058 /* Otherwise, in order to encourage CSE (and thence loop strength
7059 reduce) scaled addresses, emit as (OP0 + OP1) + CONST. */
7063 }
7065 HOST_WIDE_INT size;
7067 {
7069 mode);
7071 {
7075 }
7076 }
7077 }
7080 }
7082 /* Return the reload icode required for a constant pool in mode. */
7083 static enum insn_code
7085 {
7087 {
7123 }
7126 }
7127 static reg_class_t
7129 reg_class_t rclass,
7130 machine_mode mode,
7132 {
7137 {
7140 }
7142 /* If we have to disable direct literal pool loads and stores because the
7143 function is too big, then we need a scratch register. */
7148 {
7151 }
7153 /* Without the TARGET_SIMD instructions we cannot move a Q register
7154 to a Q register directly. We need a scratch. */
7158 {
7164 }
7166 /* A TFmode or TImode memory access should be handled via an FP_REGS
7167 because AArch64 has richer addressing modes for LDR/STR instructions
7168 than LDP/STP instructions. */
7177 }
7179 static bool
7181 {
7184 /* If we need a frame pointer, ARG_POINTER_REGNUM and FRAME_POINTER_REGNUM
7185 can only eliminate to HARD_FRAME_POINTER_REGNUM. */
7189 }
7191 poly_int64
7193 {
7197 {
7204 }
7207 {
7211 }
7214 }
7216 /* Implement RETURN_ADDR_RTX. We do not support moving back to a
7217 previous frame. */
7219 rtx
7221 {
7225 }
7228 static void
7230 {
7232 {
7235 }
7236 else
7237 {
7240 }
7245 }
7247 static void
7249 {
7253 /* Don't need to copy the trailing D-words, we fill those in below. */
7265 /* XXX We should really define a "clear_cache" pattern and use
7266 gen_clear_cache(). */
7271 ptr_mode);
7272 }
7274 static unsigned char
7276 {
7277 /* ??? Logically we should only need to provide a value when
7278 HARD_REGNO_MODE_OK says that at least one register in REGCLASS
7279 can hold MODE, but at the moment we need to handle all modes.
7280 Just ignore any runtime parts for registers that can't store them. */
7284 {
7310 }
7312 }
7314 static reg_class_t
7316 {
7321 {
7327 }
7329 /* Register eliminiation can result in a request for
7330 SP+constant->FP_REGS. We cannot support such operations which
7331 use SP as source and an FP_REG as destination, so reject out
7332 right now. */
7334 {
7337 /* Look through a possible SUBREG introduced by ILP32. */
7343 POINTER_REGS));
7345 }
7348 }
7350 void
7352 {
7354 }
7356 static void
7358 {
7361 else
7362 {
7364 /* While priority is known to be in range [0, 65535], so 18 bytes
7365 would be enough, the compiler might not know that. To avoid
7366 -Wformat-truncation false positive, use a larger size. */
7373 }
7374 }
7376 static void
7378 {
7381 else
7382 {
7384 /* While priority is known to be in range [0, 65535], so 18 bytes
7385 would be enough, the compiler might not know that. To avoid
7386 -Wformat-truncation false positive, use a larger size. */
7393 }
7394 }
7398 {
7404 {
7405 {
7408 },
7409 {
7412 },
7413 {
7416 },
7417 /* We assume that DImode is only generated when not optimizing and
7418 that we don't really need 64-bit address offsets. That would
7419 imply an object file with 8GB of code in a single function! */
7420 {
7423 }
7424 };
7433 /* Need to implement table size reduction, by chaning the code below. */
7443 }
7446 /* Return size in bits of an arithmetic operand which is shifted/scaled and
7447 masked such that it is suitable for a UXTB, UXTH, or UXTW extend
7448 operator. */
7450 int
7452 {
7454 {
7457 {
7461 }
7462 }
7464 }
7466 /* Constant pools are per function only when PC relative
7467 literal loads are true or we are in the large memory
7468 model. */
7472 {
7475 }
7477 static bool
7479 {
7480 /* Fixme:: In an ideal world this would work similar
7481 to the logic in aarch64_select_rtx_section but this
7482 breaks bootstrap in gcc go. For now we workaround
7483 this by returning false here. */
7485 }
7487 /* Select appropriate section for constants depending
7488 on where we place literal pools. */
7492 rtx x,
7494 {
7499 }
7501 /* Implement ASM_OUTPUT_POOL_EPILOGUE. */
7502 void
7504 HOST_WIDE_INT offset)
7505 {
7506 /* When using per-function literal pools, we must ensure that any code
7507 section is aligned to the minimal instruction length, lest we get
7508 errors from the assembler re "unaligned instructions". */
7511 }
7513 /* Costs. */
7515 /* Helper function for rtx cost calculation. Strip a shift expression
7516 from X. Returns the inner operand if successful, or the original
7517 expression on failure. */
7518 static rtx
7520 {
7523 /* We accept both ROTATERT and ROTATE: since the RHS must be a constant
7524 we can convert both to ROR during final output. */
7539 }
7541 /* Helper function for rtx cost calculation. Strip an extend
7542 expression from X. Returns the inner operand if successful, or the
7543 original expression on failure. We deal with a number of possible
7544 canonicalization variations here. If STRIP_SHIFT is true, then
7545 we can strip off a shift also. */
7546 static rtx
7548 {
7549 scalar_int_mode mode;
7555 /* Zero and sign extraction of a widened value. */
7563 /* It can also be represented (for zero-extend) as an AND with an
7564 immediate. */
7573 /* Now handle extended register, as this may also have an optional
7574 left shift by 1..4. */
7589 }
7591 /* Return true iff CODE is a shift supported in combination
7592 with arithmetic instructions. */
7594 static bool
7596 {
7598 }
7601 /* Return true iff X is a cheap shift without a sign extend. */
7603 static bool
7605 {
7631 }
7633 /* Helper function for rtx cost calculation. Calculate the cost of
7634 a MULT or ASHIFT, which may be part of a compound PLUS/MINUS rtx.
7635 Return the calculated cost of the expression, recursing manually in to
7636 operands where needed. */
7638 static int
7640 {
7656 /* Integer multiply/fma. */
7658 {
7659 /* The multiply will be canonicalized as a shift, cost it as such. */
7663 {
7667 {
7669 {
7670 /* If the shift is considered cheap,
7671 then don't add any cost. */
7673 ;
7675 /* ARITH + shift-by-register. */
7678 /* ARITH + extended register. We don't have a cost field
7679 for ARITH+EXTEND+SHIFT, so use extend_arith here. */
7681 else
7682 /* ARITH + shift-by-immediate. */
7684 }
7685 else
7686 /* LSL (immediate). */
7689 }
7690 /* Strip extends as we will have costed them in the case above. */
7697 }
7699 /* MNEG or [US]MNEGL. Extract the NEG operand and indicate that it's a
7700 compound and let the below cases handle it. After all, MNEG is a
7701 special-case alias of MSUB. */
7703 {
7706 }
7708 /* Integer multiplies or FMAs have zero/sign extending variants. */
7713 {
7718 {
7720 /* SMADDL/UMADDL/UMSUBL/SMSUBL. */
7722 else
7723 /* MUL/SMULL/UMULL. */
7725 }
7728 }
7730 /* This is either an integer multiply or a MADD. In both cases
7731 we want to recurse and cost the operands. */
7736 {
7738 /* MADD/MSUB. */
7740 else
7741 /* MUL. */
7743 }
7746 }
7747 else
7748 {
7750 {
7751 /* Floating-point FMA/FMUL can also support negations of the
7752 operands, unless the rounding mode is upward or downward in
7753 which case FNMUL is different than FMUL with operand negation. */
7757 {
7762 }
7765 /* FMADD/FNMADD/FNMSUB/FMSUB. */
7767 else
7768 /* FMUL/FNMUL. */
7770 }
7775 }
7776 }
7778 static int
7780 machine_mode mode,
7781 addr_space_t as ATTRIBUTE_UNUSED,
7783 {
7791 {
7793 {
7794 /* This is a CONST or SYMBOL ref which will be split
7795 in a different way depending on the code model in use.
7796 Cost it through the generic infrastructure. */
7798 /* Divide through by the cost of one instruction to
7799 bring it to the same units as the address costs. */
7801 /* The cost is then the cost of preparing the address,
7802 followed by an immediate (possibly 0) offset. */
7804 }
7805 else
7806 {
7807 /* This is most likely a jump table from a case
7808 statement. */
7810 }
7811 }
7814 {
7826 else
7845 }
7849 {
7850 /* For the sake of calculating the cost of the shifted register
7851 component, we can treat same sized modes in the same way. */
7858 else
7859 /* We can't tell, or this is a 128-bit vector. */
7861 }
7864 }
7866 /* Return the cost of a branch. If SPEED_P is true then the compiler is
7867 optimizing for speed. If PREDICTABLE_P is true then the branch is predicted
7868 to be taken. */
7870 int
7872 {
7873 /* When optimizing for speed, use the cost of unpredictable branches. */
7879 else
7881 }
7883 /* Return true if the RTX X in mode MODE is a zero or sign extract
7884 usable in an ADD or SUB (extended register) instruction. */
7885 static bool
7887 {
7888 /* Catch add with a sign extract.
7889 This is add_<optab><mode>_multp2. */
7892 {
7903 op1))
7904 {
7906 }
7907 }
7908 /* The simple case <ARITH>, XD, XN, XM, [us]xt.
7909 No shift. */
7915 }
7917 static bool
7919 {
7921 {
7933 }
7934 }
7936 /* Return true iff X is an rtx that will match an extr instruction
7937 i.e. as described in the *extr<mode>5_insn family of patterns.
7938 OP0 and OP1 will be set to the operands of the shifts involved
7939 on success and will be NULL_RTX otherwise. */
7941 static bool
7943 {
7945 scalar_int_mode mode;
7960 {
7961 /* Canonicalise locally to ashift in op0, lshiftrt in op1. */
7973 {
7977 }
7978 }
7981 }
7983 /* Calculate the cost of calculating (if_then_else (OP0) (OP1) (OP2)),
7984 storing it in *COST. Result is true if the total cost of the operation
7985 has now been calculated. */
7986 static bool
7988 {
7989 rtx inner;
7990 rtx comparator;
7994 {
7998 }
7999 else
8000 {
8004 }
8007 {
8008 /* Conditional branch. */
8011 else
8012 {
8014 {
8016 {
8017 /* TBZ/TBNZ/CBZ/CBNZ. */
8019 /* TBZ/TBNZ. */
8022 else
8023 /* CBZ/CBNZ. */
8027 }
8028 }
8030 {
8031 /* TBZ/TBNZ. */
8034 }
8035 }
8036 }
8038 {
8039 /* CCMP. */
8041 {
8042 /* Increase cost of CCMP reg, 0, imm, CC to prefer CMP reg, 0. */
8046 {
8053 else
8055 }
8057 }
8059 /* It's a conditional operation based on the status flags,
8060 so it must be some flavor of CSEL. */
8062 /* CSNEG, CSINV, and CSINC are handled for free as part of CSEL. */
8068 {
8069 /* CSEL with zero-extension (*cmovdi_insn_uxtw). */
8072 }
8077 }
8079 /* We don't know what this is, cost all operands. */
8081 }
8083 /* Check whether X is a bitfield operation of the form shift + extend that
8084 maps down to a UBFIZ/SBFIZ/UBFX/SBFX instruction. If so, return the
8085 operand to which the bitfield operation is applied. Otherwise return
8086 NULL_RTX. */
8088 static rtx
8090 {
8104 {
8122 }
8125 }
8127 /* Return true if the mask and a shift amount from an RTX of the form
8128 (x << SHFT_AMNT) & MASK are valid to combine into a UBFIZ instruction of
8129 mode MODE. See the *andim_ashift<mode>_bfiz pattern. */
8131 bool
8133 rtx shft_amnt)
8134 {
8139 }
8141 /* Calculate the cost of calculating X, storing it in *COST. Result
8142 is true if the total cost of the operation has now been calculated. */
8143 static bool
8146 {
8151 scalar_int_mode int_mode;
8153 /* By default, assume that everything has equivalent cost to the
8154 cheapest instruction. Any additional costs are applied as a delta
8155 above this default. */
8159 {
8161 /* The cost depends entirely on the operands to SET. */
8167 {
8170 {
8184 }
8193 /* Fall through. */
8195 /* The cost is one per vector-register copied. */
8197 {
8200 }
8201 /* const0_rtx is in general free, but we will use an
8202 instruction to set a register to 0. */
8204 {
8205 /* The cost is 1 per register copied. */
8208 }
8209 else
8210 /* Cost is just the cost of the RHS of the set. */
8216 /* Bit-field insertion. Strip any redundant widening of
8217 the RHS to meet the width of the target. */
8228 {
8229 /* MOV immediate is assumed to always be cheap. */
8231 }
8232 else
8233 {
8234 /* BFM. */
8238 }
8243 /* We can't make sense of this, assume default cost. */
8246 }
8250 /* If an instruction can incorporate a constant within the
8251 instruction, the instruction's expression avoids calling
8252 rtx_cost() on the constant. If rtx_cost() is called on a
8253 constant, then it is usually because the constant must be
8254 moved into a register by one or more instructions.
8256 The exception is constant 0, which can be expressed
8257 as XZR/WZR and is therefore free. The exception to this is
8258 if we have (set (reg) (const0_rtx)) in which case we must cost
8259 the move. However, we can catch that when we cost the SET, so
8260 we don't need to consider that here. */
8263 else
8264 {
8265 /* To an approximation, building any other constant is
8266 proportionally expensive to the number of instructions
8267 required to build that constant. This is true whether we
8268 are compiling for SPEED or otherwise. */
8273 }
8278 /* First determine number of instructions to do the move
8279 as an integer constant. */
8283 {
8289 ? SImode
8295 }
8298 {
8299 /* mov[df,sf]_aarch64. */
8301 /* FMOV (scalar immediate). */
8304 {
8305 /* This will be a load from memory. */
8308 else
8310 }
8311 else
8312 /* Otherwise this is +0.0. We get this using MOVI d0, #0
8313 or MOV v0.s[0], wzr - neither of which are modeled by the
8314 cost tables. Just use the default cost. */
8315 {
8316 }
8317 }
8323 {
8324 /* For loads we want the base cost of a load, plus an
8325 approximation for the additional cost of the addressing
8326 mode. */
8340 }
8348 {
8350 {
8351 /* FNEG. */
8353 }
8355 }
8358 {
8361 {
8362 /* CSETM. */
8365 }
8367 /* Cost this as SUB wzr, X. */
8371 }
8374 {
8375 /* Support (neg(fma...)) as a single instruction only if
8376 sign of zeros is unimportant. This matches the decision
8377 making in aarch64.md. */
8379 {
8380 /* FNMADD. */
8383 }
8385 {
8386 /* FNMUL. */
8389 }
8391 /* FNEG. */
8394 }
8401 {
8404 else
8406 }
8416 {
8420 }
8423 {
8424 /* TODO: A write to the CC flags possibly costs extra, this
8425 needs encoding in the cost tables. */
8428 /* ANDS. */
8430 {
8433 }
8436 {
8437 /* ADDS (and CMN alias). */
8440 }
8443 {
8444 /* SUBS. */
8447 }
8452 {
8453 /* COMPARE of ZERO_EXTRACT form of TST-immediate.
8454 Handle it here directly rather than going to cost_logic
8455 since we know the immediate generated for the TST is valid
8456 so we can avoid creating an intermediate rtx for it only
8457 for costing purposes. */
8464 }
8467 {
8468 /* CMN. */
8475 }
8477 /* CMP.
8479 Compare can freely swap the order of operands, and
8480 canonicalization puts the more complex operation first.
8481 But the integer MINUS logic expects the shift/extend
8482 operation in op1. */
8485 {
8488 }
8490 }
8493 {
8494 /* FCMP. */
8499 {
8501 /* FCMP supports constant 0.0 for no extra cost. */
8503 }
8505 }
8508 {
8509 /* Vector compare. */
8514 {
8515 /* Vector cm (eq|ge|gt|lt|le) supports constant 0.0 for no extra
8516 cost. */
8518 }
8520 }
8524 {
8528 cost_minus:
8531 /* Detect valid immediates. */
8537 {
8539 /* SUB(S) (immediate). */
8542 }
8544 /* Look for SUB (extended register). */
8547 {
8555 }
8559 /* Cost this as an FMA-alike operation. */
8563 {
8566 speed);
8568 }
8573 {
8575 {
8576 /* Vector SUB. */
8578 }
8580 {
8581 /* SUB(S). */
8583 }
8585 {
8586 /* FSUB. */
8588 }
8589 }
8591 }
8594 {
8595 rtx new_op0;
8600 cost_plus:
8603 {
8604 /* CSINC. */
8608 }
8613 {
8617 /* ADD (immediate). */
8620 }
8624 /* Look for ADD (extended register). */
8627 {
8635 }
8637 /* Strip any extend, leave shifts behind as we will
8638 cost them through mult_cost. */
8643 {
8645 speed);
8647 }
8652 {
8654 {
8655 /* Vector ADD. */
8657 }
8659 {
8660 /* ADD. */
8662 }
8664 {
8665 /* FADD. */
8667 }
8668 }
8670 }
8676 {
8679 else
8681 }
8686 {
8690 {
8693 else
8695 }
8697 }
8700 {
8707 }
8708 /* Fall through. */
8711 cost_logic:
8716 {
8720 }
8728 {
8729 /* This is a UBFM/SBFM. */
8734 }
8737 {
8739 {
8740 /* We have a mask + shift version of a UBFIZ
8741 i.e. the *andim_ashift<mode>_bfiz pattern. */
8745 {
8752 }
8754 {
8755 /* We possibly get the immediate for free, this is not
8756 modelled. */
8763 }
8764 }
8765 else
8766 {
8769 /* Handle ORN, EON, or BIC. */
8775 /* If we had a shift on op0 then this is a logical-shift-
8776 by-register/immediate operation. Otherwise, this is just
8777 a logical operation. */
8779 {
8781 {
8782 /* Shift by immediate. */
8785 else
8787 }
8788 else
8790 }
8792 /* In both cases we want to cost both operands. */
8799 }
8800 }
8808 {
8809 /* Vector NOT. */
8812 }
8814 /* MVN-shifted-reg. */
8816 {
8823 }
8824 /* EON can have two forms: (xor (not a) b) but also (not (xor a b)).
8825 Handle the second form here taking care that 'a' in the above can
8826 be a shift. */
8828 {
8837 {
8840 else
8842 }
8845 }
8846 /* MVN. */
8855 /* If a value is written in SI mode, then zero extended to DI
8856 mode, the operation will in general be free as a write to
8857 a 'w' register implicitly zeroes the upper bits of an 'x'
8858 register. However, if this is
8860 (set (reg) (zero_extend (reg)))
8862 we must cost the explicit register move. */
8866 {
8869 /* If OP_COST is non-zero, then the cost of the zero extend
8870 is effectively the cost of the inner operation. Otherwise
8871 we have a MOV instruction and we take the cost from the MOV
8872 itself. This is true independently of whether we are
8873 optimizing for space or time. */
8878 }
8880 {
8881 /* All loads can zero extend to any size for free. */
8884 }
8888 {
8893 }
8896 {
8898 {
8899 /* UMOV. */
8901 }
8902 else
8903 {
8904 /* We generate an AND instead of UXTB/UXTH. */
8906 }
8907 }
8912 {
8913 /* LDRSH. */
8915 {
8922 }
8924 }
8928 {
8933 }
8936 {
8939 else
8941 }
8949 {
8951 {
8953 {
8954 /* Vector shift (immediate). */
8956 }
8957 else
8958 {
8959 /* LSL (immediate), UBMF, UBFIZ and friends. These are all
8960 aliases. */
8962 }
8963 }
8965 /* We can incorporate zero/sign extend for free. */
8972 }
8973 else
8974 {
8976 {
8978 /* Vector shift (register). */
8980 }
8981 else
8982 {
8984 /* LSLV. */
8991 {
8993 /* We already demanded XEXP (op1, 0) to be REG_P, so
8994 don't recurse into it. */
8996 }
8997 }
8999 }
9009 {
9010 /* ASR (immediate) and friends. */
9012 {
9015 else
9017 }
9021 }
9022 else
9023 {
9025 {
9027 /* Vector shift (register). */
9029 }
9030 else
9031 {
9033 /* ASR (register) and friends. */
9040 {
9042 /* We already demanded XEXP (op1, 0) to be REG_P, so
9043 don't recurse into it. */
9045 }
9046 }
9048 }
9054 {
9055 /* LDR. */
9058 }
9061 {
9062 /* ADRP, followed by ADD. */
9066 }
9069 {
9070 /* ADR. */
9073 }
9076 {
9077 /* One extra load instruction, after accessing the GOT. */
9081 }
9086 /* ADRP/ADD (immediate). */
9093 /* UBFX/SBFX. */
9095 {
9098 else
9100 }
9102 /* We can trust that the immediates used will be correct (there
9103 are no by-register forms), so we need only cost op0. */
9109 /* aarch64_rtx_mult_cost always handles recursion to its
9110 operands. */
9114 /* We can expand signed mod by power of 2 using a NEGS, two parallel
9115 ANDs and a CSNEG. Assume here that CSNEG is the same as the cost of
9116 an unconditional negate. This case should only ever be reached through
9117 the set_smod_pow2_cheap check in expmed.c. */
9121 {
9122 /* We expand to 4 instructions. Reset the baseline. */
9130 }
9132 /* Fall-through. */
9135 {
9136 /* Slighly prefer UMOD over SMOD. */
9143 }
9150 {
9154 /* There is no integer SQRT, so only DIV and UDIV can get
9155 here. */
9157 /* Slighly prefer UDIV over SDIV. */
9159 else
9161 }
9187 {
9190 else
9192 }
9194 /* FMSUB, FNMADD, and FNMSUB are free. */
9201 /* aarch64_fnma4_elt_to_64v2df has the NEG as operand 1,
9202 and the by-element operand as operand 0. */
9206 /* Catch vector-by-element operations. The by-element operand can
9207 either be (vec_duplicate (vec_select (x))) or just
9208 (vec_select (x)), depending on whether we are multiplying by
9209 a vector or a scalar.
9211 Canonicalization is not very good in these cases, FMA4 will put the
9212 by-element operand as operand 0, FNMA4 will have it as operand 1. */
9223 /* If the remaining parameters are not registers,
9224 get the cost to put them into registers. */
9238 {
9240 {
9241 /*Vector truncate. */
9243 }
9244 else
9246 }
9251 {
9253 {
9254 /*Vector conversion. */
9256 }
9257 else
9259 }
9265 /* Strip the rounding part. They will all be implemented
9266 by the fcvt* family of instructions anyway. */
9268 {
9277 }
9280 {
9283 else
9285 }
9287 /* We can combine fmul by a power of 2 followed by a fcvt into a single
9288 fixed-point fcvt. */
9293 {
9297 }
9304 {
9305 /* ABS (vector). */
9308 }
9310 {
9313 /* FABD, which is analogous to FADD. */
9315 {
9322 }
9323 /* Simple FABS is analogous to FNEG. */
9326 }
9327 else
9328 {
9329 /* Integer ABS will either be split to
9330 two arithmetic instructions, or will be an ABS
9331 (scalar), which we don't model. */
9335 }
9341 {
9344 else
9345 {
9346 /* FMAXNM/FMINNM/FMAX/FMIN.
9347 TODO: This may not be accurate for all implementations, but
9348 we do not model this in the cost tables. */
9350 }
9351 }
9355 /* The floating point round to integer frint* instructions. */
9357 {
9362 }
9365 {
9370 }
9375 /* Decompose <su>muldi3_highpart. */
9377 mode == DImode
9378 /* (lshiftrt:TI */
9381 /* (mult:TI */
9383 /* (ANY_EXTEND:TI (reg:DI))
9384 (ANY_EXTEND:TI (reg:DI))) */
9391 /* (const_int 64) */
9394 {
9395 /* UMULH/SMULH. */
9403 }
9405 /* Fall through. */
9408 }
9416 }
9418 /* Wrapper around aarch64_rtx_costs, dumps the partial, or total cost
9419 calculated for X. This cost is stored in *COST. Returns true
9420 if the total cost of X was calculated. */
9421 static bool
9424 {
9429 {
9434 }
9437 }
9439 static int
9442 {
9448 /* Caller save and pointer regs are equivalent to GENERAL_REGS. */
9455 /* Moving between GPR and stack cost is the same as GP2GP. */
9460 /* To/From the stack register, we move via the gprs. */
9466 {
9467 /* 128-bit operations on general registers require 2 instructions. */
9475 /* When AdvSIMD instructions are disabled it is not possible to move
9476 a 128-bit value directly between Q registers. This is handled in
9477 secondary reload. A general register is used as a scratch to move
9478 the upper DI value and the lower DI value is moved directly,
9479 hence the cost is the sum of three moves. */
9484 }
9494 }
9496 static int
9498 reg_class_t rclass ATTRIBUTE_UNUSED,
9500 {
9502 }
9504 /* Return true if it is safe and beneficial to use the approximate rsqrt optabs
9505 to optimize 1.0/sqrt. */
9507 static bool
9509 {
9511 && flag_unsafe_math_optimizations
9515 }
9517 /* Function to decide when to use the approximate reciprocal square root
9518 builtin. */
9520 static tree
9522 {
9528 }
9532 /* Select reciprocal square root initial estimate insn depending on machine
9533 mode. */
9535 static rsqrte_type
9537 {
9539 {
9546 }
9547 }
9551 /* Select reciprocal square root series step insn depending on machine mode. */
9553 static rsqrts_type
9555 {
9557 {
9564 }
9565 }
9567 /* Emit instruction sequence to compute either the approximate square root
9568 or its approximate reciprocal, depending on the flag RECP, and return
9569 whether the sequence was emitted or not. */
9571 bool
9573 {
9577 {
9580 }
9583 {
9590 || flag_trapping_math
9591 || !flag_unsafe_math_optimizations
9594 }
9595 else
9596 /* Caller assumes we cannot fail. */
9602 /* When calculating the approximate square root, compare the
9603 argument with 0.0 and create a mask. */
9609 /* Estimate the approximate reciprocal square root. */
9613 /* Iterate over the series twice for SF and thrice for DF. */
9616 /* Optionally iterate over the series once less for faster performance
9617 while sacrificing the accuracy. */
9620 iterations--;
9622 /* Iterate over the series to calculate the approximate reciprocal square
9623 root. */
9626 {
9634 }
9637 {
9638 /* Qualify the approximate reciprocal square root when the argument is
9639 0.0 by squashing the intermediary result to 0.0. */
9645 /* Calculate the approximate square root. */
9647 }
9649 /* Finalize the approximation. */
9653 }
9657 /* Select reciprocal initial estimate insn depending on machine mode. */
9659 static recpe_type
9661 {
9663 {
9670 }
9671 }
9675 /* Select reciprocal series step insn depending on machine mode. */
9677 static recps_type
9679 {
9681 {
9688 }
9689 }
9691 /* Emit the instruction sequence to compute the approximation for the division
9692 of NUM by DEN in QUO and return whether the sequence was emitted or not. */
9694 bool
9696 {
9707 || flag_trapping_math
9708 || !flag_unsafe_math_optimizations
9716 /* Estimate the approximate reciprocal. */
9720 /* Iterate over the series twice for SF and thrice for DF. */
9723 /* Optionally iterate over the series once less for faster performance,
9724 while sacrificing the accuracy. */
9726 iterations--;
9728 /* Iterate over the series to calculate the approximate reciprocal. */
9731 {
9736 }
9739 {
9740 /* As the approximate reciprocal of DEN is already calculated, only
9741 calculate the approximate division when NUM is not 1.0. */
9744 }
9746 /* Finalize the approximation. */
9749 }
9751 /* Return the number of instructions that can be issued per cycle. */
9752 static int
9754 {
9756 }
9758 static int
9760 {
9764 }
9767 /* Implement TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD as
9768 autopref_multipass_dfa_lookahead_guard from haifa-sched.c. It only
9769 has an effect if PARAM_SCHED_AUTOPREF_QUEUE_DEPTH > 0. */
9771 static int
9774 {
9776 }
9779 /* Vectorizer cost model target hooks. */
9781 /* Implement targetm.vectorize.builtin_vectorization_cost. */
9782 static int
9784 tree vectype,
9786 {
9795 {
9846 }
9847 }
9849 /* Implement targetm.vectorize.add_stmt_cost. */
9850 static unsigned
9854 {
9859 {
9864 /* Statements in an inner loop relative to the loop being
9865 vectorized are weighted more heavily. The value here is
9866 arbitrary and could potentially be improved with analysis. */
9872 }
9875 }
9879 /* Parse the TO_PARSE string and put the architecture struct that it
9880 selects into RES and the architectural features into ISA_FLAGS.
9881 Return an aarch64_parse_opt_result describing the parse result.
9882 If there is an error parsing, RES and ISA_FLAGS are left unchanged. */
9884 static enum aarch64_parse_opt_result
9887 {
9899 else
9906 /* Loop through the list of supported ARCHes to find a match. */
9908 {
9910 {
9914 {
9915 /* TO_PARSE string contains at least one extension. */
9916 enum aarch64_parse_opt_result ext_res
9921 }
9922 /* Extension parsing was successful. Confirm the result
9923 arch and ISA flags. */
9927 }
9928 }
9930 /* ARCH name not found in list. */
9932 }
9934 /* Parse the TO_PARSE string and put the result tuning in RES and the
9935 architecture flags in ISA_FLAGS. Return an aarch64_parse_opt_result
9936 describing the parse result. If there is an error parsing, RES and
9937 ISA_FLAGS are left unchanged. */
9939 static enum aarch64_parse_opt_result
9942 {
9954 else
9961 /* Loop through the list of supported CPUs to find a match. */
9963 {
9965 {
9970 {
9971 /* TO_PARSE string contains at least one extension. */
9972 enum aarch64_parse_opt_result ext_res
9977 }
9978 /* Extension parsing was successfull. Confirm the result
9979 cpu and ISA flags. */
9983 }
9984 }
9986 /* CPU name not found in list. */
9988 }
9990 /* Parse the TO_PARSE string and put the cpu it selects into RES.
9991 Return an aarch64_parse_opt_result describing the parse result.
9992 If the parsing fails the RES does not change. */
9994 static enum aarch64_parse_opt_result
9996 {
10002 /* Loop through the list of supported CPUs to find a match. */
10004 {
10006 {
10009 }
10010 }
10012 /* CPU name not found in list. */
10014 }
10016 /* Parse TOKEN, which has length LENGTH to see if it is an option
10017 described in FLAG. If it is, return the index bit for that fusion type.
10018 If not, error (printing OPTION_NAME) and return zero. */
10020 static unsigned int
10025 {
10027 {
10031 }
10035 }
10037 /* Parse OPTION which is a comma-separated list of flags to enable.
10038 FLAGS gives the list of flags we understand, INITIAL_STATE gives any
10039 default state we inherit from the CPU tuning structures. OPTION_NAME
10040 gives the top-level option we are parsing in the -moverride string,
10041 for use in error messages. */
10043 static unsigned int
10048 {
10055 {
10058 token_length,
10059 flags,
10060 option_name);
10061 /* If we find "none" (or, for simplicity's sake, an error) anywhere
10062 in the token stream, reset the supported operations. So:
10064 adrp+add.cmp+branch.none.adrp+add
10066 would have the result of turning on only adrp+add fusion. */
10072 }
10074 /* We ended with a comma, print something. */
10076 {
10079 }
10081 /* We still have one more token to parse. */
10084 token_length,
10085 flags,
10086 option_name);
10092 }
10094 /* Support for overriding instruction fusion. */
10096 static void
10099 {
10101 aarch64_fusible_pairs,
10104 }
10106 /* Support for overriding other tuning flags. */
10108 static void
10111 {
10112 tune->extra_tuning_flags
10114 aarch64_tuning_flags,
10117 }
10119 /* Parse TOKEN, which has length LENGTH to see if it is a tuning option
10120 we understand. If it is, extract the option string and handoff to
10121 the appropriate function. */
10123 void
10127 {
10133 {
10136 }
10138 /* Get the length of the option name. */
10140 /* Skip the '=' to get to the option string. */
10141 option_part++;
10144 {
10146 {
10149 }
10150 }
10154 }
10156 /* A checking mechanism for the implementation of the tls size. */
10158 static void
10160 {
10165 {
10167 /* Both the default and maximum TLS size allowed under tiny is 1M which
10168 needs two instructions to address, so we clamp the size to 24. */
10173 /* The maximum TLS size allowed under small is 4G. */
10178 /* The maximum TLS size allowed under large is 16E.
10179 FIXME: 16E should be 64bit, we only support 48bit offset now. */
10185 }
10188 }
10190 /* Parse STRING looking for options in the format:
10191 string :: option:string
10192 option :: name=substring
10193 name :: {a-z}
10194 substring :: defined by option. */
10196 static void
10199 {
10210 {
10212 /* Make this substring look like a string. */
10216 }
10218 /* One last option to parse. */
10221 }
10224 static void
10226 {
10227 /* PR 70044: We have to be careful about being called multiple times for the
10228 same function. This means all changes should be repeatable. */
10230 /* If the frame pointer is enabled, set it to a special value that behaves
10231 similar to frame pointer omission. If we don't do this all leaf functions
10232 will get a frame pointer even if flag_omit_leaf_frame_pointer is set.
10233 If flag_omit_frame_pointer has this special value, we must force the
10234 frame pointer if not in a leaf function. We also need to force it in a
10235 leaf function if flag_omit_frame_pointer is not set or if LR is used. */
10239 /* If not optimizing for size, set the default
10240 alignment to what the target wants. */
10242 {
10249 }
10251 /* We default to no pc-relative literal loads. */
10255 /* If -mpc-relative-literal-loads is set on the command line, this
10256 implies that the user asked for PC relative literal loads. */
10260 /* In the tiny memory model it makes no sense to disallow PC relative
10261 literal pool loads. */
10266 /* When enabling the lower precision Newton series for the square root, also
10267 enable it for the reciprocal square root, since the latter is an
10268 intermediary step for the former. */
10271 }
10273 /* 'Unpack' up the internal tuning structs and update the options
10274 in OPTS. The caller must have set up selected_tune and selected_arch
10275 as all the other target-specific codegen decisions are
10276 derived from them. */
10278 void
10280 {
10283 /* Make a copy of the tuning parameters attached to the core, which
10284 we may later overwrite. */
10292 /* This target defaults to strict volatile bitfields. */
10301 {
10313 }
10315 /* We don't mind passing in global_options_set here as we don't use
10316 the *options_set structs anyway. */
10318 queue_depth,
10322 /* Set up parameters to be used in prefetching algorithm. Do not
10323 override the defaults unless we are tuning for a core we have
10324 researched values for. */
10346 /* Use the alternative scheduling-pressure algorithm by default. */
10351 /* Enable sw prefetching at specified optimization level for
10352 CPUS that have prefetch. Lower optimization level threshold by 1
10353 when profiling is enabled. */
10361 }
10363 /* Print a hint with a suggestion for a core or architecture name that
10364 most closely resembles what the user passed in STR. ARCH is true if
10365 the user is asking for an architecture name. ARCH is false if the user
10366 is asking for a core name. */
10368 static void
10370 {
10381 }
10383 /* Print a hint with a suggestion for a core name that most closely resembles
10384 what the user passed in STR. */
10388 {
10390 }
10392 /* Print a hint with a suggestion for an architecture name that most closely
10393 resembles what the user passed in STR. */
10397 {
10399 }
10401 /* Validate a command-line -mcpu option. Parse the cpu and extensions (if any)
10402 specified in STR and throw errors if appropriate. Put the results if
10403 they are valid in RES and ISA_FLAGS. Return whether the option is
10404 valid. */
10406 static bool
10409 {
10410 enum aarch64_parse_opt_result parse_res
10417 {
10430 }
10433 }
10435 /* Validate a command-line -march option. Parse the arch and extensions
10436 (if any) specified in STR and throw errors if appropriate. Put the
10437 results, if they are valid, in RES and ISA_FLAGS. Return whether the
10438 option is valid. */
10440 static bool
10443 {
10444 enum aarch64_parse_opt_result parse_res
10451 {
10464 }
10467 }
10469 /* Validate a command-line -mtune option. Parse the cpu
10470 specified in STR and throw errors if appropriate. Put the
10471 result, if it is valid, in RES. Return whether the option is
10472 valid. */
10474 static bool
10476 {
10477 enum aarch64_parse_opt_result parse_res
10484 {
10494 }
10496 }
10498 /* Return the CPU corresponding to the enum CPU.
10499 If it doesn't specify a cpu, return the default. */
10503 {
10507 /* The & 0x3f is to extract the bottom 6 bits that encode the
10508 default cpu as selected by the --with-cpu GCC configure option
10509 in config.gcc.
10510 ???: The whole TARGET_CPU_DEFAULT and AARCH64_CPU_DEFAULT_FLAGS
10511 flags mechanism should be reworked to make it more sane. */
10513 }
10515 /* Return the architecture corresponding to the enum ARCH.
10516 If it doesn't specify a valid architecture, return the default. */
10520 {
10527 }
10529 /* Return the VG value associated with -msve-vector-bits= value VALUE. */
10531 static poly_uint16
10533 {
10534 /* For now generate vector-length agnostic code for -msve-vector-bits=128.
10535 This ensures we can clearly distinguish SVE and Advanced SIMD modes when
10536 deciding which .md file patterns to use and when deciding whether
10537 something is a legitimate address or constant. */
10540 else
10542 }
10544 /* Implement TARGET_OPTION_OVERRIDE. This is called once in the beginning
10545 and is used to parse the -m{cpu,tune,arch} strings and setup the initial
10546 tuning structs. In particular it must set selected_tune and
10547 aarch64_isa_flags that define the available ISA features and tuning
10548 decisions. It must also set selected_arch as this will be used to
10549 output the .arch asm tags for each function. */
10551 static void
10553 {
10566 /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
10567 If either of -march or -mtune is given, they override their
10568 respective component of -mcpu. */
10580 /* If the user did not specify a processor, choose the default
10581 one for them. This will be the CPU set during configuration using
10582 --with-cpu, otherwise it is "generic". */
10584 {
10586 {
10590 }
10591 else
10592 {
10593 /* Get default configure-time CPU. */
10596 }
10600 }
10601 /* If both -mcpu and -march are specified check that they are architecturally
10602 compatible, warn if they're not and prefer the -march ISA flags. */
10604 {
10606 {
10610 }
10615 }
10616 else
10617 {
10618 /* -mcpu but no -march. */
10625 }
10627 /* Set the arch as well as we will need it when outputing
10628 the .arch directive in assembly. */
10630 {
10633 }
10638 #ifndef HAVE_AS_MABI_OPTION
10639 /* The compiler may have been configured with 2.23.* binutils, which does
10640 not have support for ILP32. */
10643 #endif
10645 /* Convert -msve-vector-bits to a VG count. */
10651 /* Make sure we properly set up the explicit options. */
10662 /* Save these options as the default ones in case we push and pop them later
10663 while processing functions with potential target attributes. */
10664 target_option_default_node = target_option_current_node
10666 }
10668 /* Implement targetm.override_options_after_change. */
10670 static void
10672 {
10674 }
10678 {
10682 }
10684 void
10686 {
10688 }
10690 /* A checking mechanism for the implementation of the various code models. */
10691 static void
10693 {
10695 {
10697 {
10702 #ifdef HAVE_AS_SMALL_PIC_RELOCS
10704 ? AARCH64_CMODEL_SMALL_PIC
10706 #else
10708 #endif
10716 }
10717 }
10718 else
10720 }
10722 /* Implement TARGET_OPTION_SAVE. */
10724 static void
10726 {
10728 }
10730 /* Implements TARGET_OPTION_RESTORE. Restore the backend codegen decisions
10731 using the information saved in PTR. */
10733 static void
10735 {
10743 }
10745 /* Implement TARGET_OPTION_PRINT. */
10747 static void
10749 {
10760 }
10764 void
10766 {
10768 }
10770 /* Restore or save the TREE_TARGET_GLOBALS from or to NEW_TREE.
10771 Used by aarch64_set_current_function and aarch64_pragma_target_parse to
10772 make sure optab availability predicates are recomputed when necessary. */
10774 void
10776 {
10781 else
10783 }
10785 /* Implement TARGET_SET_CURRENT_FUNCTION. Unpack the codegen decisions
10786 like tuning and ISA features from the DECL_FUNCTION_SPECIFIC_TARGET
10787 of the function, if such exists. This function may be called multiple
10788 times on a single function so use aarch64_previous_fndecl to avoid
10789 setting up identical state. */
10791 static void
10793 {
10797 tree old_tree = (aarch64_previous_fndecl
10803 /* If current function has no attributes but the previous one did,
10804 use the default node. */
10808 /* If nothing to do, return. #pragma GCC reset or #pragma GCC pop to
10809 the default have been handled by aarch64_save_restore_target_globals from
10810 aarch64_pragma_target_parse. */
10816 /* First set the target options. */
10820 }
10822 /* Enum describing the various ways we can handle attributes.
10823 In many cases we can reuse the generic option handling machinery. */
10825 enum aarch64_attr_opt_type
10826 {
10830 aarch64_attr_custom /* Attribute requires a custom handling function. */
10831 };
10833 /* All the information needed to handle a target attribute.
10834 NAME is the name of the attribute.
10835 ATTR_TYPE specifies the type of behavior of the attribute as described
10836 in the definition of enum aarch64_attr_opt_type.
10837 ALLOW_NEG is true if the attribute supports a "no-" form.
10838 HANDLER is the function that takes the attribute string as an argument
10839 It is needed only when the ATTR_TYPE is aarch64_attr_custom.
10840 OPT_NUM is the enum specifying the option that the attribute modifies.
10841 This is needed for attributes that mirror the behavior of a command-line
10842 option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
10843 aarch64_attr_enum. */
10845 struct aarch64_attribute_info
10846 {
10852 };
10854 /* Handle the ARCH_STR argument to the arch= target attribute. */
10856 static bool
10858 {
10860 enum aarch64_parse_opt_result parse_res
10864 {
10869 }
10872 {
10885 }
10888 }
10890 /* Handle the argument CPU_STR to the cpu= target attribute. */
10892 static bool
10894 {
10896 enum aarch64_parse_opt_result parse_res
10900 {
10908 }
10911 {
10924 }
10927 }
10929 /* Handle the argument STR to the tune= target attribute. */
10931 static bool
10933 {
10935 enum aarch64_parse_opt_result parse_res
10939 {
10944 }
10947 {
10954 }
10957 }
10959 /* Parse an architecture extensions target attribute string specified in STR.
10960 For example "+fp+nosimd". Show any errors if needed. Return TRUE
10961 if successful. Update aarch64_isa_flags to reflect the ISA features
10962 modified. */
10964 static bool
10966 {
10970 /* We allow "+nothing" in the beginning to clear out all architectural
10971 features if the user wants to handpick specific features. */
10973 {
10976 }
10981 {
10984 }
10987 {
10998 }
11001 }
11003 /* The target attributes that we support. On top of these we also support just
11004 ISA extensions, like __attribute__ ((target ("+crc"))), but that case is
11005 handled explicitly in aarch64_process_one_target_attr. */
11008 {
11010 OPT_mgeneral_regs_only },
11012 OPT_mfix_cortex_a53_835769 },
11014 OPT_mfix_cortex_a53_843419 },
11018 OPT_momit_leaf_frame_pointer },
11021 OPT_march_ },
11024 OPT_mtune_ },
11026 OPT_msign_return_address_ },
11028 };
11030 /* Parse ARG_STR which contains the definition of one target attribute.
11031 Show appropriate errors if any or return true if the attribute is valid. */
11033 static bool
11035 {
11041 {
11044 }
11049 /* Skip leading whitespace. */
11051 str_to_check++;
11053 /* We have something like __attribute__ ((target ("+fp+nosimd"))).
11054 It is easier to detect and handle it explicitly here rather than going
11055 through the machinery for the rest of the target attributes in this
11056 function. */
11061 {
11064 }
11067 /* If we found opt=foo then terminate STR_TO_CHECK at the '='
11068 and point ARG to "foo". */
11070 {
11072 arg++;
11073 }
11077 {
11078 /* If the names don't match up, or the user has given an argument
11079 to an attribute that doesn't accept one, or didn't give an argument
11080 to an attribute that expects one, fail to match. */
11089 {
11092 }
11094 /* If the name matches but the attribute does not allow "no-" versions
11095 then we can't match. */
11097 {
11100 }
11103 {
11104 /* Has a custom handler registered.
11105 For example, cpu=, arch=, tune=. */
11112 /* Either set or unset a boolean option. */
11114 {
11122 }
11123 /* Set or unset a bit in the target_flags. aarch64_handle_option
11124 should know what mask to apply given the option number. */
11126 {
11128 /* We only need to specify the option number.
11129 aarch64_handle_option will know which mask to apply. */
11135 }
11136 /* Use the option setting machinery to set an option to an enum. */
11138 {
11145 {
11148 global_dc);
11149 }
11150 else
11151 {
11153 }
11155 }
11158 }
11159 }
11161 /* If we reached here we either have found an attribute and validated
11162 it or didn't match any. If we matched an attribute but its arguments
11163 were malformed we will have returned false already. */
11165 }
11167 /* Count how many times the character C appears in
11168 NULL-terminated string STR. */
11170 static unsigned int
11172 {
11175 {
11177 res++;
11179 str++;
11180 }
11183 }
11185 /* Parse the tree in ARGS that contains the target attribute information
11186 and update the global target options space. */
11188 bool
11190 {
11192 {
11193 do
11194 {
11197 {
11200 }
11205 }
11208 {
11211 }
11218 {
11221 }
11223 /* Used to catch empty spaces between commas i.e.
11224 attribute ((target ("attr1,,attr2"))). */
11227 /* Handle multiple target attributes separated by ','. */
11232 {
11233 num_attrs++;
11235 {
11238 }
11241 }
11244 {
11247 }
11250 }
11252 /* Implement TARGET_OPTION_VALID_ATTRIBUTE_P. This is used to
11253 process attribute ((target ("..."))). */
11255 static bool
11257 {
11260 tree old_optimize;
11264 /* If what we're processing is the current pragma string then the
11265 target option node is already stored in target_option_current_node
11266 by aarch64_pragma_target_parse in aarch64-c.c. Use that to avoid
11267 having to re-parse the string. This is especially useful to keep
11268 arm_neon.h compile times down since that header contains a lot
11269 of intrinsics enclosed in pragmas. */
11271 {
11274 }
11280 /* If the function changed the optimization levels as well as setting
11281 target options, start with the optimizations specified. */
11286 /* Save the current target options to restore at the end. */
11289 /* If fndecl already has some target attributes applied to it, unpack
11290 them so that we add this attribute on top of them, rather than
11291 overwriting them. */
11293 {
11299 }
11300 else
11306 /* Set up any additional state. */
11308 {
11310 /* Initialize SIMD builtins if we haven't already.
11311 Set current_target_pragma to NULL for the duration so that
11312 the builtin initialization code doesn't try to tag the functions
11313 being built with the attributes specified by any current pragma, thus
11314 going into an infinite recursion. */
11316 {
11321 }
11323 }
11324 else
11330 {
11335 }
11343 }
11345 /* Helper for aarch64_can_inline_p. In the case where CALLER and CALLEE are
11346 tri-bool options (yes, no, don't care) and the default value is
11347 DEF, determine whether to reject inlining. */
11349 static bool
11352 {
11353 /* If the callee doesn't care, always allow inlining. */
11357 /* If the caller doesn't care, always allow inlining. */
11361 /* Otherwise, allow inlining if either the callee and caller values
11362 agree, or if the callee is using the default value. */
11364 }
11366 /* Implement TARGET_CAN_INLINE_P. Decide whether it is valid
11367 to inline CALLEE into CALLER based on target-specific info.
11368 Make sure that the caller and callee have compatible architectural
11369 features. Then go through the other possible target attributes
11370 and see if they can block inlining. Try not to reject always_inline
11371 callees unless they are incompatible architecturally. */
11373 static bool
11375 {
11379 /* If callee has no option attributes, then it is ok to inline. */
11390 /* Callee's ISA flags should be a subset of the caller's. */
11395 /* Allow non-strict aligned functions inlining into strict
11396 aligned ones. */
11406 /* If the architectural features match up and the callee is always_inline
11407 then the other attributes don't matter. */
11419 /* Honour explicit requests to workaround errata. */
11432 /* If the user explicitly specified -momit-leaf-frame-pointer for the
11433 caller and calle and they don't match up, reject inlining. */
11440 /* If the callee has specific tuning overrides, respect them. */
11445 /* If the user specified tuning override strings for the
11446 caller and callee and they don't match up, reject inlining.
11447 We just do a string compare here, we don't analyze the meaning
11448 of the string, as it would be too costly for little gain. */
11456 }
11458 /* Return true if SYMBOL_REF X binds locally. */
11460 static bool
11462 {
11466 }
11468 /* Return true if SYMBOL_REF X is thread local */
11469 static bool
11471 {
11479 }
11481 /* Classify a TLS symbol into one of the TLS kinds. */
11482 enum aarch64_symbol_type
11484 {
11488 {
11495 {
11501 }
11512 else
11521 }
11522 }
11524 /* Return the correct method for accessing X + OFFSET, where X is either
11525 a SYMBOL_REF or LABEL_REF. */
11527 enum aarch64_symbol_type
11529 {
11531 {
11533 {
11548 }
11549 }
11552 {
11557 {
11559 /* When we retrieve symbol + offset address, we have to make sure
11560 the offset does not cause overflow of the final address. But
11561 we have no way of knowing the address of symbol at compile time
11562 so we can't accurately say if the distance between the PC and
11563 symbol + offset is outside the addressible range of +/-1M in the
11564 TINY code model. So we rely on images not being greater than
11565 1M and cap the offset at 1M and anything beyond 1M will have to
11566 be loaded using an alternative mechanism. Furthermore if the
11567 symbol is a weak reference to something that isn't known to
11568 resolve to a symbol in this module, then force to memory. */
11576 /* Same reasoning as the tiny code model, but the offset cap here is
11577 4G. */
11598 /* This is alright even in PIC code as the constant
11599 pool reference is always PC relative and within
11600 the same translation unit. */
11603 else
11608 }
11609 }
11611 /* By default push everything into the constant pool. */
11613 }
11615 bool
11617 {
11619 }
11621 bool
11623 {
11631 }
11633 /* Implement TARGET_LEGITIMATE_CONSTANT_P hook. Return true for constants
11634 that should be rematerialized rather than spilled. */
11636 static bool
11638 {
11639 /* Support CSE and rematerialization of common constants. */
11643 /* Do not allow vector struct mode constants for Advanced SIMD.
11644 We could support 0 and -1 easily, but they need support in
11645 aarch64-simd.md. */
11650 /* Do not allow wide int constants - this requires support in movti. */
11654 /* Only accept variable-length vector constants if they can be
11655 handled directly.
11657 ??? It would be possible to handle rematerialization of other
11658 constants via secondary reloads. */
11665 /* Accept polynomial constants that can be calculated by using the
11666 destination of a move as the sole temporary. Constants that
11667 require a second temporary cannot be rematerialized (they can't be
11668 forced to memory and also aren't legitimate constants). */
11669 poly_int64 offset;
11673 /* If an offset is being added to something else, we need to allow the
11674 base to be moved into the destination register, meaning that there
11675 are no free temporaries for the offset. */
11680 /* Do not allow const (plus (anchor_symbol, const_int)). */
11684 /* Treat symbols as constants. Avoid TLS symbols as they are complex,
11685 so spilling them is better than rematerialization. */
11689 /* Label references are always constant. */
11694 }
11696 rtx
11698 {
11704 /* Can return in any reg. */
11707 }
11709 /* On AAPCS systems, this is the "struct __va_list". */
11712 /* Implement TARGET_BUILD_BUILTIN_VA_LIST.
11713 Return the type to use as __builtin_va_list.
11715 AAPCS64 \S 7.1.4 requires that va_list be a typedef for a type defined as:
11717 struct __va_list
11718 {
11719 void *__stack;
11720 void *__gr_top;
11721 void *__vr_top;
11722 int __gr_offs;
11723 int __vr_offs;
11724 }; */
11726 static tree
11728 {
11729 tree va_list_name;
11732 /* Create the type. */
11734 /* Give it the required name. */
11736 TYPE_DECL,
11738 va_list_type);
11743 /* Create the fields. */
11746 ptr_type_node);
11749 ptr_type_node);
11752 ptr_type_node);
11755 integer_type_node);
11758 integer_type_node);
11760 /* Tell tree-stdarg pass about our internal offset fields.
11761 NOTE: va_list_gpr/fpr_counter_field are only used for tree comparision
11762 purpose to identify whether the code is updating va_list internal
11763 offset fields through irregular way. */
11785 /* Compute its layout. */
11789 }
11791 /* Implement TARGET_EXPAND_BUILTIN_VA_START. */
11792 static void
11794 {
11798 tree t;
11812 {
11815 }
11824 NULL_TREE);
11826 NULL_TREE);
11828 NULL_TREE);
11830 NULL_TREE);
11832 NULL_TREE);
11834 /* Emit code to initialize STACK, which points to the next varargs stack
11835 argument. CUM->AAPCS_STACK_SIZE gives the number of stack words used
11836 by named arguments. STACK is 8-byte aligned. */
11843 /* Emit code to initialize GRTOP, the top of the GR save area.
11844 virtual_incoming_args_rtx should have been 16 byte aligned. */
11849 /* Emit code to initialize VRTOP, the top of the VR save area.
11850 This address is gr_save_area_bytes below GRTOP, rounded
11851 down to the next 16-byte boundary. */
11861 /* Emit code to initialize GROFF, the offset from GRTOP of the
11862 next GPR argument. */
11867 /* Likewise emit code to initialize VROFF, the offset from FTOP
11868 of the next VR argument. */
11872 }
11874 /* Implement TARGET_GIMPLIFY_VA_ARG_EXPR. */
11876 static tree
11879 {
11880 tree addr;
11886 machine_mode mode;
11913 type,
11917 {
11918 /* No frontends can create types with variable-sized modes, so we
11919 shouldn't be asked to pass or return them. */
11922 /* TYPE passed in fp/simd registers. */
11934 {
11937 }
11940 {
11942 }
11943 }
11944 else
11945 {
11946 /* TYPE passed in general registers. */
11959 {
11961 }
11962 }
11964 /* Get a local temporary for the field value. */
11967 /* Emit code to branch if off >= 0. */
11973 {
11974 /* Emit: offs = (offs + 15) & -16. */
11980 }
11981 else
11984 /* Update ap.__[g|v]r_offs */
11989 /* String up. */
11993 /* [cond2] if (ap.__[g|v]r_offs > 0) */
11998 /* String up: make sure the assignment happens before the use. */
12002 /* Prepare the trees handling the argument that is passed on the stack;
12003 the top level node will store in ON_STACK. */
12006 {
12007 /* if (alignof(type) > 8) (arg = arg + 15) & -16; */
12015 }
12016 else
12018 /* Advance ap.__stack */
12026 /* String up roundup and advance. */
12029 /* String up with arg */
12031 /* Big-endianness related address adjustment. */
12034 {
12038 }
12043 /* Adjustment to OFFSET in the case of BIG_ENDIAN. */
12053 {
12054 /* type ha; // treat as "struct {ftype field[n];}"
12055 ... [computing offs]
12056 for (i = 0; i <nregs; ++i, offs += 16)
12057 ha.field[i] = *((ftype *)(ap.__vr_top + offs));
12058 return ha; */
12062 /* Declare a local variable. */
12066 /* Establish the base type. */
12068 {
12087 {
12091 }
12095 }
12097 /* *(field_ptr_t)&ha = *((field_ptr_t)vr_saved_area */
12105 /* ha.field[i] = *((field_ptr_t)vr_saved_area + i) */
12107 {
12117 }
12121 }
12131 }
12133 /* Implement TARGET_SETUP_INCOMING_VARARGS. */
12135 static void
12139 {
12141 CUMULATIVE_ARGS local_cum;
12145 /* The caller has advanced CUM up to, but not beyond, the last named
12146 argument. Advance a local copy of CUM past the last "real" named
12147 argument, to find out how many registers are left over. */
12151 /* Found out how many registers we need to save.
12152 Honor tree-stdvar analysis results. */
12161 {
12164 }
12167 {
12169 {
12172 /* virtual_incoming_args_rtx should have been 16-byte aligned. */
12180 }
12182 {
12183 /* We can't use move_block_from_reg, because it will use
12184 the wrong mode, storing D regs only. */
12188 /* Set OFF to the offset from virtual_incoming_args_rtx of
12189 the first vector register. The VR save area lies below
12190 the GR one, and is aligned to 16 bytes. */
12197 {
12205 }
12206 }
12207 }
12209 /* We don't save the size into *PRETEND_SIZE because we want to avoid
12210 any complication of having crtl->args.pretend_args_size changed. */
12215 }
12217 static void
12219 {
12222 {
12224 {
12227 }
12228 }
12231 {
12234 }
12235 }
12237 /* Walk down the type tree of TYPE counting consecutive base elements.
12238 If *MODEP is VOIDmode, then set it to the first valid floating point
12239 type. If a non-floating point type is found, or if a floating point
12240 type that doesn't match a non-VOIDmode *MODEP is found, then return -1,
12241 otherwise return the count in the sub-tree. */
12242 static int
12244 {
12245 machine_mode mode;
12246 HOST_WIDE_INT size;
12249 {
12279 /* Use V2SImode and V4SImode as representatives of all 64-bit
12280 and 128-bit vector types. */
12283 {
12292 }
12297 /* Vector modes are considered to be opaque: two vectors are
12298 equivalent for the purposes of being homogeneous aggregates
12299 if they are the same size. */
12306 {
12310 /* Can't handle incomplete types nor sizes that are not
12311 fixed. */
12318 || !index
12329 /* There must be no padding. */
12335 }
12338 {
12341 tree field;
12343 /* Can't handle incomplete types nor sizes that are not
12344 fixed. */
12350 {
12358 }
12360 /* There must be no padding. */
12366 }
12370 {
12371 /* These aren't very interesting except in a degenerate case. */
12374 tree field;
12376 /* Can't handle incomplete types nor sizes that are not
12377 fixed. */
12383 {
12391 }
12393 /* There must be no padding. */
12399 }
12403 }
12406 }
12408 /* Return TRUE if the type, as described by TYPE and MODE, is a short vector
12409 type as described in AAPCS64 \S 4.1.2.
12411 See the comment above aarch64_composite_type_p for the notes on MODE. */
12413 static bool
12415 machine_mode mode)
12416 {
12426 }
12428 /* Return TRUE if the type, as described by TYPE and MODE, is a composite
12429 type as described in AAPCS64 \S 4.3. This includes aggregate, union and
12430 array types. The C99 floating-point complex types are also considered
12431 as composite types, according to AAPCS64 \S 7.1.1. The complex integer
12432 types, which are GCC extensions and out of the scope of AAPCS64, are
12433 treated as composite types here as well.
12435 Note that MODE itself is not sufficient in determining whether a type
12436 is such a composite type or not. This is because
12437 stor-layout.c:compute_record_mode may have already changed the MODE
12438 (BLKmode) of a RECORD_TYPE TYPE to some other mode. For example, a
12439 structure with only one field may have its MODE set to the mode of the
12440 field. Also an integer mode whose size matches the size of the
12441 RECORD_TYPE type may be used to substitute the original mode
12442 (i.e. BLKmode) in certain circumstances. In other words, MODE cannot be
12443 solely relied on. */
12445 static bool
12447 machine_mode mode)
12448 {
12461 }
12463 /* Return TRUE if an argument, whose type is described by TYPE and MODE,
12464 shall be passed or returned in simd/fp register(s) (providing these
12465 parameter passing registers are available).
12467 Upon successful return, *COUNT returns the number of needed registers,
12468 *BASE_MODE returns the mode of the individual register and when IS_HAF
12469 is not NULL, *IS_HA indicates whether or not the argument is a homogeneous
12470 floating-point aggregate or a homogeneous short-vector aggregate. */
12472 static bool
12474 const_tree type,
12478 {
12486 {
12489 }
12491 {
12495 }
12497 {
12501 {
12504 }
12505 else
12507 }
12508 else
12513 }
12515 /* Implement TARGET_STRUCT_VALUE_RTX. */
12517 static rtx
12520 {
12522 }
12524 /* Implements target hook vector_mode_supported_p. */
12525 static bool
12527 {
12530 }
12532 /* Return appropriate SIMD container
12533 for MODE within a vector of WIDTH bits. */
12534 static machine_mode
12536 {
12539 {
12556 }
12560 {
12563 {
12580 }
12581 else
12583 {
12596 }
12597 }
12599 }
12601 /* Return 128-bit container as the preferred SIMD mode for MODE. */
12602 static machine_mode
12604 {
12607 }
12609 /* Return a list of possible vector sizes for the vectorizer
12610 to iterate over. */
12611 static void
12613 {
12618 }
12620 /* Implement TARGET_MANGLE_TYPE. */
12624 {
12625 /* The AArch64 ABI documents say that "__va_list" has to be
12626 managled as if it is in the "std" namespace. */
12630 /* Half-precision float. */
12634 /* Mangle AArch64-specific internal types. TYPE_NAME is non-NULL_TREE for
12635 builtin types. */
12639 /* Use the default mangling. */
12641 }
12643 /* Find the first rtx_insn before insn that will generate an assembly
12644 instruction. */
12648 {
12652 do
12653 {
12655 }
12659 }
12661 static bool
12663 {
12665 /* A number of these may be AArch32 only. */
12670 };
12673 {
12676 }
12679 }
12681 /* Check if there is a register dependency between a load and the insn
12682 for which we hold recog_data. */
12684 static bool
12686 {
12687 rtx load_reg;
12697 {
12703 }
12705 }
12708 /* When working around the Cortex-A53 erratum 835769,
12709 given rtx_insn INSN, return true if it is a 64-bit multiply-accumulate
12710 instruction and has a preceding memory instruction such that a NOP
12711 should be inserted between them. */
12713 bool
12715 {
12718 rtx body;
12731 /* aarch64_prev_real_insn can call recog_memoized on insns other than INSN.
12732 Restore recog state to INSN to avoid state corruption. */
12740 /* If the previous insn is a memory op and there is no dependency between
12741 it and the DImode madd, emit a NOP between them. If body is NULL then we
12742 have a complex memory operation, probably a load/store pair.
12743 Be conservative for now and emit a NOP. */
12750 }
12753 /* Implement FINAL_PRESCAN_INSN. */
12755 void
12757 {
12760 }
12763 /* Return the equivalent letter for size. */
12764 static char
12766 {
12768 {
12774 }
12775 }
12777 /* Return true if BASE_OR_STEP is a valid immediate operand for an SVE INDEX
12778 instruction. */
12780 bool
12782 {
12785 }
12787 /* Return true if X is a valid immediate for the SVE ADD and SUB
12788 instructions. Negate X first if NEGATE_P is true. */
12790 bool
12792 {
12793 rtx elt;
12807 }
12809 /* Return true if X is a valid immediate operand for an SVE logical
12810 instruction such as AND. */
12812 bool
12814 {
12815 rtx elt;
12821 }
12823 /* Return true if X is a valid immediate for the SVE DUP and CPY
12824 instructions. */
12826 bool
12828 {
12829 rtx elt;
12839 }
12841 /* Return true if X is a valid immediate operand for an SVE CMP instruction.
12842 SIGNED_P says whether the operand is signed rather than unsigned. */
12844 bool
12846 {
12847 rtx elt;
12851 && (signed_p
12854 }
12856 /* Return true if X is a valid immediate operand for an SVE FADD or FSUB
12857 instruction. Negate X first if NEGATE_P is true. */
12859 bool
12861 {
12862 rtx elt;
12863 REAL_VALUE_TYPE r;
12879 }
12881 /* Return true if X is a valid immediate operand for an SVE FMUL
12882 instruction. */
12884 bool
12886 {
12887 rtx elt;
12889 /* GCC will never generate a multiply with an immediate of 2, so there is no
12890 point testing for it (even though it is a valid constant). */
12894 }
12896 /* Return true if replicating VAL32 is a valid 2-byte or 4-byte immediate
12897 for the Advanced SIMD operation described by WHICH and INSN. If INFO
12898 is nonnull, use it to describe valid immediates. */
12899 static bool
12904 {
12905 /* Try a 4-byte immediate with LSL. */
12908 {
12913 }
12915 /* Try a 2-byte immediate with LSL. */
12920 {
12925 }
12927 /* Try a 4-byte immediate with MSL, except for cases that MVN
12928 can handle. */
12931 {
12934 {
12939 }
12940 }
12943 }
12945 /* Return true if replicating VAL64 is a valid immediate for the
12946 Advanced SIMD operation described by WHICH. If INFO is nonnull,
12947 use it to describe valid immediates. */
12948 static bool
12952 {
12958 {
12969 /* Try using a replicated byte. */
12973 {
12977 }
12978 }
12980 /* Try using a bit-to-bytemask. */
12982 {
12985 {
12989 }
12991 {
12995 }
12996 }
12998 }
13000 /* Return true if replicating VAL64 gives a valid immediate for an SVE MOV
13001 instruction. If INFO is nonnull, use it to describe valid immediates. */
13003 static bool
13006 {
13010 {
13014 {
13019 }
13020 }
13023 {
13024 /* DUP with no shift. */
13028 }
13030 {
13031 /* DUP with LSL #8. */
13035 }
13037 {
13038 /* DUPM. */
13042 }
13044 }
13046 /* Return true if OP is a valid SIMD immediate for the operation
13047 described by WHICH. If INFO is nonnull, use it to describe valid
13048 immediates. */
13049 bool
13052 {
13065 {
13074 }
13078 else
13081 /* Handle PFALSE and PTRUE. */
13086 scalar_float_mode elt_float_mode;
13091 {
13095 }
13103 /* Expand the vector constant out into a byte vector, with the least
13104 significant byte of the register first. */
13108 {
13110 /* The vector is provided in gcc endian-neutral fashion.
13111 For aarch64_be, it must be laid out in the vector register
13112 in reverse order. */
13123 {
13126 }
13127 }
13129 /* The immediate must repeat every eight bytes. */
13135 /* Get the repeating 8-byte value as an integer. No endian correction
13136 is needed here because bytes is already in lsb-first order. */
13144 else
13146 }
13148 /* Check whether X is a VEC_SERIES-like constant that starts at 0 and
13149 has a step in the range of INDEX. Return the index expression if so,
13150 otherwise return null. */
13151 rtx
13153 {
13160 }
13162 /* Check of immediate shift constants are within range. */
13163 bool
13165 {
13169 else
13171 }
13173 /* Return the bitmask CONST_INT to select the bits required by a zero extract
13174 operation of width WIDTH at bit position POS. */
13176 rtx
13178 {
13182 unsigned HOST_WIDE_INT mask
13185 }
13187 bool
13189 {
13208 }
13210 /* Return a const_int vector of VAL. */
13211 rtx
13213 {
13216 }
13218 /* Check OP is a legal scalar immediate for the MOVI instruction. */
13220 bool
13222 {
13223 machine_mode vmode;
13228 }
13230 /* Construct and return a PARALLEL RTX vector with elements numbering the
13231 lanes of either the high (HIGH == TRUE) or low (HIGH == FALSE) half of
13232 the vector - from the perspective of the architecture. This does not
13233 line up with GCC's perspective on lane numbers, so we end up with
13234 different masks depending on our target endian-ness. The diagram
13235 below may help. We must draw the distinction when building masks
13236 which select one half of the vector. An instruction selecting
13237 architectural low-lanes for a big-endian target, must be described using
13238 a mask selecting GCC high-lanes.
13240 Big-Endian Little-Endian
13242 GCC 0 1 2 3 3 2 1 0
13243 | x | x | x | x | | x | x | x | x |
13244 Architecture 3 2 1 0 3 2 1 0
13246 Low Mask: { 2, 3 } { 0, 1 }
13247 High Mask: { 0, 1 } { 2, 3 }
13249 MODE Is the mode of the vector and NUNITS is the number of units in it. */
13251 rtx
13253 {
13258 rtx t1;
13263 else
13271 }
13273 /* Check OP for validity as a PARALLEL RTX vector with elements
13274 numbering the lanes of either the high (HIGH == TRUE) or low lanes,
13275 from the perspective of the architecture. See the diagram above
13276 aarch64_simd_vect_par_cnst_half for more details. */
13278 bool
13281 {
13295 {
13302 }
13304 }
13306 /* Bounds-check lanes. Ensure OPERAND lies between LOW (inclusive) and
13307 HIGH (exclusive). */
13308 void
13310 const_tree exp)
13311 {
13312 HOST_WIDE_INT lane;
13317 {
13320 else
13322 }
13323 }
13325 /* Peform endian correction on lane number N, which indexes a vector
13326 of mode MODE, and return the result as an SImode rtx. */
13328 rtx
13330 {
13332 }
13334 /* Return TRUE if OP is a valid vector addressing mode. */
13336 bool
13338 {
13341 }
13343 /* Return true if OP is a valid MEM operand for an SVE LD1R instruction. */
13345 bool
13347 {
13349 scalar_mode mode;
13356 }
13358 /* Return true if OP is a valid MEM operand for an SVE LDR instruction.
13359 The conditions for STR are the same. */
13360 bool
13362 {
13369 }
13371 /* Emit a register copy from operand to operand, taking care not to
13372 early-clobber source registers in the process.
13374 COUNT is the number of components into which the copy needs to be
13375 decomposed. */
13376 void
13379 {
13389 else
13393 }
13395 /* Compute and return the length of aarch64_simd_reglist<mode>, where <mode> is
13396 one of VSTRUCT modes: OI, CI, or XI. */
13397 int
13399 {
13400 /* This is only used (and only meaningful) for Advanced SIMD, not SVE. */
13402 }
13404 /* Implement target hook TARGET_VECTOR_ALIGNMENT. The AAPCS64 sets the maximum
13405 alignment of a vector to 128 bits. SVE predicates have an alignment of
13406 16 bits. */
13407 static HOST_WIDE_INT
13409 {
13411 /* ??? Checking the mode isn't ideal, but VECTOR_BOOLEAN_TYPE_P can
13412 be set for non-predicate vectors of booleans. Modes are the most
13413 direct way we have of identifying real SVE predicate types. */
13417 }
13419 /* Implement target hook TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT. */
13420 static HOST_WIDE_INT
13422 {
13424 {
13425 /* If the length of the vector is fixed, try to align to that length,
13426 otherwise don't try to align at all. */
13427 HOST_WIDE_INT result;
13431 }
13433 }
13435 /* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE. */
13436 static bool
13438 {
13442 /* For fixed-length vectors, check that the vectorizer will aim for
13443 full-vector alignment. This isn't true for generic GCC vectors
13444 that are wider than the ABI maximum of 128 bits. */
13450 /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned. */
13452 }
13454 /* Return true if the vector misalignment factor is supported by the
13455 target. */
13456 static bool
13460 {
13462 {
13463 /* Return if movmisalign pattern is not supported for this mode. */
13467 /* Misalignment factor is unknown at compile time. */
13470 }
13472 is_packed);
13473 }
13475 /* If VALS is a vector constant that can be loaded into a register
13476 using DUP, generate instructions to do so and return an RTX to
13477 assign to the register. Otherwise return NULL_RTX. */
13478 static rtx
13480 {
13483 rtx x;
13488 /* We can load this constant by using DUP and a constant in a
13489 single ARM register. This will be cheaper than a vector
13490 load. */
13493 }
13496 /* Generate code to load VALS, which is a PARALLEL containing only
13497 constants (for vec_init) or CONST_VECTOR, efficiently into a
13498 register. Returns an RTX to copy into the register, or NULL_RTX
13499 for a PARALLEL that can not be converted into a CONST_VECTOR. */
13500 static rtx
13502 {
13504 rtx const_dup;
13512 {
13513 /* A CONST_VECTOR must contain only CONST_INTs and
13514 CONST_DOUBLEs, but CONSTANT_P allows more (e.g. SYMBOL_REF).
13515 Only store valid constants in a CONST_VECTOR. */
13518 {
13521 n_const++;
13522 }
13525 }
13526 else
13531 /* Load using MOVI/MVNI. */
13534 /* Loaded using DUP. */
13537 /* Load from constant pool. We can not take advantage of single-cycle
13538 LD1 because we need a PC-relative addressing mode. */
13540 else
13541 /* A PARALLEL containing something not valid inside CONST_VECTOR.
13542 We can not construct an initializer. */
13544 }
13546 /* Expand a vector initialisation sequence, such that TARGET is
13547 initialised to contain VALS. */
13549 void
13551 {
13554 /* The number of vector elements. */
13556 /* The number of vector elements which are not constant. */
13559 /* The first element of vals. */
13563 /* Count the number of variable elements to initialise. */
13565 {
13569 else
13573 }
13575 /* No variable elements, hand off to aarch64_simd_make_constant which knows
13576 how best to handle this. */
13578 {
13581 {
13584 }
13585 }
13587 /* Splat a single non-constant element if we can. */
13589 {
13593 }
13598 /* If there are only variable elements, try to optimize
13599 the insertion using dup for the most common element
13600 followed by insertions. */
13602 /* The algorithm will fill matches[*][0] with the earliest matching element,
13603 and matches[X][1] with the count of duplicate elements (if X is the
13604 earliest element which has duplicates). */
13607 {
13610 {
13612 {
13614 {
13618 }
13619 }
13620 }
13625 {
13628 }
13630 /* Create a duplicate of the most common element. */
13634 /* Insert the rest. */
13636 {
13642 }
13644 }
13646 /* Initialise a vector which is part-variable. We want to first try
13647 to build those lanes which are constant in the most efficient way we
13648 can. */
13650 {
13653 /* Load constant part of vector. We really don't care what goes into the
13654 parts we will overwrite, but we're more likely to be able to load the
13655 constant efficiently if it has fewer, larger, repeating parts
13656 (see aarch64_simd_valid_immediate). */
13658 {
13664 {
13665 /* Look in the copied vector, as more elements are const. */
13668 {
13671 }
13672 }
13674 }
13676 }
13678 /* Insert the variable lanes directly. */
13680 {
13686 }
13687 }
13689 static unsigned HOST_WIDE_INT
13691 {
13695 }
13697 /* Select a format to encode pointers in exception handling data. */
13698 int
13700 {
13703 {
13709 /* text+got+data < 4Gb. 4-byte signed relocs are sufficient
13710 for everything. */
13714 /* No assumptions here. 8-byte relocs required. */
13717 }
13719 }
13721 /* The last .arch and .tune assembly strings that we printed. */
13725 /* Implement ASM_DECLARE_FUNCTION_NAME. Output the ISA features used
13726 by the function fndecl. */
13728 void
13730 tree fndecl)
13731 {
13737 else
13748 /* Only update the assembler .arch string if it is distinct from the last
13749 such string we printed. */
13752 {
13755 }
13757 /* Print the cpu name we're tuning for in the comments, might be
13758 useful to readers of the generated asm. Do it only when it changes
13759 from function to function and verbose assembly is requested. */
13764 {
13768 }
13770 /* Don't forget the type directive for ELF. */
13773 }
13775 /* Implements TARGET_ASM_FILE_START. Output the assembly header. */
13777 static void
13779 {
13796 }
13798 /* Emit load exclusive. */
13800 static void
13803 {
13807 {
13814 }
13817 }
13819 /* Emit store exclusive. */
13821 static void
13824 {
13828 {
13835 }
13838 }
13840 /* Mark the previous jump instruction as unlikely. */
13842 static void
13844 {
13847 }
13849 /* Expand a compare and swap pattern. */
13851 void
13853 {
13858 gen_cas_fn gen;
13860 {
13861 gen_aarch64_compare_and_swapqi,
13862 gen_aarch64_compare_and_swaphi,
13863 gen_aarch64_compare_and_swapsi,
13864 gen_aarch64_compare_and_swapdi
13865 };
13867 {
13868 gen_aarch64_compare_and_swapqi_lse,
13869 gen_aarch64_compare_and_swaphi_lse,
13870 gen_aarch64_compare_and_swapsi_lse,
13871 gen_aarch64_compare_and_swapdi_lse
13872 };
13885 /* Normally the succ memory model must be stronger than fail, but in the
13886 unlikely event of fail being ACQUIRE and succ being RELEASE we need to
13887 promote succ to ACQ_REL so that we don't lose the acquire semantics. */
13894 {
13897 /* For short modes, we're going to perform the comparison in SImode,
13898 so do the zero-extension now. */
13902 /* Fall through. */
13906 /* Force the value into a register if needed. */
13913 }
13916 {
13923 }
13926 else
13937 }
13939 /* Test whether the target supports using a atomic load-operate instruction.
13940 CODE is the operation and AFTER is TRUE if the data in memory after the
13941 operation should be returned and FALSE if the data before the operation
13942 should be returned. Returns FALSE if the operation isn't supported by the
13943 architecture. */
13945 bool
13947 {
13952 {
13962 }
13963 }
13965 /* Emit a barrier, that is appropriate for memory model MODEL, at the end of a
13966 sequence implementing an atomic operation. */
13968 static void
13970 {
13977 {
13979 }
13980 }
13982 /* Emit an atomic compare-and-swap operation. RVAL is the destination register
13983 for the data in memory. EXPECTED is the value expected to be in memory.
13984 DESIRED is the value to store to memory. MEM is the memory location. MODEL
13985 is the memory ordering to use. */
13987 void
13990 rtx model)
13991 {
13993 machine_mode mode;
13998 {
14005 }
14007 /* Move the expected value into the CAS destination register. */
14010 /* Emit the CAS. */
14013 /* Compare the expected value with the value loaded by the CAS, to establish
14014 whether the swap was made. */
14016 }
14018 /* Split a compare and swap pattern. */
14020 void
14022 {
14024 machine_mode mode;
14029 rtx model_rtx;
14041 /* When OLDVAL is zero and we want the strong version we can emit a tighter
14042 loop:
14043 .label1:
14044 LD[A]XR rval, [mem]
14045 CBNZ rval, .label2
14046 ST[L]XR scratch, newval, [mem]
14047 CBNZ scratch, .label1
14048 .label2:
14049 CMP rval, 0. */
14054 {
14057 }
14060 /* The initial load can be relaxed for a __sync operation since a final
14061 barrier will be emitted to stop code hoisting. */
14065 else
14069 {
14074 }
14075 else
14076 {
14082 }
14087 {
14092 }
14093 else
14094 {
14098 }
14101 /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL
14102 to set the condition flags. If this is not used it will be removed by
14103 later passes. */
14105 {
14109 }
14110 /* Emit any final barrier needed for a __sync operation. */
14113 }
14115 /* Emit a BIC instruction. */
14117 static void
14119 {
14124 {
14129 }
14132 }
14134 /* Emit an atomic swap. */
14136 static void
14139 {
14143 {
14150 }
14153 }
14155 /* Operations supported by aarch64_emit_atomic_load_op. */
14157 enum aarch64_atomic_load_op_code
14158 {
14162 AARCH64_LDOP_BIC /* A & ~B */
14163 };
14165 /* Emit an atomic load-operate. */
14167 static void
14171 {
14174 {
14175 gen_aarch64_atomic_loadaddqi,
14176 gen_aarch64_atomic_loadaddhi,
14177 gen_aarch64_atomic_loadaddsi,
14178 gen_aarch64_atomic_loadadddi
14179 };
14181 {
14182 gen_aarch64_atomic_loadeorqi,
14183 gen_aarch64_atomic_loadeorhi,
14184 gen_aarch64_atomic_loadeorsi,
14185 gen_aarch64_atomic_loadeordi
14186 };
14188 {
14189 gen_aarch64_atomic_loadsetqi,
14190 gen_aarch64_atomic_loadsethi,
14191 gen_aarch64_atomic_loadsetsi,
14192 gen_aarch64_atomic_loadsetdi
14193 };
14195 {
14196 gen_aarch64_atomic_loadclrqi,
14197 gen_aarch64_atomic_loadclrhi,
14198 gen_aarch64_atomic_loadclrsi,
14199 gen_aarch64_atomic_loadclrdi
14200 };
14201 aarch64_atomic_load_op_fn gen;
14205 {
14212 }
14215 {
14222 }
14225 }
14227 /* Emit an atomic load+operate. CODE is the operation. OUT_DATA is the
14228 location to store the data read from memory. OUT_RESULT is the location to
14229 store the result of the operation. MEM is the memory location to read and
14230 modify. MODEL_RTX is the memory ordering to use. VALUE is the second
14231 operand for the operation. Either OUT_DATA or OUT_RESULT, but not both, can
14232 be NULL. */
14234 void
14237 {
14241 aarch64_atomic_load_op_code ldop_code;
14242 rtx src;
14243 rtx x;
14251 /* Make sure the value is in a register, putting it into a destination
14252 register if it needs to be manipulated. */
14255 {
14258 }
14259 else
14263 /* Preprocess the data for the operation as necessary. If the operation is
14264 a SET then emit a swap instruction and finish. */
14266 {
14272 /* Negate the value and treat it as a PLUS. */
14273 {
14274 rtx neg_src;
14276 /* Resize the value if necessary. */
14285 }
14286 /* Fall-through. */
14300 {
14301 rtx not_src;
14303 /* Resize the value if necessary. */
14312 }
14317 /* The operation can't be done with atomic instructions. */
14319 }
14323 /* If necessary, calculate the data in memory after the update by redoing the
14324 operation from values in registers. */
14329 {
14333 }
14338 {
14354 }
14359 }
14361 /* Split an atomic operation. */
14363 void
14366 {
14372 rtx x;
14374 /* Split the atomic operation into a sequence. */
14382 else
14386 /* The initial load can be relaxed for a __sync operation since a final
14387 barrier will be emitted to stop code hoisting. */
14391 else
14395 {
14409 {
14412 }
14413 /* Fall through. */
14419 }
14429 /* Emit any final barrier needed for a __sync operation. */
14432 }
14434 static void
14436 {
14437 /* Half-precision float operations. The compiler handles all operations
14438 with NULL libfuncs by converting to SFmode. */
14440 /* Conversions. */
14444 /* Arithmetic. */
14451 /* Comparisons. */
14459 }
14461 /* Target hook for c_mode_for_suffix. */
14462 static machine_mode
14464 {
14469 }
14471 /* We can only represent floating point constants which will fit in
14472 "quarter-precision" values. These values are characterised by
14473 a sign bit, a 4-bit mantissa and a 3-bit exponent. And are given
14474 by:
14476 (-1)^s * (n/16) * 2^r
14478 Where:
14479 's' is the sign bit.
14480 'n' is an integer in the range 16 <= n <= 31.
14481 'r' is an integer in the range -3 <= r <= 4. */
14483 /* Return true iff X can be represented by a quarter-precision
14484 floating point immediate operand X. Note, we cannot represent 0.0. */
14485 bool
14487 {
14488 /* This represents our current view of how many bits
14489 make up the mantissa. */
14499 /* We don't support HFmode constants yet. */
14505 /* We cannot represent infinities, NaNs or +/-zero. We won't
14506 know if we have +zero until we analyse the mantissa, but we
14507 can reject the other invalid values. */
14512 /* Extract exponent. */
14516 /* For the mantissa, we expand into two HOST_WIDE_INTS, apart from the
14517 highest (sign) bit, with a fixed binary point at bit point_pos.
14518 m1 holds the low part of the mantissa, m2 the high part.
14519 WARNING: If we ever have a representation using more than 2 * H_W_I - 1
14520 bits for the mantissa, this can fail (low bits will be lost). */
14524 /* If the low part of the mantissa has bits set we cannot represent
14525 the value. */
14528 /* We have rejected the lower HOST_WIDE_INT, so update our
14529 understanding of how many bits lie in the mantissa and
14530 look only at the high HOST_WIDE_INT. */
14534 /* We can only represent values with a mantissa of the form 1.xxxx. */
14539 /* Having filtered unrepresentable values, we may now remove all
14540 but the highest 5 bits. */
14543 /* We cannot represent the value 0.0, so reject it. This is handled
14544 elsewhere. */
14548 /* Then, as bit 4 is always set, we can mask it off, leaving
14549 the mantissa in the range [0, 15]. */
14553 /* GCC internally does not use IEEE754-like encoding (where normalized
14554 significands are in the range [1, 2). GCC uses [0.5, 1) (see real.c).
14555 Our mantissa values are shifted 4 places to the left relative to
14556 normalized IEEE754 so we must modify the exponent returned by REAL_EXP
14557 by 5 places to correct for GCC's representation. */
14561 }
14563 /* Returns the string with the instruction for AdvSIMD MOVI, MVNI, ORR or BIC
14564 immediate with a CONST_VECTOR of MODE and WIDTH. WHICH selects whether to
14565 output MOVI/MVNI, ORR or BIC immediate. */
14569 {
14579 /* This will return true to show const_vector is legal for use as either
14580 a AdvSIMD MOVI instruction (or, implicitly, MVNI), ORR or BIC immediate.
14581 It will also update INFO to show how the immediate should be generated.
14582 WHICH selects whether to check for MOVI/MVNI, ORR or BIC. */
14590 {
14592 /* For FP zero change it to a CONST_INT 0 and use the integer SIMD
14593 move immediate path. */
14596 else
14597 {
14606 else
14610 }
14611 }
14616 {
14626 else
14630 }
14631 else
14632 {
14633 /* For AARCH64_CHECK_BIC and AARCH64_CHECK_ORR. */
14639 else
14643 }
14645 }
14649 {
14651 /* If a floating point number was passed and we desire to use it in an
14652 integer mode do the conversion to integer. */
14654 {
14659 }
14661 machine_mode vmode;
14662 /* use a 64 bit mode for everything except for DI/DF mode, where we use
14663 a 128 bit vector mode. */
14669 }
14671 /* Return the output string to use for moving immediate CONST_VECTOR
14672 into an SVE register. */
14676 {
14687 {
14692 }
14695 {
14698 else
14699 {
14709 }
14710 }
14715 }
14717 /* Return the asm format for a PTRUE instruction whose destination has
14718 mode MODE. SUFFIX is the element size suffix. */
14722 {
14727 else
14730 }
14732 /* Split operands into moves from op[1] + op[2] into op[0]. */
14734 void
14736 {
14747 {
14748 /* No-op move. Can't split to nothing; emit something. */
14751 }
14753 /* Preserve register attributes for variable tracking. */
14758 /* Special case of reversed high/low parts. */
14761 {
14765 }
14767 {
14768 /* Try to avoid unnecessary moves if part of the result
14769 is in the right place already. */
14774 }
14775 else
14776 {
14781 }
14782 }
14784 /* vec_perm support. */
14786 struct expand_vec_perm_d
14787 {
14789 vec_perm_indices perm;
14790 machine_mode vmode;
14794 };
14796 /* Generate a variable permutation. */
14798 static void
14800 {
14811 {
14813 {
14814 /* Expand the argument to a V16QI mode by duplicating it. */
14818 }
14819 else
14820 {
14822 }
14823 }
14824 else
14825 {
14826 rtx pair;
14829 {
14833 }
14834 else
14835 {
14839 }
14840 }
14841 }
14843 /* Expand a vec_perm with the operands given by TARGET, OP0, OP1 and SEL.
14844 NELT is the number of elements in the vector. */
14846 void
14849 {
14852 rtx mask;
14854 /* The TBL instruction does not use a modulo index, so we must take care
14855 of that ourselves. */
14860 /* For big-endian, we also need to reverse the index within the vector
14861 (but not which vector). */
14863 {
14864 /* If one_vector_p, mask is a vector of (nelt - 1)'s already. */
14869 }
14871 }
14873 /* Generate (set TARGET (unspec [OP0 OP1] CODE)). */
14875 static void
14877 {
14881 }
14883 /* Expand an SVE vec_perm with the given operands. */
14885 void
14887 {
14890 /* Enforced by the pattern condition. */
14893 /* Note: vec_perm indices are supposed to wrap when they go beyond the
14894 size of the two value vectors, i.e. the upper bits of the indices
14895 are effectively ignored. SVE TBL instead produces 0 for any
14896 out-of-range indices, so we need to modulo all the vec_perm indices
14897 to ensure they are all in range. */
14900 /* Check if the sel only references the first values vector. */
14903 {
14906 }
14908 /* Check if the two values vectors are the same. */
14910 {
14916 }
14918 /* Run TBL on for each value vector and combine the results. */
14925 {
14930 }
14937 else
14939 }
14941 /* Recognize patterns suitable for the TRN instructions. */
14942 static bool
14944 {
14945 HOST_WIDE_INT odd;
14953 /* Note that these are little-endian tests.
14954 We correct for big-endian later. */
14961 /* Success! */
14967 /* We don't need a big-endian lane correction for SVE; see the comment
14968 at the head of aarch64-sve.md for details. */
14970 {
14973 }
14979 }
14981 /* Recognize patterns suitable for the UZP instructions. */
14982 static bool
14984 {
14985 HOST_WIDE_INT odd;
14992 /* Note that these are little-endian tests.
14993 We correct for big-endian later. */
14999 /* Success! */
15005 /* We don't need a big-endian lane correction for SVE; see the comment
15006 at the head of aarch64-sve.md for details. */
15008 {
15011 }
15017 }
15019 /* Recognize patterns suitable for the ZIP instructions. */
15020 static bool
15022 {
15031 /* Note that these are little-endian tests.
15032 We correct for big-endian later. */
15040 /* Success! */
15046 /* We don't need a big-endian lane correction for SVE; see the comment
15047 at the head of aarch64-sve.md for details. */
15049 {
15052 }
15058 }
15060 /* Recognize patterns for the EXT insn. */
15062 static bool
15064 {
15065 HOST_WIDE_INT location;
15066 rtx offset;
15068 /* The first element always refers to the first vector.
15069 Check if the extracted indices are increasing by one. */
15075 /* Success! */
15079 /* The case where (location == 0) is a no-op for both big- and little-endian,
15080 and is removed by the mid-end at optimization levels -O1 and higher.
15082 We don't need a big-endian lane correction for SVE; see the comment
15083 at the head of aarch64-sve.md for details. */
15085 {
15086 /* After setup, we want the high elements of the first vector (stored
15087 at the LSB end of the register), and the low elements of the second
15088 vector (stored at the MSB end of the register). So swap. */
15090 /* location != 0 (above), so safe to assume (nelt - location) < nelt.
15091 to_constant () is safe since this is restricted to Advanced SIMD
15092 vectors. */
15094 }
15100 UNSPEC_EXT));
15102 }
15104 /* Recognize patterns for the REV{64,32,16} insns, which reverse elements
15105 within each 64-bit, 32-bit or 16-bit granule. */
15107 static bool
15109 {
15110 HOST_WIDE_INT diff;
15112 machine_mode pred_mode;
15121 {
15124 }
15126 {
15129 }
15131 {
15134 }
15135 else
15143 /* Success! */
15149 {
15152 UNSPEC_MERGE_PTRUE);
15153 }
15156 }
15158 /* Recognize patterns for the REV insn, which reverses elements within
15159 a full vector. */
15161 static bool
15163 {
15172 /* Success! */
15179 }
15181 static bool
15183 {
15185 rtx in0;
15186 HOST_WIDE_INT elt;
15188 rtx lane;
15198 /* Success! */
15202 /* The generic preparation in aarch64_expand_vec_perm_const_1
15203 swaps the operand order and the permute indices if it finds
15204 d->perm[0] to be in the second operand. Thus, we can always
15205 use d->op0 and need not do any extra arithmetic to get the
15206 correct lane number. */
15214 }
15216 static bool
15218 {
15222 /* Make sure that the indices are constant. */
15231 /* Generic code will try constant permutation twice. Once with the
15232 original mode and again with the elements lowered to QImode.
15233 So wait and don't do the selector expansion ourselves. */
15237 /* to_constant is safe since this routine is specific to Advanced SIMD
15238 vectors. */
15241 /* If big-endian and two vectors we end up with a weird mixed-endian
15242 mode on NEON. Reverse the index within each word but not the word
15243 itself. to_constant is safe because we checked is_constant above. */
15253 }
15255 /* Try to implement D using an SVE TBL instruction. */
15257 static bool
15259 {
15262 /* Permuting two variable-length vectors could overflow the
15263 index range. */
15274 }
15276 static bool
15278 {
15279 /* The pattern matching functions above are written to look for a small
15280 number to begin the sequence (0, 1, N/2). If we begin with an index
15281 from the second operand, we can swap the operands. */
15284 {
15287 }
15293 {
15312 }
15314 }
15316 /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
15318 static bool
15321 {
15324 /* Check whether the mask can be applied to a single vector. */
15328 {
15331 }
15333 {
15336 }
15337 else
15357 }
15359 /* Generate a byte permute mask for a register of mode MODE,
15360 which has NUNITS units. */
15362 rtx
15364 {
15365 /* We have to reverse each vector because we dont have
15366 a permuted load that can reverse-load according to ABI rules. */
15367 rtx mask;
15380 }
15382 /* Return true if X is a valid second operand for the SVE instruction
15383 that implements integer comparison OP_CODE. */
15385 static bool
15387 {
15392 {
15407 }
15408 }
15410 /* Return the UNSPEC_COND_* code for comparison CODE. */
15412 static unsigned int
15414 {
15416 {
15441 }
15442 }
15444 /* Return an (unspec:PRED_MODE [PRED OP0 OP1] UNSPEC_COND_<X>) expression,
15445 where <X> is the operation associated with comparison CODE. */
15447 static rtx
15450 {
15453 }
15455 /* Expand an SVE integer comparison:
15457 TARGET = CODE (OP0, OP1). */
15459 void
15461 {
15471 }
15473 /* Emit an instruction:
15475 (set TARGET (unspec:PRED_MODE [PRED OP0 OP1] UNSPEC_COND_<X>))
15477 where <X> is the operation associated with comparison CODE. */
15479 static void
15482 {
15485 }
15487 /* Emit:
15489 (set TMP1 (unspec:PRED_MODE [PTRUE OP0 OP1] UNSPEC_COND_<X1>))
15490 (set TMP2 (unspec:PRED_MODE [PTRUE OP0 OP1] UNSPEC_COND_<X2>))
15491 (set TARGET (and:PRED_MODE (ior:PRED_MODE TMP1 TMP2) PTRUE))
15493 where <Xi> is the operation associated with comparison CODEi. */
15495 static void
15499 {
15506 ptrue));
15507 }
15509 /* If CAN_INVERT_P, emit an instruction:
15511 (set TARGET (unspec:PRED_MODE [PRED OP0 OP1] UNSPEC_COND_<X>))
15513 where <X> is the operation associated with comparison CODE. Otherwise
15514 emit:
15516 (set TMP (unspec:PRED_MODE [PRED OP0 OP1] UNSPEC_COND_<X>))
15517 (set TARGET (and:PRED_MODE (not:PRED_MODE TMP) PTRUE))
15519 where the second instructions sets TARGET to the inverse of TMP. */
15521 static void
15525 {
15528 else
15529 {
15534 ptrue));
15535 }
15536 }
15538 /* Expand an SVE floating-point comparison:
15540 TARGET = CODE (OP0, OP1)
15542 If CAN_INVERT_P is true, the caller can also handle inverted results;
15543 return true if the result is in fact inverted. */
15545 bool
15548 {
15554 {
15556 /* UNORDERED has no immediate form. */
15567 /* There is native support for the comparison. */
15572 /* There is native support for the inverse comparison. */
15576 can_invert_p);
15580 /* This is a trapping operation (LT or GT). */
15586 {
15587 /* This would trap for signaling NaNs. */
15592 }
15593 /* fall through */
15599 {
15602 {
15603 /* Only compare the elements that are known to be ordered. */
15608 }
15611 else
15616 }
15620 }
15621 }
15623 /* Expand an SVE vcond pattern with operands OPS. DATA_MODE is the mode
15624 of the data being selected and CMP_MODE is the mode of the values being
15625 compared. */
15627 void
15630 {
15631 machine_mode pred_mode
15636 {
15640 }
15641 else
15646 }
15648 /* Implement TARGET_MODES_TIEABLE_P. In principle we should always return
15649 true. However due to issues with register allocation it is preferable
15650 to avoid tieing integer scalar and FP scalar modes. Executing integer
15651 operations in general registers is better than treating them as scalar
15652 vector operations. This reduces latency and avoids redundant int<->FP
15653 moves. So tie modes if they are either the same class, or vector modes
15654 with other vector modes, vector structs or any scalar mode. */
15656 static bool
15658 {
15662 /* We specifically want to allow elements of "structure" modes to
15663 be tieable to the structure. This more general condition allows
15664 other rarer situations too. The reason we don't extend this to
15665 predicate modes is that there are no predicate structure modes
15666 nor any specific instructions for extracting part of a predicate
15667 register. */
15672 /* Also allow any scalar modes with vectors. */
15678 }
15680 /* Return a new RTX holding the result of moving POINTER forward by
15681 AMOUNT bytes. */
15683 static rtx
15685 {
15690 }
15692 /* Return a new RTX holding the result of moving POINTER forward by the
15693 size of the mode it points to. */
15695 static rtx
15697 {
15699 }
15701 /* Copy one MODE sized block from SRC to DST, then progress SRC and DST by
15702 MODE bytes. */
15704 static void
15706 machine_mode mode)
15707 {
15710 /* "Cast" the pointers to the correct mode. */
15713 /* Emit the memcpy. */
15716 /* Move the pointers forward. */
15719 }
15721 /* Expand movmem, as if from a __builtin_memcpy. Return true if
15722 we succeed, otherwise return false. */
15724 bool
15726 {
15730 rtx base;
15733 /* When optimizing for size, give a better estimate of the length of a
15734 memcpy call, but use the default otherwise. */
15737 /* We can't do anything smart if the amount to copy is not constant. */
15743 /* Try to keep the number of instructions low. For cases below 16 bytes we
15744 need to make at most two moves. For cases above 16 bytes it will be one
15745 move for each 16 byte chunk, then at most two additional moves. */
15755 /* Simple cases. Copy 0-3 bytes, as (if applicable) a 2-byte, then a
15756 1-byte chunk. */
15758 {
15760 {
15763 }
15769 }
15771 /* Copy 4-8 bytes. First a 4-byte chunk, then (if applicable) a second
15772 4-byte chunk, partially overlapping with the previously copied chunk. */
15774 {
15778 {
15784 }
15786 }
15788 /* Copy more than 8 bytes. Copy chunks of 16 bytes until we run out of
15789 them, then (if applicable) an 8-byte chunk. */
15791 {
15793 {
15796 }
15797 else
15798 {
15801 }
15802 }
15804 /* Finish the final bytes of the copy. We can always do this in one
15805 instruction. We either copy the exact amount we need, or partially
15806 overlap with the previous chunk we copied and copy 8-bytes. */
15815 else
15816 {
15818 {
15822 }
15823 else
15824 {
15830 }
15831 }
15834 }
15836 /* Split a DImode store of a CONST_INT SRC to MEM DST as two
15837 SImode stores. Handle the case when the constant has identical
15838 bottom and top halves. This is beneficial when the two stores can be
15839 merged into an STP and we avoid synthesising potentially expensive
15840 immediates twice. Return true if such a split is possible. */
15842 bool
15844 {
15853 unsigned int orig_cost
15855 unsigned int lo_cost
15858 /* We want to transform:
15859 MOV x1, 49370
15860 MOVK x1, 0x140, lsl 16
15861 MOVK x1, 0xc0da, lsl 32
15862 MOVK x1, 0x140, lsl 48
15863 STR x1, [x0]
15864 into:
15865 MOV w1, 49370
15866 MOVK w1, 0x140, lsl 16
15867 STP w1, w1, [x0]
15868 So we want to perform this only when we save two instructions
15869 or more. When optimizing for size, however, accept any code size
15870 savings we can. */
15885 /* Don't emit an explicit store pair as this may not be always profitable.
15886 Let the sched-fusion logic decide whether to merge them. */
15891 }
15893 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
15895 static unsigned HOST_WIDE_INT
15897 {
15899 }
15901 static rtx
15904 {
15908 insn_code icode;
15919 {
15947 }
15952 {
15955 }
15964 {
15967 }
15973 }
15975 static rtx
15978 {
15982 insn_code icode;
15994 {
16022 }
16027 {
16030 }
16038 {
16043 }
16054 {
16057 }
16063 }
16065 #undef TARGET_GEN_CCMP_FIRST
16066 #define TARGET_GEN_CCMP_FIRST aarch64_gen_ccmp_first
16068 #undef TARGET_GEN_CCMP_NEXT
16069 #define TARGET_GEN_CCMP_NEXT aarch64_gen_ccmp_next
16071 /* Implement TARGET_SCHED_MACRO_FUSION_P. Return true if target supports
16072 instruction fusion of some sort. */
16074 static bool
16076 {
16078 }
16081 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P. Return true if PREV and CURR
16082 should be kept together during scheduling. */
16084 static bool
16086 {
16087 rtx set_dest;
16090 /* prev and curr are simple SET insns i.e. no flag setting or branching. */
16097 {
16098 /* We are trying to match:
16099 prev (mov) == (set (reg r0) (const_int imm16))
16100 curr (movk) == (set (zero_extract (reg r0)
16101 (const_int 16)
16102 (const_int 16))
16103 (const_int imm16_1)) */
16115 {
16117 }
16118 }
16121 {
16123 /* We're trying to match:
16124 prev (adrp) == (set (reg r1)
16125 (high (symbol_ref ("SYM"))))
16126 curr (add) == (set (reg r0)
16127 (lo_sum (reg r1)
16128 (symbol_ref ("SYM"))))
16129 Note that r0 need not necessarily be the same as r1, especially
16130 during pre-regalloc scheduling. */
16134 {
16142 }
16143 }
16146 {
16148 /* We're trying to match:
16149 prev (movk) == (set (zero_extract (reg r0)
16150 (const_int 16)
16151 (const_int 32))
16152 (const_int imm16_1))
16153 curr (movk) == (set (zero_extract (reg r0)
16154 (const_int 16)
16155 (const_int 48))
16156 (const_int imm16_2)) */
16172 }
16174 {
16175 /* We're trying to match:
16176 prev (adrp) == (set (reg r0)
16177 (high (symbol_ref ("SYM"))))
16178 curr (ldr) == (set (reg r1)
16179 (mem (lo_sum (reg r0)
16180 (symbol_ref ("SYM")))))
16181 or
16182 curr (ldr) == (set (reg r1)
16183 (zero_extend (mem
16184 (lo_sum (reg r0)
16185 (symbol_ref ("SYM")))))) */
16188 {
16201 }
16202 }
16210 {
16214 rtx cc_reg_1;
16219 && prev
16221 {
16222 /* FIXME: this misses some which is considered simple arthematic
16223 instructions for ThunderX. Simple shifts are missed here. */
16229 }
16230 }
16233 && curr_set
16236 {
16237 /* We're trying to match:
16238 prev (alu_insn) == (set (r0) plus ((r0) (r1/imm)))
16239 curr (cbz) == (set (pc) (if_then_else (eq/ne) (r0)
16240 (const_int 0))
16241 (label_ref ("SYM"))
16242 (pc)) */
16249 {
16250 /* Fuse ALU operations followed by conditional branch instruction. */
16252 {
16273 }
16274 }
16275 }
16278 }
16280 /* Return true iff the instruction fusion described by OP is enabled. */
16282 bool
16284 {
16286 }
16288 /* If MEM is in the form of [base+offset], extract the two parts
16289 of address and set to BASE and OFFSET, otherwise return false
16290 after clearing BASE and OFFSET. */
16292 bool
16294 {
16295 rtx addr;
16302 {
16306 }
16310 {
16314 }
16320 }
16322 /* Types for scheduling fusion. */
16323 enum sched_fusion_type
16324 {
16326 SCHED_FUSION_LD_SIGN_EXTEND,
16327 SCHED_FUSION_LD_ZERO_EXTEND,
16328 SCHED_FUSION_LD,
16329 SCHED_FUSION_ST,
16330 SCHED_FUSION_NUM
16331 };
16333 /* If INSN is a load or store of address in the form of [base+offset],
16334 extract the two parts and set to BASE and OFFSET. Return scheduling
16335 fusion type this INSN is. */
16337 static enum sched_fusion_type
16339 {
16357 {
16362 }
16364 {
16369 }
16374 {
16377 }
16378 else
16385 }
16387 /* Implement the TARGET_SCHED_FUSION_PRIORITY hook.
16389 Currently we only support to fuse ldr or str instructions, so FUSION_PRI
16390 and PRI are only calculated for these instructions. For other instruction,
16391 FUSION_PRI and PRI are simply set to MAX_PRI - 1. In the future, other
16392 type instruction fusion can be added by returning different priorities.
16394 It's important that irrelevant instructions get the largest FUSION_PRI. */
16396 static void
16399 {
16409 {
16413 }
16415 /* Set FUSION_PRI according to fusion type and base register. */
16418 /* Calculate PRI. */
16421 /* INSN with smaller offset goes first. */
16425 else
16430 }
16432 /* Implement the TARGET_SCHED_ADJUST_PRIORITY hook.
16433 Adjust priority of sha1h instructions so they are scheduled before
16434 other SHA1 instructions. */
16436 static int
16438 {
16442 {
16447 }
16450 }
16452 /* Given OPERANDS of consecutive load/store, check if we can merge
16453 them into ldp/stp. LOAD is true if they are load instructions.
16454 MODE is the mode of memory operands. */
16456 bool
16458 machine_mode mode)
16459 {
16465 {
16473 }
16474 else
16475 {
16480 }
16482 /* The mems cannot be volatile. */
16486 /* If we have SImode and slow unaligned ldp,
16487 check the alignment to be at least 8 byte. */
16491 && !optimize_size
16495 /* Check if the addresses are in the form of [base+offset]. */
16503 /* Check if the bases are same. */
16509 /* We should only be trying this for fixed-sized modes. There is no
16510 SVE LDP/STP instruction. */
16512 /* Check if the offsets are consecutive. */
16516 /* Check if the addresses are clobbered by load. */
16518 {
16522 /* In increasing order, the last load can clobber the address. */
16525 }
16529 else
16534 else
16537 /* Check if the registers are of same class. */
16542 }
16544 /* Given OPERANDS of consecutive load/store, check if we can merge
16545 them into ldp/stp by adjusting the offset. LOAD is true if they
16546 are load instructions. MODE is the mode of memory operands.
16548 Given below consecutive stores:
16550 str w1, [xb, 0x100]
16551 str w1, [xb, 0x104]
16552 str w1, [xb, 0x108]
16553 str w1, [xb, 0x10c]
16555 Though the offsets are out of the range supported by stp, we can
16556 still pair them after adjusting the offset, like:
16558 add scratch, xb, 0x100
16559 stp w1, w1, [scratch]
16560 stp w1, w1, [scratch, 0x8]
16562 The peephole patterns detecting this opportunity should guarantee
16563 the scratch register is avaliable. */
16565 bool
16567 scalar_mode mode)
16568 {
16575 {
16588 }
16589 else
16590 {
16599 }
16600 /* Skip if memory operand is by itslef valid for ldp/stp. */
16604 /* The mems cannot be volatile. */
16609 /* Check if the addresses are in the form of [base+offset]. */
16623 /* Check if the bases are same. */
16634 /* Check if the offsets are consecutive. */
16643 /* Check if the addresses are clobbered by load. */
16645 {
16651 /* In increasing order, the last load can clobber the address. */
16654 }
16656 /* If we have SImode and slow unaligned ldp,
16657 check the alignment to be at least 8 byte. */
16661 && !optimize_size
16667 else
16672 else
16677 else
16682 else
16685 /* Check if the registers are of same class. */
16690 }
16692 /* Given OPERANDS of consecutive load/store, this function pairs them
16693 into ldp/stp after adjusting the offset. It depends on the fact
16694 that addresses of load/store instructions are in increasing order.
16695 MODE is the mode of memory operands. CODE is the rtl operator
16696 which should be applied to all memory operands, it's SIGN_EXTEND,
16697 ZERO_EXTEND or UNKNOWN. */
16699 bool
16702 {
16708 {
16713 }
16714 else
16715 {
16721 }
16726 /* Adjust offset thus it can fit in ldp/stp instruction. */
16734 /* Further adjust to make sure all offsets are OK. */
16736 {
16739 }
16741 /* Make sure the adjustment can be done with ADD/SUB instructions. */
16746 {
16749 }
16751 /* Create new memory references. */
16755 /* Check if the adjusted address is OK for ldp/stp. */
16774 {
16779 }
16781 {
16786 }
16789 {
16794 }
16795 else
16796 {
16801 }
16803 /* Emit adjusting instruction. */
16805 /* Emit ldp/stp instructions. */
16813 }
16815 /* Return 1 if pseudo register should be created and used to hold
16816 GOT address for PIC code. */
16818 bool
16820 {
16822 }
16824 /* Implement TARGET_UNSPEC_MAY_TRAP_P. */
16826 static int
16828 {
16830 {
16837 }
16840 }
16843 /* If X is a positive CONST_DOUBLE with a value that is a power of 2
16844 return the log2 of that value. Otherwise return -1. */
16846 int
16848 {
16863 }
16865 /* If X is a vector of equal CONST_DOUBLE values and that value is
16866 Y, return the aarch64_fpconst_pow_of_2 of Y. Otherwise return -1. */
16868 int
16870 {
16888 }
16890 /* Implement TARGET_PROMOTED_TYPE to promote 16-bit floating point types
16891 to float.
16893 __fp16 always promotes through this hook.
16894 _Float16 may promote if TARGET_FLT_EVAL_METHOD is 16, but we do that
16895 through the generic excess precision logic rather than here. */
16897 static tree
16899 {
16905 }
16907 /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
16909 static bool
16911 optimization_type opt_type)
16912 {
16914 {
16920 }
16921 }
16923 /* Implement the TARGET_DWARF_POLY_INDETERMINATE_VALUE hook. */
16925 static unsigned int
16928 {
16929 /* Polynomial invariant 1 == (VG / 2) - 1. */
16934 }
16936 /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
16937 if MODE is HFmode, and punt to the generic implementation otherwise. */
16939 static bool
16941 {
16945 }
16947 /* Implement TARGET_SCALAR_MODE_SUPPORTED_P - return TRUE
16948 if MODE is HFmode, and punt to the generic implementation otherwise. */
16950 static bool
16952 {
16956 }
16958 /* Set the value of FLT_EVAL_METHOD.
16959 ISO/IEC TS 18661-3 defines two values that we'd like to make use of:
16961 0: evaluate all operations and constants, whose semantic type has at
16962 most the range and precision of type float, to the range and
16963 precision of float; evaluate all other operations and constants to
16964 the range and precision of the semantic type;
16966 N, where _FloatN is a supported interchange floating type
16967 evaluate all operations and constants, whose semantic type has at
16968 most the range and precision of _FloatN type, to the range and
16969 precision of the _FloatN type; evaluate all other operations and
16970 constants to the range and precision of the semantic type;
16972 If we have the ARMv8.2-A extensions then we support _Float16 in native
16973 precision, so we should set this to 16. Otherwise, we support the type,
16974 but want to evaluate expressions in float precision, so set this to
16975 0. */
16977 static enum flt_eval_method
16979 {
16981 {
16984 /* We can calculate either in 16-bit range and precision or
16985 32-bit range and precision. Make that decision based on whether
16986 we have native support for the ARMv8.2-A 16-bit floating-point
16987 instructions or not. */
16989 ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
16995 }
16997 }
16999 /* Implement TARGET_SCHED_CAN_SPECULATE_INSN. Return true if INSN can be
17000 scheduled for speculative execution. Reject the long-running division
17001 and square-root instructions. */
17003 static bool
17005 {
17007 {
17025 }
17026 }
17028 /* Implement TARGET_COMPUTE_PRESSURE_CLASSES. */
17030 static int
17032 {
17036 /* PR_REGS isn't a useful pressure class because many predicate pseudo
17037 registers need to go in PR_LO_REGS at some point during their
17038 lifetime. Splitting it into two halves has the effect of making
17039 all predicates count against PR_LO_REGS, so that we try whenever
17040 possible to restrict the number of live predicates to 8. This
17041 greatly reduces the amount of spilling in certain loops. */
17045 }
17047 /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
17049 static bool
17052 {
17053 /* See the comment at the head of aarch64-sve.md for details. */
17058 }
17060 /* Target-specific selftests. */
17062 #if CHECKING_P
17066 /* Selftest for the RTL loader.
17067 Verify that the RTL loader copes with a dump from
17068 print_rtx_function. This is essentially just a test that class
17069 function_reader can handle a real dump, but it also verifies
17070 that lookup_reg_by_dump_name correctly handles hard regs.
17071 The presence of hard reg names in the dump means that the test is
17072 target-specific, hence it is in this file. */
17074 static void
17076 {
17088 /* Verify crtl->return_rtx. */
17092 }
17094 /* Run all target-specific selftests. */
17096 static void
17098 {
17100 }
17106 #undef TARGET_ADDRESS_COST
17107 #define TARGET_ADDRESS_COST aarch64_address_cost
17109 /* This hook will determines whether unnamed bitfields affect the alignment
17110 of the containing structure. The hook returns true if the structure
17111 should inherit the alignment requirements of an unnamed bitfield's
17112 type. */
17113 #undef TARGET_ALIGN_ANON_BITFIELD
17114 #define TARGET_ALIGN_ANON_BITFIELD hook_bool_void_true
17116 #undef TARGET_ASM_ALIGNED_DI_OP
17119 #undef TARGET_ASM_ALIGNED_HI_OP
17122 #undef TARGET_ASM_ALIGNED_SI_OP
17125 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
17126 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK \
17127 hook_bool_const_tree_hwi_hwi_const_tree_true
17129 #undef TARGET_ASM_FILE_START
17130 #define TARGET_ASM_FILE_START aarch64_start_file
17132 #undef TARGET_ASM_OUTPUT_MI_THUNK
17133 #define TARGET_ASM_OUTPUT_MI_THUNK aarch64_output_mi_thunk
17135 #undef TARGET_ASM_SELECT_RTX_SECTION
17136 #define TARGET_ASM_SELECT_RTX_SECTION aarch64_select_rtx_section
17138 #undef TARGET_ASM_TRAMPOLINE_TEMPLATE
17139 #define TARGET_ASM_TRAMPOLINE_TEMPLATE aarch64_asm_trampoline_template
17141 #undef TARGET_BUILD_BUILTIN_VA_LIST
17142 #define TARGET_BUILD_BUILTIN_VA_LIST aarch64_build_builtin_va_list
17144 #undef TARGET_CALLEE_COPIES
17145 #define TARGET_CALLEE_COPIES hook_bool_CUMULATIVE_ARGS_mode_tree_bool_false
17147 #undef TARGET_CAN_ELIMINATE
17148 #define TARGET_CAN_ELIMINATE aarch64_can_eliminate
17150 #undef TARGET_CAN_INLINE_P
17151 #define TARGET_CAN_INLINE_P aarch64_can_inline_p
17153 #undef TARGET_CANNOT_FORCE_CONST_MEM
17154 #define TARGET_CANNOT_FORCE_CONST_MEM aarch64_cannot_force_const_mem
17156 #undef TARGET_CASE_VALUES_THRESHOLD
17157 #define TARGET_CASE_VALUES_THRESHOLD aarch64_case_values_threshold
17159 #undef TARGET_CONDITIONAL_REGISTER_USAGE
17160 #define TARGET_CONDITIONAL_REGISTER_USAGE aarch64_conditional_register_usage
17162 /* Only the least significant bit is used for initialization guard
17163 variables. */
17164 #undef TARGET_CXX_GUARD_MASK_BIT
17165 #define TARGET_CXX_GUARD_MASK_BIT hook_bool_void_true
17167 #undef TARGET_C_MODE_FOR_SUFFIX
17168 #define TARGET_C_MODE_FOR_SUFFIX aarch64_c_mode_for_suffix
17170 #ifdef TARGET_BIG_ENDIAN_DEFAULT
17171 #undef TARGET_DEFAULT_TARGET_FLAGS
17172 #define TARGET_DEFAULT_TARGET_FLAGS (MASK_BIG_END)
17173 #endif
17175 #undef TARGET_CLASS_MAX_NREGS
17176 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
17178 #undef TARGET_BUILTIN_DECL
17179 #define TARGET_BUILTIN_DECL aarch64_builtin_decl
17181 #undef TARGET_BUILTIN_RECIPROCAL
17182 #define TARGET_BUILTIN_RECIPROCAL aarch64_builtin_reciprocal
17184 #undef TARGET_C_EXCESS_PRECISION
17185 #define TARGET_C_EXCESS_PRECISION aarch64_excess_precision
17187 #undef TARGET_EXPAND_BUILTIN
17188 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
17190 #undef TARGET_EXPAND_BUILTIN_VA_START
17191 #define TARGET_EXPAND_BUILTIN_VA_START aarch64_expand_builtin_va_start
17193 #undef TARGET_FOLD_BUILTIN
17194 #define TARGET_FOLD_BUILTIN aarch64_fold_builtin
17196 #undef TARGET_FUNCTION_ARG
17197 #define TARGET_FUNCTION_ARG aarch64_function_arg
17199 #undef TARGET_FUNCTION_ARG_ADVANCE
17200 #define TARGET_FUNCTION_ARG_ADVANCE aarch64_function_arg_advance
17202 #undef TARGET_FUNCTION_ARG_BOUNDARY
17203 #define TARGET_FUNCTION_ARG_BOUNDARY aarch64_function_arg_boundary
17205 #undef TARGET_FUNCTION_ARG_PADDING
17206 #define TARGET_FUNCTION_ARG_PADDING aarch64_function_arg_padding
17208 #undef TARGET_GET_RAW_RESULT_MODE
17209 #define TARGET_GET_RAW_RESULT_MODE aarch64_get_reg_raw_mode
17210 #undef TARGET_GET_RAW_ARG_MODE
17211 #define TARGET_GET_RAW_ARG_MODE aarch64_get_reg_raw_mode
17213 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
17214 #define TARGET_FUNCTION_OK_FOR_SIBCALL aarch64_function_ok_for_sibcall
17216 #undef TARGET_FUNCTION_VALUE
17217 #define TARGET_FUNCTION_VALUE aarch64_function_value
17219 #undef TARGET_FUNCTION_VALUE_REGNO_P
17220 #define TARGET_FUNCTION_VALUE_REGNO_P aarch64_function_value_regno_p
17222 #undef TARGET_GIMPLE_FOLD_BUILTIN
17223 #define TARGET_GIMPLE_FOLD_BUILTIN aarch64_gimple_fold_builtin
17225 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
17226 #define TARGET_GIMPLIFY_VA_ARG_EXPR aarch64_gimplify_va_arg_expr
17228 #undef TARGET_INIT_BUILTINS
17229 #define TARGET_INIT_BUILTINS aarch64_init_builtins
17231 #undef TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS
17232 #define TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS \
17233 aarch64_ira_change_pseudo_allocno_class
17235 #undef TARGET_LEGITIMATE_ADDRESS_P
17236 #define TARGET_LEGITIMATE_ADDRESS_P aarch64_legitimate_address_hook_p
17238 #undef TARGET_LEGITIMATE_CONSTANT_P
17239 #define TARGET_LEGITIMATE_CONSTANT_P aarch64_legitimate_constant_p
17241 #undef TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT
17242 #define TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT \
17243 aarch64_legitimize_address_displacement
17245 #undef TARGET_LIBGCC_CMP_RETURN_MODE
17246 #define TARGET_LIBGCC_CMP_RETURN_MODE aarch64_libgcc_cmp_return_mode
17248 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
17249 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
17250 aarch64_libgcc_floating_mode_supported_p
17252 #undef TARGET_MANGLE_TYPE
17253 #define TARGET_MANGLE_TYPE aarch64_mangle_type
17255 #undef TARGET_MEMORY_MOVE_COST
17256 #define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
17258 #undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
17259 #define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
17261 #undef TARGET_MUST_PASS_IN_STACK
17262 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
17264 /* This target hook should return true if accesses to volatile bitfields
17265 should use the narrowest mode possible. It should return false if these
17266 accesses should use the bitfield container type. */
17267 #undef TARGET_NARROW_VOLATILE_BITFIELD
17268 #define TARGET_NARROW_VOLATILE_BITFIELD hook_bool_void_false
17270 #undef TARGET_OPTION_OVERRIDE
17271 #define TARGET_OPTION_OVERRIDE aarch64_override_options
17273 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
17274 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE \
17275 aarch64_override_options_after_change
17277 #undef TARGET_OPTION_SAVE
17278 #define TARGET_OPTION_SAVE aarch64_option_save
17280 #undef TARGET_OPTION_RESTORE
17281 #define TARGET_OPTION_RESTORE aarch64_option_restore
17283 #undef TARGET_OPTION_PRINT
17284 #define TARGET_OPTION_PRINT aarch64_option_print
17286 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
17287 #define TARGET_OPTION_VALID_ATTRIBUTE_P aarch64_option_valid_attribute_p
17289 #undef TARGET_SET_CURRENT_FUNCTION
17290 #define TARGET_SET_CURRENT_FUNCTION aarch64_set_current_function
17292 #undef TARGET_PASS_BY_REFERENCE
17293 #define TARGET_PASS_BY_REFERENCE aarch64_pass_by_reference
17295 #undef TARGET_PREFERRED_RELOAD_CLASS
17296 #define TARGET_PREFERRED_RELOAD_CLASS aarch64_preferred_reload_class
17298 #undef TARGET_SCHED_REASSOCIATION_WIDTH
17299 #define TARGET_SCHED_REASSOCIATION_WIDTH aarch64_reassociation_width
17301 #undef TARGET_PROMOTED_TYPE
17302 #define TARGET_PROMOTED_TYPE aarch64_promoted_type
17304 #undef TARGET_SECONDARY_RELOAD
17305 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
17307 #undef TARGET_SHIFT_TRUNCATION_MASK
17308 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
17310 #undef TARGET_SETUP_INCOMING_VARARGS
17311 #define TARGET_SETUP_INCOMING_VARARGS aarch64_setup_incoming_varargs
17313 #undef TARGET_STRUCT_VALUE_RTX
17314 #define TARGET_STRUCT_VALUE_RTX aarch64_struct_value_rtx
17316 #undef TARGET_REGISTER_MOVE_COST
17317 #define TARGET_REGISTER_MOVE_COST aarch64_register_move_cost
17319 #undef TARGET_RETURN_IN_MEMORY
17320 #define TARGET_RETURN_IN_MEMORY aarch64_return_in_memory
17322 #undef TARGET_RETURN_IN_MSB
17323 #define TARGET_RETURN_IN_MSB aarch64_return_in_msb
17325 #undef TARGET_RTX_COSTS
17326 #define TARGET_RTX_COSTS aarch64_rtx_costs_wrapper
17328 #undef TARGET_SCALAR_MODE_SUPPORTED_P
17329 #define TARGET_SCALAR_MODE_SUPPORTED_P aarch64_scalar_mode_supported_p
17331 #undef TARGET_SCHED_ISSUE_RATE
17332 #define TARGET_SCHED_ISSUE_RATE aarch64_sched_issue_rate
17334 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
17335 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
17336 aarch64_sched_first_cycle_multipass_dfa_lookahead
17338 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
17339 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD \
17340 aarch64_first_cycle_multipass_dfa_lookahead_guard
17342 #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
17343 #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS \
17344 aarch64_get_separate_components
17346 #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
17347 #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB \
17348 aarch64_components_for_bb
17350 #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
17351 #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS \
17352 aarch64_disqualify_components
17354 #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
17355 #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
17356 aarch64_emit_prologue_components
17358 #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
17359 #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
17360 aarch64_emit_epilogue_components
17362 #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
17363 #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS \
17364 aarch64_set_handled_components
17366 #undef TARGET_TRAMPOLINE_INIT
17367 #define TARGET_TRAMPOLINE_INIT aarch64_trampoline_init
17369 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
17370 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P aarch64_use_blocks_for_constant_p
17372 #undef TARGET_VECTOR_MODE_SUPPORTED_P
17373 #define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
17375 #undef TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT
17376 #define TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT \
17377 aarch64_builtin_support_vector_misalignment
17379 #undef TARGET_ARRAY_MODE_SUPPORTED_P
17380 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
17382 #undef TARGET_VECTORIZE_ADD_STMT_COST
17383 #define TARGET_VECTORIZE_ADD_STMT_COST aarch64_add_stmt_cost
17385 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
17386 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
17387 aarch64_builtin_vectorization_cost
17389 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
17390 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
17392 #undef TARGET_VECTORIZE_BUILTINS
17393 #define TARGET_VECTORIZE_BUILTINS
17395 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
17396 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
17397 aarch64_builtin_vectorized_function
17399 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
17400 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
17401 aarch64_autovectorize_vector_sizes
17403 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
17404 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV \
17405 aarch64_atomic_assign_expand_fenv
17407 /* Section anchor support. */
17409 #undef TARGET_MIN_ANCHOR_OFFSET
17410 #define TARGET_MIN_ANCHOR_OFFSET -256
17412 /* Limit the maximum anchor offset to 4k-1, since that's the limit for a
17413 byte offset; we can do much more for larger data types, but have no way
17414 to determine the size of the access. We assume accesses are aligned. */
17415 #undef TARGET_MAX_ANCHOR_OFFSET
17416 #define TARGET_MAX_ANCHOR_OFFSET 4095
17418 #undef TARGET_VECTOR_ALIGNMENT
17419 #define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
17421 #undef TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
17422 #define TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT \
17423 aarch64_vectorize_preferred_vector_alignment
17424 #undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
17425 #define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
17426 aarch64_simd_vector_alignment_reachable
17428 /* vec_perm support. */
17430 #undef TARGET_VECTORIZE_VEC_PERM_CONST
17431 #define TARGET_VECTORIZE_VEC_PERM_CONST \
17432 aarch64_vectorize_vec_perm_const
17434 #undef TARGET_VECTORIZE_GET_MASK_MODE
17435 #define TARGET_VECTORIZE_GET_MASK_MODE aarch64_get_mask_mode
17437 #undef TARGET_INIT_LIBFUNCS
17438 #define TARGET_INIT_LIBFUNCS aarch64_init_libfuncs
17440 #undef TARGET_FIXED_CONDITION_CODE_REGS
17441 #define TARGET_FIXED_CONDITION_CODE_REGS aarch64_fixed_condition_code_regs
17443 #undef TARGET_FLAGS_REGNUM
17444 #define TARGET_FLAGS_REGNUM CC_REGNUM
17446 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
17447 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
17449 #undef TARGET_ASAN_SHADOW_OFFSET
17450 #define TARGET_ASAN_SHADOW_OFFSET aarch64_asan_shadow_offset
17452 #undef TARGET_LEGITIMIZE_ADDRESS
17453 #define TARGET_LEGITIMIZE_ADDRESS aarch64_legitimize_address
17455 #undef TARGET_SCHED_CAN_SPECULATE_INSN
17456 #define TARGET_SCHED_CAN_SPECULATE_INSN aarch64_sched_can_speculate_insn
17458 #undef TARGET_CAN_USE_DOLOOP_P
17459 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
17461 #undef TARGET_SCHED_ADJUST_PRIORITY
17462 #define TARGET_SCHED_ADJUST_PRIORITY aarch64_sched_adjust_priority
17464 #undef TARGET_SCHED_MACRO_FUSION_P
17465 #define TARGET_SCHED_MACRO_FUSION_P aarch64_macro_fusion_p
17467 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
17468 #define TARGET_SCHED_MACRO_FUSION_PAIR_P aarch_macro_fusion_pair_p
17470 #undef TARGET_SCHED_FUSION_PRIORITY
17471 #define TARGET_SCHED_FUSION_PRIORITY aarch64_sched_fusion_priority
17473 #undef TARGET_UNSPEC_MAY_TRAP_P
17474 #define TARGET_UNSPEC_MAY_TRAP_P aarch64_unspec_may_trap_p
17476 #undef TARGET_USE_PSEUDO_PIC_REG
17477 #define TARGET_USE_PSEUDO_PIC_REG aarch64_use_pseudo_pic_reg
17479 #undef TARGET_PRINT_OPERAND
17480 #define TARGET_PRINT_OPERAND aarch64_print_operand
17482 #undef TARGET_PRINT_OPERAND_ADDRESS
17483 #define TARGET_PRINT_OPERAND_ADDRESS aarch64_print_operand_address
17485 #undef TARGET_OPTAB_SUPPORTED_P
17486 #define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p
17488 #undef TARGET_OMIT_STRUCT_RETURN_REG
17489 #define TARGET_OMIT_STRUCT_RETURN_REG true
17491 #undef TARGET_DWARF_POLY_INDETERMINATE_VALUE
17492 #define TARGET_DWARF_POLY_INDETERMINATE_VALUE \
17493 aarch64_dwarf_poly_indeterminate_value
17495 /* The architecture reserves bits 0 and 1 so use bit 2 for descriptors. */
17496 #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
17497 #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 4
17499 #undef TARGET_HARD_REGNO_NREGS
17500 #define TARGET_HARD_REGNO_NREGS aarch64_hard_regno_nregs
17501 #undef TARGET_HARD_REGNO_MODE_OK
17502 #define TARGET_HARD_REGNO_MODE_OK aarch64_hard_regno_mode_ok
17504 #undef TARGET_MODES_TIEABLE_P
17505 #define TARGET_MODES_TIEABLE_P aarch64_modes_tieable_p
17507 #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
17508 #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
17509 aarch64_hard_regno_call_part_clobbered
17511 #undef TARGET_CONSTANT_ALIGNMENT
17512 #define TARGET_CONSTANT_ALIGNMENT aarch64_constant_alignment
17514 #undef TARGET_COMPUTE_PRESSURE_CLASSES
17515 #define TARGET_COMPUTE_PRESSURE_CLASSES aarch64_compute_pressure_classes
17517 #undef TARGET_CAN_CHANGE_MODE_CLASS
17518 #define TARGET_CAN_CHANGE_MODE_CLASS aarch64_can_change_mode_class
17520 #if CHECKING_P
17521 #undef TARGET_RUN_TARGET_SELFTESTS
17522 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests