| blob | 13b5448aca88555222481f0955237b6fdcbb38b9 |
1 /* Machine description for AArch64 architecture.
2 Copyright (C) 2009-2018 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
24 #define INCLUDE_STRING
75 /* This file should be included last. */
78 /* Defined for convenience. */
79 #define POINTER_BYTES (POINTER_SIZE / BITS_PER_UNIT)
81 /* Classifies an address.
83 ADDRESS_REG_IMM
84 A simple base register plus immediate offset.
86 ADDRESS_REG_WB
87 A base register indexed by immediate offset with writeback.
89 ADDRESS_REG_REG
90 A base register indexed by (optionally scaled) register.
92 ADDRESS_REG_UXTW
93 A base register indexed by (optionally scaled) zero-extended register.
95 ADDRESS_REG_SXTW
96 A base register indexed by (optionally scaled) sign-extended register.
98 ADDRESS_LO_SUM
99 A LO_SUM rtx with a base register and "LO12" symbol relocation.
101 ADDRESS_SYMBOLIC:
102 A constant symbolic address, in pc-relative literal pool. */
105 ADDRESS_REG_IMM,
106 ADDRESS_REG_WB,
107 ADDRESS_REG_REG,
108 ADDRESS_REG_UXTW,
109 ADDRESS_REG_SXTW,
110 ADDRESS_LO_SUM,
111 ADDRESS_SYMBOLIC
112 };
116 rtx base;
117 rtx offset;
118 poly_int64 const_offset;
121 };
123 /* Information about a legitimate vector immediate operand. */
124 struct simd_immediate_info
125 {
136 /* The mode of the elements. */
137 scalar_mode elt_mode;
139 /* The value of each element if all elements are the same, or the
140 first value if the constant is a series. */
141 rtx value;
143 /* The value of the step if the constant is a series, null otherwise. */
144 rtx step;
146 /* The instruction to use to move the immediate into a vector. */
147 insn_type insn;
149 /* The kind of shift modifier to use, and the number of bits to shift.
150 This is (LSL, 0) if no shift is needed. */
151 modifier_type modifier;
153 };
155 /* Construct a floating-point immediate in which each element has mode
156 ELT_MODE_IN and value VALUE_IN. */
157 inline simd_immediate_info
161 {}
163 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
164 and value VALUE_IN. The other parameters are as for the structure
165 fields. */
166 inline simd_immediate_info
173 {}
175 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
176 and where element I is equal to VALUE_IN + I * STEP_IN. */
177 inline simd_immediate_info
181 {}
183 /* The current code model. */
186 /* The number of 64-bit elements in an SVE vector. */
187 poly_uint16 aarch64_sve_vg;
189 #ifdef HAVE_AS_TLS
190 #undef TARGET_HAVE_TLS
191 #define TARGET_HAVE_TLS 1
192 #endif
196 const_tree,
205 const_tree type,
210 aarch64_addr_query_type);
212 /* Major revision number of the ARM Architecture implemented by the target. */
215 /* The processor for which instructions should be scheduled. */
218 /* Mask to specify which instruction scheduling options should be used. */
221 /* Global flag for PC relative loads. */
224 /* Global flag for whether frame pointer is enabled. */
227 /* Support for command line parsing of boolean flags in the tuning
228 structures. */
229 struct aarch64_flag_desc
230 {
233 };
235 #define AARCH64_FUSION_PAIR(name, internal_name) \
236 { name, AARCH64_FUSE_##internal_name },
238 {
243 };
245 #define AARCH64_EXTRA_TUNING_OPTION(name, internal_name) \
246 { name, AARCH64_EXTRA_TUNE_##internal_name },
248 {
253 };
255 /* Tuning parameters. */
258 {
259 {
264 },
271 };
274 {
275 {
280 },
287 };
290 {
291 {
296 },
303 };
306 {
307 {
312 },
319 };
322 {
323 {
328 },
335 };
338 {
340 /* Avoid the use of slow int<->fp moves for spilling by setting
341 their cost higher than memmov_cost. */
345 };
348 {
350 /* Avoid the use of slow int<->fp moves for spilling by setting
351 their cost higher than memmov_cost. */
355 };
358 {
360 /* Avoid the use of slow int<->fp moves for spilling by setting
361 their cost higher than memmov_cost. */
365 };
368 {
370 /* Avoid the use of slow int<->fp moves for spilling by setting
371 their cost higher than memmov_cost (actual, 4 and 9). */
375 };
378 {
383 };
386 {
388 /* Avoid the use of slow int<->fp moves for spilling by setting
389 their cost higher than memmov_cost. */
393 };
396 {
398 /* Avoid the use of int<->fp moves for spilling. */
402 };
405 {
407 /* Avoid the use of int<->fp moves for spilling. */
411 };
413 /* Generic costs for vector insn classes. */
415 {
431 };
433 /* ThunderX costs for vector insn classes. */
435 {
451 };
453 /* Generic costs for vector insn classes. */
455 {
471 };
474 {
490 };
492 /* Generic costs for vector insn classes. */
494 {
510 };
512 /* Costs for vector insn classes for Vulcan. */
514 {
530 };
532 /* Generic costs for branch instructions. */
534 {
537 };
539 /* Generic approximation modes. */
541 {
544 AARCH64_APPROX_NONE /* recip_sqrt */
545 };
547 /* Approximation modes for Exynos M1. */
549 {
552 AARCH64_APPROX_ALL /* recip_sqrt */
553 };
555 /* Approximation modes for X-Gene 1. */
557 {
560 AARCH64_APPROX_ALL /* recip_sqrt */
561 };
563 /* Generic prefetch settings (which disable prefetch). */
565 {
573 };
576 {
584 };
587 {
595 };
598 {
606 };
609 {
617 };
620 {
628 };
631 {
652 &generic_prefetch_tune
653 };
656 {
678 &generic_prefetch_tune
679 };
682 {
704 &generic_prefetch_tune
705 };
708 {
730 &generic_prefetch_tune
731 };
734 {
756 &generic_prefetch_tune
757 };
760 {
782 &generic_prefetch_tune
783 };
788 {
809 &exynosm1_prefetch_tune
810 };
813 {
834 &thunderxt88_prefetch_tune
835 };
838 {
858 (AARCH64_EXTRA_TUNE_SLOW_UNALIGNED_LDPW
860 &thunderx_prefetch_tune
861 };
864 {
885 &generic_prefetch_tune
886 };
889 {
911 &qdf24xx_prefetch_tune
912 };
914 /* Tuning structure for the Qualcomm Saphira core. Default to falkor values
915 for now. */
917 {
939 &generic_prefetch_tune
940 };
943 {
965 &thunderx2t99_prefetch_tune
966 };
968 /* Support for fine-grained override of the tuning structures. */
969 struct aarch64_tuning_override_function
970 {
973 };
978 static const struct aarch64_tuning_override_function
979 aarch64_tuning_override_functions[] =
980 {
984 };
986 /* A processor implementing AArch64. */
987 struct processor
988 {
996 };
998 /* Architectures implementing AArch64. */
1000 {
1001 #define AARCH64_ARCH(NAME, CORE, ARCH_IDENT, ARCH_REV, FLAGS) \
1002 {NAME, CORE, CORE, AARCH64_ARCH_##ARCH_IDENT, ARCH_REV, FLAGS, NULL},
1005 };
1007 /* Processor cores implementing AArch64. */
1009 {
1010 #define AARCH64_CORE(NAME, IDENT, SCHED, ARCH, FLAGS, COSTS, IMP, PART, VARIANT) \
1011 {NAME, IDENT, SCHED, AARCH64_ARCH_##ARCH, \
1012 all_architectures[AARCH64_ARCH_##ARCH].architecture_version, \
1013 FLAGS, &COSTS##_tunings},
1018 };
1021 /* Target specification. These are populated by the -march, -mtune, -mcpu
1022 handling code or by target attributes. */
1027 /* The current tuning set. */
1030 #define AARCH64_CPU_DEFAULT_FLAGS ((selected_cpu) ? selected_cpu->flags : 0)
1032 /* An ISA extension in the co-processor and main instruction set space. */
1033 struct aarch64_option_extension
1034 {
1038 };
1041 {
1045 }
1046 aarch64_cc;
1048 #define AARCH64_INVERSE_CONDITION_CODE(X) ((aarch64_cc) (((int) X) ^ 1))
1050 /* The condition codes of the processor, and the inverse function. */
1052 {
1055 };
1057 /* Generate code to enable conditional branches in functions over 1 MiB. */
1061 {
1078 }
1080 void
1082 {
1087 else
1090 else
1094 else
1097 }
1099 /* Implement TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS.
1100 The register allocator chooses POINTER_AND_FP_REGS if FP_REGS and
1101 GENERAL_REGS have the same cost - even if POINTER_AND_FP_REGS has a much
1102 higher cost. POINTER_AND_FP_REGS is also used if the cost of both FP_REGS
1103 and GENERAL_REGS is lower than the memory cost (in this case the best class
1104 is the lowest cost one). Using POINTER_AND_FP_REGS irrespectively of its
1105 cost results in bad allocations with many redundant int<->FP moves which
1106 are expensive on various cores.
1107 To avoid this we don't allow POINTER_AND_FP_REGS as the allocno class, but
1108 force a decision between FP_REGS and GENERAL_REGS. We use the allocno class
1109 if it isn't POINTER_AND_FP_REGS. Similarly, use the best class if it isn't
1110 POINTER_AND_FP_REGS. Otherwise set the allocno class depending on the mode.
1111 The result of this is that it is no longer inefficient to have a higher
1112 memory move cost than the register move cost.
1113 */
1115 static reg_class_t
1117 reg_class_t best_class)
1118 {
1119 machine_mode mode;
1131 }
1133 static unsigned int
1135 {
1139 }
1141 /* Return the reassociation width of treeop OPC with mode MODE. */
1142 static int
1144 {
1149 /* Avoid reassociating floating point addition so we emit more FMAs. */
1153 }
1155 /* Provide a mapping from gcc register numbers to dwarf register numbers. */
1156 unsigned
1158 {
1170 /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
1171 equivalent DWARF register. */
1173 }
1175 /* Return true if MODE is any of the Advanced SIMD structure modes. */
1176 static bool
1178 {
1181 }
1183 /* Return true if MODE is an SVE predicate mode. */
1184 static bool
1186 {
1192 }
1194 /* Three mutually-exclusive flags describing a vector or predicate type. */
1198 /* Can be used in combination with VEC_ADVSIMD or VEC_SVE_DATA to indicate
1199 a structure of 2, 3 or 4 vectors. */
1201 /* Useful combinations of the above. */
1205 /* Return a set of flags describing the vector properties of mode MODE.
1206 Ignore modes that are not supported by the current target. */
1207 static unsigned int
1209 {
1225 {
1227 {
1234 }
1236 /* This includes V1DF but not V1DI (which doesn't exist). */
1241 }
1244 }
1246 /* Return true if MODE is any of the data vector modes, including
1247 structure modes. */
1248 static bool
1250 {
1252 }
1254 /* Return true if MODE is an SVE data vector mode; either a single vector
1255 or a structure of vectors. */
1256 static bool
1258 {
1260 }
1262 /* Implement target hook TARGET_ARRAY_MODE. */
1263 static opt_machine_mode
1265 {
1272 }
1274 /* Implement target hook TARGET_ARRAY_MODE_SUPPORTED_P. */
1275 static bool
1278 {
1286 }
1288 /* Return the SVE predicate mode to use for elements that have
1289 ELEM_NBYTES bytes, if such a mode exists. */
1291 opt_machine_mode
1293 {
1295 {
1304 }
1306 }
1308 /* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
1310 static opt_machine_mode
1312 {
1314 {
1316 machine_mode pred_mode;
1319 }
1322 }
1324 /* Implement TARGET_PREFERRED_ELSE_VALUE. For binary operations,
1325 prefer to use the first arithmetic operand as the else value if
1326 the else value doesn't matter, since that exactly matches the SVE
1327 destructive merging form. For ternary operations we could either
1328 pick the first operand and use FMAD-like instructions or the last
1329 operand and use FMLA-like instructions; the latter seems more
1330 natural. */
1332 static tree
1334 {
1336 }
1338 /* Implement TARGET_HARD_REGNO_NREGS. */
1340 static unsigned int
1342 {
1343 /* ??? Logically we should only need to provide a value when
1344 HARD_REGNO_MODE_OK says that the combination is valid,
1345 but at the moment we need to handle all modes. Just ignore
1346 any runtime parts for registers that can't store them. */
1349 {
1362 }
1364 }
1366 /* Implement TARGET_HARD_REGNO_MODE_OK. */
1368 static bool
1370 {
1375 /* This must have the same size as _Unwind_Word. */
1386 /* The purpose of comparing with ptr_mode is to support the
1387 global register variable associated with the stack pointer
1388 register via the syntax of asm ("wsp") in ILP32. */
1398 {
1401 else
1403 }
1406 }
1408 /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The callee only saves
1409 the lower 64 bits of a 128-bit register. Tell the compiler the callee
1410 clobbers the top 64 bits when restoring the bottom 64 bits. */
1412 static bool
1414 {
1416 }
1418 /* Implement REGMODE_NATURAL_SIZE. */
1419 poly_uint64
1421 {
1422 /* The natural size for SVE data modes is one SVE data vector,
1423 and similarly for predicates. We can't independently modify
1424 anything smaller than that. */
1425 /* ??? For now, only do this for variable-width SVE registers.
1426 Doing it for constant-sized registers breaks lower-subreg.c. */
1427 /* ??? And once that's fixed, we should probably have similar
1428 code for Advanced SIMD. */
1430 {
1436 }
1438 }
1440 /* Implement HARD_REGNO_CALLER_SAVE_MODE. */
1441 machine_mode
1443 machine_mode mode)
1444 {
1445 /* The predicate mode determines which bits are significant and
1446 which are "don't care". Decreasing the number of lanes would
1447 lose data while increasing the number of lanes would make bits
1448 unnecessarily significant. */
1453 else
1455 }
1457 /* Implement TARGET_CONSTANT_ALIGNMENT. Make strings word-aligned so
1458 that strcpy from constants will be faster. */
1460 static HOST_WIDE_INT
1462 {
1466 }
1468 /* Return true if calls to DECL should be treated as
1469 long-calls (ie called via a register). */
1470 static bool
1472 {
1474 }
1476 /* Return true if calls to symbol-ref SYM should be treated as
1477 long-calls (ie called via a register). */
1478 bool
1480 {
1482 }
1484 /* Return true if calls to symbol-ref SYM should not go through
1485 plt stubs. */
1487 bool
1489 {
1493 && decl
1494 && (!flag_plt
1500 }
1502 /* Return true if the offsets to a zero/sign-extract operation
1503 represent an expression that matches an extend operation. The
1504 operands represent the paramters from
1506 (extract:MODE (mult (reg) (MULT_IMM)) (EXTRACT_IMM) (const_int 0)). */
1507 bool
1509 rtx extract_imm)
1510 {
1527 }
1529 /* Emit an insn that's a simple single-set. Both the operands must be
1530 known to be valid. */
1533 {
1535 }
1537 /* X and Y are two things to compare using CODE. Emit the compare insn and
1538 return the rtx for register 0 in the proper mode. */
1539 rtx
1541 {
1547 }
1549 /* Build the SYMBOL_REF for __tls_get_addr. */
1553 rtx
1555 {
1559 }
1561 /* Return the TLS model to use for ADDR. */
1563 static enum tls_model
1565 {
1568 {
1569 poly_int64 addend;
1573 }
1578 }
1580 /* We'll allow lo_sum's in addresses in our legitimate addresses
1581 so that combine would take care of combining addresses where
1582 necessary, but for generation purposes, we'll generate the address
1583 as :
1584 RTL Absolute
1585 tmp = hi (symbol_ref); adrp x1, foo
1586 dest = lo_sum (tmp, symbol_ref); add dest, x1, :lo_12:foo
1587 nop
1589 PIC TLS
1590 adrp x1, :got:foo adrp tmp, :tlsgd:foo
1591 ldr x1, [:got_lo12:foo] add dest, tmp, :tlsgd_lo12:foo
1592 bl __tls_get_addr
1593 nop
1595 Load TLS symbol, depending on TLS mechanism and TLS access model.
1597 Global Dynamic - Traditional TLS:
1598 adrp tmp, :tlsgd:imm
1599 add dest, tmp, #:tlsgd_lo12:imm
1600 bl __tls_get_addr
1602 Global Dynamic - TLS Descriptors:
1603 adrp dest, :tlsdesc:imm
1604 ldr tmp, [dest, #:tlsdesc_lo12:imm]
1605 add dest, dest, #:tlsdesc_lo12:imm
1606 blr tmp
1607 mrs tp, tpidr_el0
1608 add dest, dest, tp
1610 Initial Exec:
1611 mrs tp, tpidr_el0
1612 adrp tmp, :gottprel:imm
1613 ldr dest, [tmp, #:gottprel_lo12:imm]
1614 add dest, dest, tp
1616 Local Exec:
1617 mrs tp, tpidr_el0
1618 add t0, tp, #:tprel_hi12:imm, lsl #12
1619 add t0, t0, #:tprel_lo12_nc:imm
1620 */
1622 static void
1625 {
1627 {
1629 {
1630 /* In ILP32, the mode of dest can be either SImode or DImode. */
1642 }
1649 {
1652 rtx insn;
1653 rtx mem;
1655 /* NOTE: pic_offset_table_rtx can be NULL_RTX, because we can reach
1656 here before rtl expand. Tree IVOPT will generate rtl pattern to
1657 decide rtx costs, in which case pic_offset_table_rtx is not
1658 initialized. For that case no need to generate the first adrp
1659 instruction as the final cost for global variable access is
1660 one instruction. */
1662 {
1663 /* -fpic for -mcmodel=small allow 32K GOT table size (but we are
1664 using the page base as GOT base, the first page may be wasted,
1665 in the worst scenario, there is only 28K space for GOT).
1667 The generate instruction sequence for accessing global variable
1668 is:
1670 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym]
1672 Only one instruction needed. But we must initialize
1673 pic_offset_table_rtx properly. We generate initialize insn for
1674 every global access, and allow CSE to remove all redundant.
1676 The final instruction sequences will look like the following
1677 for multiply global variables access.
1679 adrp pic_offset_table_rtx, _GLOBAL_OFFSET_TABLE_
1681 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym1]
1682 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym2]
1683 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym3]
1684 ... */
1693 }
1696 {
1699 else
1703 }
1704 else
1705 {
1710 }
1712 /* The operand is expected to be MEM. Whenever the related insn
1713 pattern changed, above code which calculate mem should be
1714 updated. */
1720 }
1723 {
1724 /* In ILP32, the mode of dest can be either SImode or DImode,
1725 while the got entry is always of SImode size. The mode of
1726 dest depends on how dest is used: if dest is assigned to a
1727 pointer (e.g. in the memory), it has SImode; it may have
1728 DImode if dest is dereferenced to access the memeory.
1729 This is why we have to handle three different ldr_got_small
1730 patterns here (two patterns for ILP32). */
1732 rtx insn;
1733 rtx mem;
1742 {
1745 else
1749 }
1750 else
1751 {
1756 }
1763 }
1766 {
1774 else
1782 }
1785 {
1788 rtx tp;
1792 /* In ILP32, the got entry is always of SImode size. Unlike
1793 small GOT, the dest is fixed at reg 0. */
1796 else
1807 }
1810 {
1811 /* In ILP32, the mode of dest can be either SImode or DImode,
1812 while the got entry is always of SImode size. The mode of
1813 dest depends on how dest is used: if dest is assigned to a
1814 pointer (e.g. in the memory), it has SImode; it may have
1815 DImode if dest is dereferenced to access the memeory.
1816 This is why we have to handle three different tlsie_small
1817 patterns here (two patterns for ILP32). */
1823 {
1826 else
1827 {
1830 }
1831 }
1832 else
1833 {
1836 }
1842 }
1848 {
1856 {
1879 }
1884 }
1891 {
1896 {
1899 else
1900 {
1903 }
1904 }
1905 else
1906 {
1909 }
1914 }
1918 }
1919 }
1921 /* Emit a move from SRC to DEST. Assume that the move expanders can
1922 handle all moves if !can_create_pseudo_p (). The distinction is
1923 important because, unlike emit_move_insn, the move expanders know
1924 how to force Pmode objects into the constant pool even when the
1925 constant pool address is not itself legitimate. */
1926 static rtx
1928 {
1932 }
1934 /* Apply UNOPTAB to OP and store the result in DEST. */
1936 static void
1938 {
1942 }
1944 /* Apply BINOPTAB to OP0 and OP1 and store the result in DEST. */
1946 static void
1948 {
1950 OPTAB_DIRECT);
1953 }
1955 /* Split a 128-bit move operation into two 64-bit move operations,
1956 taking care to handle partial overlap of register to register
1957 copies. Special cases are needed when moving between GP regs and
1958 FP regs. SRC can be a register, constant or memory; DST a register
1959 or memory. If either operand is memory it must not have any side
1960 effects. */
1961 void
1963 {
1974 {
1978 /* Handle FP <-> GP regs. */
1980 {
1987 }
1989 {
1996 }
1997 }
2004 /* At most one pairing may overlap. */
2006 {
2009 }
2010 else
2011 {
2014 }
2015 }
2017 bool
2019 {
2022 }
2024 /* Split a complex SIMD combine. */
2026 void
2028 {
2039 }
2041 /* Split a complex SIMD move. */
2043 void
2045 {
2052 {
2055 }
2056 }
2058 bool
2061 {
2065 }
2068 static rtx
2070 {
2073 else
2074 {
2078 }
2079 }
2081 /* Return true if we can move VALUE into a register using a single
2082 CNT[BHWD] instruction. */
2084 static bool
2086 {
2088 /* The coefficient must be [1, 16] * {2, 4, 8, 16}. */
2093 }
2095 /* Likewise for rtx X. */
2097 bool
2099 {
2100 poly_int64 value;
2102 }
2104 /* Return the asm string for an instruction with a CNT-like vector size
2105 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2106 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2107 first part of the operands template (the part that comes before the
2108 vector size itself). FACTOR is the number of quadwords.
2109 NELTS_PER_VQ, if nonzero, is the number of elements in each quadword.
2110 If it is zero, we can use any element size. */
2116 {
2120 /* There is some overlap in the ranges of the four CNT instructions.
2121 Here we always use the smallest possible element size, so that the
2122 multiplier is 1 whereever possible. */
2133 else
2138 }
2140 /* Return the asm string for an instruction with a CNT-like vector size
2141 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2142 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2143 first part of the operands template (the part that comes before the
2144 vector size itself). X is the value of the vector size operand,
2145 as a polynomial integer rtx. */
2149 rtx x)
2150 {
2155 }
2157 /* Return true if we can add VALUE to a register using a single ADDVL
2158 or ADDPL instruction. */
2160 static bool
2162 {
2166 /* FACTOR counts VG / 2, so a value of 2 is one predicate width
2167 and a value of 16 is one vector width. */
2170 }
2172 /* Likewise for rtx X. */
2174 bool
2176 {
2177 poly_int64 value;
2180 }
2182 /* Return the asm string for adding ADDVL or ADDPL immediate X to operand 1
2183 and storing the result in operand 0. */
2187 {
2192 /* Use INC or DEC if possible. */
2194 {
2201 }
2206 else
2209 }
2211 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2212 instruction. If it is, store the number of elements in each vector
2213 quadword in *NELTS_PER_VQ_OUT (if nonnull) and store the multiplication
2214 factor in *FACTOR_OUT (if nonnull). */
2216 bool
2219 {
2220 rtx elt;
2221 poly_int64 value;
2229 /* There's no vector INCB. */
2236 /* The coefficient must be [1, 16] * NELTS_PER_VQ. */
2246 }
2248 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2249 instruction. */
2251 bool
2253 {
2255 }
2257 /* Return the asm template for an SVE vector INC or DEC instruction.
2258 OPERANDS gives the operands before the vector count and X is the
2259 value of the vector count operand itself. */
2263 {
2270 nelts_per_vq);
2271 else
2273 nelts_per_vq);
2274 }
2276 static int
2278 scalar_int_mode mode)
2279 {
2288 {
2292 }
2294 /* Check to see if the low 32 bits are either 0xffffXXXX or 0xXXXXffff
2295 (with XXXX non-zero). In that case check to see if the move can be done in
2296 a smaller mode. */
2301 {
2305 /* Check if we have to emit a second instruction by checking to see
2306 if any of the upper 32 bits of the original DI mode value is set. */
2317 }
2320 {
2322 {
2327 else
2330 }
2332 }
2334 /* Remaining cases are all for DImode. */
2343 {
2344 /* Try emitting a bitmask immediate with a movk replacing 16 bits.
2345 For a 64-bit bitmask try whether changing 16 bits to all ones or
2346 zeroes creates a valid bitmask. To check any repeated bitmask,
2347 try using 16 bits from the other 32-bit half of val. */
2350 {
2361 }
2363 {
2365 {
2369 }
2371 }
2372 }
2374 /* Generate 2-4 instructions, skipping 16 bits of all zeroes or ones which
2375 are emitted by the initial mov. If one_match > zero_match, skip set bits,
2376 otherwise skip zero bits. */
2388 {
2394 num_insns ++;
2395 }
2398 }
2400 /* Return whether imm is a 128-bit immediate which is simple enough to
2401 expand inline. */
2402 bool
2404 {
2415 }
2418 /* Return the number of temporary registers that aarch64_add_offset_1
2419 would need to add OFFSET to a register. */
2421 static unsigned int
2423 {
2425 }
2427 /* A subroutine of aarch64_add_offset. Set DEST to SRC + OFFSET for
2428 a non-polynomial OFFSET. MODE is the mode of the addition.
2429 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
2430 be set and CFA adjustments added to the generated instructions.
2432 TEMP1, if nonnull, is a register of mode MODE that can be used as a
2433 temporary if register allocation is already complete. This temporary
2434 register may overlap DEST but must not overlap SRC. If TEMP1 is known
2435 to hold abs (OFFSET), EMIT_MOVE_IMM can be set to false to avoid emitting
2436 the immediate again.
2438 Since this function may be used to adjust the stack pointer, we must
2439 ensure that it cannot cause transient stack deallocation (for example
2440 by first incrementing SP and then decrementing when adjusting by a
2441 large immediate). */
2443 static void
2447 {
2455 {
2457 {
2460 }
2462 }
2464 /* Single instruction adjustment. */
2466 {
2470 }
2472 /* Emit 2 additions/subtractions if the adjustment is less than 24 bits
2473 and either:
2475 a) the offset cannot be loaded by a 16-bit move or
2476 b) there is no spare register into which we can move it. */
2480 {
2489 }
2491 /* Emit a move immediate if required and an addition/subtraction. */
2493 {
2496 }
2501 {
2505 }
2506 }
2508 /* Return the number of temporary registers that aarch64_add_offset
2509 would need to move OFFSET into a register or add OFFSET to a register;
2510 ADD_P is true if we want the latter rather than the former. */
2512 static unsigned int
2514 {
2515 /* This follows the same structure as aarch64_add_offset. */
2524 /* Need one register for the ADDVL/ADDPL result. */
2527 {
2530 /* Need one register for the CNT result and one for the multiplication
2531 factor. If necessary, the second temporary can be reused for the
2532 constant part of the offset. */
2534 /* Need one register for the CNT result (which might then
2535 be shifted). */
2537 }
2539 }
2541 /* If X can be represented as a poly_int64, return the number
2542 of temporaries that are required to add it to a register.
2543 Return -1 otherwise. */
2545 int
2547 {
2548 poly_int64 offset;
2552 }
2554 /* Set DEST to SRC + OFFSET. MODE is the mode of the addition.
2555 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
2556 be set and CFA adjustments added to the generated instructions.
2558 TEMP1, if nonnull, is a register of mode MODE that can be used as a
2559 temporary if register allocation is already complete. This temporary
2560 register may overlap DEST if !FRAME_RELATED_P but must not overlap SRC.
2561 If TEMP1 is known to hold abs (OFFSET), EMIT_MOVE_IMM can be set to
2562 false to avoid emitting the immediate again.
2564 TEMP2, if nonnull, is a second temporary register that doesn't
2565 overlap either DEST or REG.
2567 Since this function may be used to adjust the stack pointer, we must
2568 ensure that it cannot cause transient stack deallocation (for example
2569 by first incrementing SP and then decrementing when adjusting by a
2570 large immediate). */
2572 static void
2576 {
2580 || !frame_related_p
2584 /* Try using ADDVL or ADDPL to add the whole value. */
2586 {
2591 }
2593 /* Coefficient 1 is multiplied by the number of 128-bit blocks in an
2594 SVE vector register, over and above the minimum size of 128 bits.
2595 This is equivalent to half the value returned by CNTD with a
2596 vector shape of ALL. */
2600 /* Try using ADDVL or ADDPL to add the VG-based part. */
2604 {
2607 {
2611 }
2612 else
2613 {
2618 }
2619 }
2620 /* Otherwise use a CNT-based sequence. */
2622 {
2623 /* Use a subtraction if we have a negative factor. */
2626 {
2629 }
2631 /* Calculate CNTD * FACTOR / 2. First try to fold the division
2632 into the multiplication. */
2633 rtx val;
2636 /* Use a right shift by 1. */
2638 else
2642 {
2644 {
2645 /* "CNTB Xn, ALL, MUL #FACTOR" is out of range, so calculate
2646 the value with the minimum multiplier and shift it into
2647 position. */
2651 }
2653 }
2654 else
2655 {
2656 /* Use CNTD, then multiply it by FACTOR. */
2660 /* Go back to using a negative multiplication factor if we have
2661 no register from which to subtract. */
2663 {
2666 }
2670 }
2673 {
2674 /* Multiply by 1 << SHIFT. */
2677 }
2679 {
2680 /* Divide by 2. */
2683 }
2685 /* Calculate SRC +/- CNTD * FACTOR / 2. */
2687 {
2690 }
2692 {
2695 }
2698 {
2701 {
2705 poly_offset)));
2706 }
2710 }
2711 else
2712 {
2716 }
2719 }
2723 }
2725 /* Like aarch64_add_offset, but the offset is given as an rtx rather
2726 than a poly_int64. */
2728 void
2731 {
2734 }
2736 /* Add DELTA to the stack pointer, marking the instructions frame-related.
2737 TEMP1 is available as a temporary if nonnull. EMIT_MOVE_IMM is false
2738 if TEMP1 already contains abs (DELTA). */
2742 {
2745 }
2747 /* Subtract DELTA from the stack pointer, marking the instructions
2748 frame-related if FRAME_RELATED_P. TEMP1 is available as a temporary
2749 if nonnull. */
2753 {
2756 }
2758 /* Set DEST to (vec_series BASE STEP). */
2760 static void
2762 {
2766 /* Each operand can be a register or an immediate in the range [-16, 15]. */
2773 }
2775 /* Try to duplicate SRC into SVE register DEST, given that SRC is an
2776 integer of mode INT_MODE. Return true on success. */
2778 static bool
2780 rtx src)
2781 {
2782 /* If the constant is smaller than 128 bits, we can do the move
2783 using a vector of SRC_MODEs. */
2785 {
2792 }
2794 /* Use LD1RQ[BHWD] to load the 128 bits from memory. */
2799 /* Make sure that the address is legitimate. */
2801 {
2804 }
2813 }
2815 /* Expand a move of general CONST_VECTOR SRC into DEST, given that it
2816 isn't a simple duplicate or series. */
2818 static void
2820 {
2827 {
2828 /* The constant is a repeating seqeuence of at least two elements,
2829 where the repeating elements occupy no more than 128 bits.
2830 Get an integer representation of the replicated value. */
2831 scalar_int_mode int_mode;
2833 /* For now, always use LD1RQ to load the value on big-endian
2834 targets, since the handling of smaller integers includes a
2835 subreg that is semantically an element reverse. */
2837 else
2838 {
2842 }
2847 }
2849 /* Expand each pattern individually. */
2850 rtx_vector_builder builder;
2853 {
2858 }
2860 /* Use permutes to interleave the separate vectors. */
2862 {
2865 {
2870 }
2871 }
2873 }
2875 /* Set DEST to immediate IMM. For SVE vector modes, GEN_VEC_DUPLICATE
2876 is a pattern that can be used to set DEST to a replicated scalar
2877 element. */
2879 void
2882 {
2885 /* Check on what type of symbol it is. */
2886 scalar_int_mode int_mode;
2892 {
2893 rtx mem;
2894 poly_int64 offset;
2895 HOST_WIDE_INT const_offset;
2898 /* If we have (const (plus symbol offset)), separate out the offset
2899 before we start classifying the symbol. */
2902 /* We must always add an offset involving VL separately, rather than
2903 folding it into the relocation. */
2905 {
2908 else
2909 {
2910 /* Do arithmetic on 32-bit values if the result is smaller
2911 than that. */
2913 {
2914 /* It is invalid to do symbol calculations in modes
2915 narrower than SImode. */
2919 }
2921 {
2925 }
2926 else
2929 }
2931 }
2935 {
2939 {
2945 }
2950 /* If we aren't generating PC relative literals, then
2951 we need to expand the literal pool access carefully.
2952 This is something that needs to be done in a number
2953 of places, so could well live as a separate function. */
2955 {
2962 }
2979 {
2985 }
2986 /* FALLTHRU */
2999 }
3000 }
3003 {
3011 {
3012 /* If the constant is out of range of an SVE vector move,
3013 load it from memory if we can, otherwise move it into
3014 a register and use a DUP. */
3020 {
3023 }
3025 }
3029 else
3030 {
3034 }
3037 }
3041 }
3043 /* Emit an SVE predicated move from SRC to DEST. PRED is a predicate
3044 that is known to contain PTRUE. */
3046 void
3048 {
3051 UNSPEC_MERGE_PTRUE)));
3052 }
3054 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one
3055 operand is in memory. In this case we need to use the predicated LD1
3056 and ST1 instead of LDR and STR, both for correctness on big-endian
3057 targets and because LD1 and ST1 support a wider range of addressing modes.
3058 PRED_MODE is the mode of the predicate.
3060 See the comment at the head of aarch64-sve.md for details about the
3061 big-endian handling. */
3063 void
3065 {
3070 {
3074 else
3077 }
3079 }
3081 /* Called only on big-endian targets. See whether an SVE vector move
3082 from SRC to DEST is effectively a REV[BHW] instruction, because at
3083 least one operand is a subreg of an SVE vector that has wider or
3084 narrower elements. Return true and emit the instruction if so.
3086 For example:
3088 (set (reg:VNx8HI R1) (subreg:VNx8HI (reg:VNx16QI R2) 0))
3090 represents a VIEW_CONVERT between the following vectors, viewed
3091 in memory order:
3093 R2: { [0].high, [0].low, [1].high, [1].low, ... }
3094 R1: { [0], [1], [2], [3], ... }
3096 The high part of lane X in R2 should therefore correspond to lane X*2
3097 of R1, but the register representations are:
3099 msb lsb
3100 R2: ...... [1].high [1].low [0].high [0].low
3101 R1: ...... [3] [2] [1] [0]
3103 where the low part of lane X in R2 corresponds to lane X*2 in R1.
3104 We therefore need a reverse operation to swap the high and low values
3105 around.
3107 This is purely an optimization. Without it we would spill the
3108 subreg operand to the stack in one mode and reload it in the
3109 other mode, which has the same effect as the REV. */
3111 bool
3113 {
3120 /* The optimization handles two single SVE REGs with different element
3121 sizes. */
3130 /* Generate *aarch64_sve_mov<mode>_subreg_be. */
3133 UNSPEC_REV_SUBREG);
3136 }
3138 /* Return a copy of X with mode MODE, without changing its other
3139 attributes. Unlike gen_lowpart, this doesn't care whether the
3140 mode change is valid. */
3142 static rtx
3144 {
3151 }
3153 /* Split a *aarch64_sve_mov<mode>_subreg_be pattern with the given
3154 operands. */
3156 void
3158 {
3159 /* Decide which REV operation we need. The mode with narrower elements
3160 determines the mode of the operands and the mode with the wider
3161 elements determines the reverse width. */
3176 else
3180 /* Emit:
3182 (set DEST (unspec [PTRUE (unspec [SRC] UNSPEC_REV<nn>)]
3183 UNSPEC_MERGE_PTRUE))
3185 with the appropriate modes. */
3191 UNSPEC_MERGE_PTRUE);
3193 }
3195 static bool
3197 tree exp ATTRIBUTE_UNUSED)
3198 {
3199 /* Currently, always true. */
3201 }
3203 /* Implement TARGET_PASS_BY_REFERENCE. */
3205 static bool
3207 machine_mode mode,
3208 const_tree type,
3210 {
3211 HOST_WIDE_INT size;
3212 machine_mode dummymode;
3215 /* GET_MODE_SIZE (BLKmode) is useless since it is 0. */
3218 else
3219 /* No frontends can create types with variable-sized modes, so we
3220 shouldn't be asked to pass or return them. */
3223 /* Aggregates are passed by reference based on their size. */
3225 {
3227 }
3229 /* Variable sized arguments are always returned by reference. */
3233 /* Can this be a candidate to be passed in fp/simd register(s)? */
3236 NULL))
3239 /* Arguments which are variable sized or larger than 2 registers are
3240 passed by reference unless they are a homogenous floating point
3241 aggregate. */
3243 }
3245 /* Return TRUE if VALTYPE is padded to its least significant bits. */
3246 static bool
3248 {
3249 machine_mode dummy_mode;
3252 /* Never happens in little-endian mode. */
3256 /* Only composite types smaller than or equal to 16 bytes can
3257 be potentially returned in registers. */
3263 /* But not a composite that is an HFA (Homogeneous Floating-point Aggregate)
3264 or an HVA (Homogeneous Short-Vector Aggregate); such a special composite
3265 is always passed/returned in the least significant bits of fp/simd
3266 register(s). */
3272 }
3274 /* Implement TARGET_FUNCTION_VALUE.
3275 Define how to find the value returned by a function. */
3277 static rtx
3280 {
3281 machine_mode mode;
3284 machine_mode ag_mode;
3291 {
3295 {
3298 }
3299 }
3303 {
3305 {
3308 }
3309 else
3310 {
3312 rtx par;
3316 {
3321 }
3323 }
3324 }
3325 else
3327 }
3329 /* Implements TARGET_FUNCTION_VALUE_REGNO_P.
3330 Return true if REGNO is the number of a hard register in which the values
3331 of called function may come back. */
3333 static bool
3335 {
3336 /* Maximum of 16 bytes can be returned in the general registers. Examples
3337 of 16-byte return values are: 128-bit integers and 16-byte small
3338 structures (excluding homogeneous floating-point aggregates). */
3342 /* Up to four fp/simd registers can return a function value, e.g. a
3343 homogeneous floating-point aggregate having four members. */
3348 }
3350 /* Implement TARGET_RETURN_IN_MEMORY.
3352 If the type T of the result of a function is such that
3353 void func (T arg)
3354 would require that arg be passed as a value in a register (or set of
3355 registers) according to the parameter passing rules, then the result
3356 is returned in the same registers as would be used for such an
3357 argument. */
3359 static bool
3361 {
3362 HOST_WIDE_INT size;
3363 machine_mode ag_mode;
3369 /* Simple scalar types always returned in registers. */
3373 type,
3376 NULL))
3379 /* Types larger than 2 registers returned in memory. */
3382 }
3384 static bool
3387 {
3390 type,
3392 nregs,
3393 NULL);
3394 }
3396 /* Given MODE and TYPE of a function argument, return the alignment in
3397 bits. The idea is to suppress any stronger alignment requested by
3398 the user and opt for the natural alignment (specified in AAPCS64 \S 4.1).
3399 This is a helper function for local use only. */
3401 static unsigned int
3403 {
3424 }
3426 /* Layout a function argument according to the AAPCS64 rules. The rule
3427 numbers refer to the rule numbers in the AAPCS64. */
3429 static void
3431 const_tree type,
3433 {
3437 HOST_WIDE_INT size;
3439 /* We need to do this once per argument. */
3445 /* Size in bytes, rounded to the nearest multiple of 8 bytes. */
3448 else
3449 /* No frontends can create types with variable-sized modes, so we
3450 shouldn't be asked to pass or return them. */
3456 mode,
3457 type,
3460 /* allocate_ncrn may be false-positive, but allocate_nvrn is quite reliable.
3461 The following code thus handles passing by SIMD/FP registers first. */
3465 /* C1 - C5 for floating point, homogenous floating point aggregates (HFA)
3466 and homogenous short-vector aggregates (HVA). */
3468 {
3473 {
3476 {
3479 }
3480 else
3481 {
3482 rtx par;
3486 {
3489 rtx offset = gen_int_mode
3493 }
3495 }
3497 }
3498 else
3499 {
3500 /* C.3 NSRN is set to 8. */
3503 }
3504 }
3509 /* C6 - C9. though the sign and zero extension semantics are
3510 handled elsewhere. This is the case where the argument fits
3511 entirely general registers. */
3513 {
3517 /* C.8 if the argument has an alignment of 16 then the NGRN is
3518 rounded up to the next even number. */
3521 /* The == 16 * BITS_PER_UNIT instead of >= 16 * BITS_PER_UNIT
3522 comparison is there because for > 16 * BITS_PER_UNIT
3523 alignment nregs should be > 2 and therefore it should be
3524 passed by reference rather than value. */
3526 {
3529 }
3531 /* NREGS can be 0 when e.g. an empty structure is to be passed.
3532 A reg is still generated for it, but the caller should be smart
3533 enough not to use it. */
3536 else
3537 {
3538 rtx par;
3543 {
3548 }
3550 }
3554 }
3556 /* C.11 */
3559 /* The argument is passed on stack; record the needed number of words for
3560 this argument and align the total size if necessary. */
3561 on_stack:
3568 }
3570 /* Implement TARGET_FUNCTION_ARG. */
3572 static rtx
3575 {
3584 }
3586 void
3588 const_tree fntype ATTRIBUTE_UNUSED,
3589 rtx libname ATTRIBUTE_UNUSED,
3590 const_tree fndecl ATTRIBUTE_UNUSED,
3592 {
3606 {
3613 }
3615 }
3617 static void
3619 machine_mode mode,
3620 const_tree type,
3622 {
3625 {
3635 }
3636 }
3638 bool
3640 {
3643 }
3645 /* Implement FUNCTION_ARG_BOUNDARY. Every parameter gets at least
3646 PARM_BOUNDARY bits of alignment, but will be given anything up
3647 to STACK_BOUNDARY bits if the type requires it. This makes sure
3648 that both before and after the layout of each argument, the Next
3649 Stacked Argument Address (NSAA) will have a minimum alignment of
3650 8 bytes. */
3652 static unsigned int
3654 {
3657 }
3659 /* Implement TARGET_GET_RAW_RESULT_MODE and TARGET_GET_RAW_ARG_MODE. */
3661 static fixed_size_mode
3663 {
3665 /* Don't use the SVE part of the register for __builtin_apply and
3666 __builtin_return. The SVE registers aren't used by the normal PCS,
3667 so using them there would be a waste of time. The PCS extensions
3668 for SVE types are fundamentally incompatible with the
3669 __builtin_return/__builtin_apply interface. */
3672 }
3674 /* Implement TARGET_FUNCTION_ARG_PADDING.
3676 Small aggregate types are placed in the lowest memory address.
3678 The related parameter passing rules are B.4, C.3, C.5 and C.14. */
3680 static pad_direction
3682 {
3683 /* On little-endian targets, the least significant byte of every stack
3684 argument is passed at the lowest byte address of the stack slot. */
3688 /* Otherwise, integral, floating-point and pointer types are padded downward:
3689 the least significant byte of a stack argument is passed at the highest
3690 byte address of the stack slot. */
3697 /* Everything else padded upward, i.e. data in first byte of stack slot. */
3699 }
3701 /* Similarly, for use by BLOCK_REG_PADDING (MODE, TYPE, FIRST).
3703 It specifies padding for the last (may also be the only)
3704 element of a block move between registers and memory. If
3705 assuming the block is in the memory, padding upward means that
3706 the last element is padded after its highest significant byte,
3707 while in downward padding, the last element is padded at the
3708 its least significant byte side.
3710 Small aggregates and small complex types are always padded
3711 upwards.
3713 We don't need to worry about homogeneous floating-point or
3714 short-vector aggregates; their move is not affected by the
3715 padding direction determined here. Regardless of endianness,
3716 each element of such an aggregate is put in the least
3717 significant bits of a fp/simd register.
3719 Return !BYTES_BIG_ENDIAN if the least significant byte of the
3720 register has useful data, and return the opposite if the most
3721 significant byte does. */
3723 bool
3726 {
3728 /* Small composite types are always padded upward. */
3730 {
3731 HOST_WIDE_INT size;
3734 else
3735 /* No frontends can create types with variable-sized modes, so we
3736 shouldn't be asked to pass or return them. */
3740 }
3742 /* Otherwise, use the default padding. */
3744 }
3746 static scalar_int_mode
3748 {
3750 }
3752 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
3754 /* We use the 12-bit shifted immediate arithmetic instructions so values
3755 must be multiple of (1 << 12), i.e. 4096. */
3756 #define ARITH_FACTOR 4096
3758 #if (PROBE_INTERVAL % ARITH_FACTOR) != 0
3759 #error Cannot use simple address calculation for stack probing
3760 #endif
3762 /* The pair of scratch registers used for stack probing. */
3763 #define PROBE_STACK_FIRST_REG 9
3764 #define PROBE_STACK_SECOND_REG 10
3766 /* Emit code to probe a range of stack addresses from FIRST to FIRST+POLY_SIZE,
3767 inclusive. These are offsets from the current stack pointer. */
3769 static void
3771 {
3772 HOST_WIDE_INT size;
3774 {
3777 }
3781 /* See the same assertion on PROBE_INTERVAL above. */
3784 /* See if we have a constant small number of probes to generate. If so,
3785 that's the easy case. */
3787 {
3794 }
3796 /* The run-time loop is made up of 8 insns in the generic case while the
3797 compile-time loop is made up of 4+2*(n-2) insns for n # of intervals. */
3799 {
3804 stack_pointer_rtx,
3808 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
3809 it exceeds SIZE. If only two probes are needed, this will not
3810 generate any code. Then probe at FIRST + SIZE. */
3812 {
3816 }
3820 {
3825 }
3826 else
3828 }
3830 /* Otherwise, do the same as above, but in a loop. Note that we must be
3831 extra careful with variables wrapping around because we might be at
3832 the very top (or the very bottom) of the address space and we have
3833 to be able to handle this case properly; in particular, we use an
3834 equality test for the loop condition. */
3835 else
3836 {
3839 /* Step 1: round SIZE to the previous multiple of the interval. */
3844 /* Step 2: compute initial and final value of the loop counter. */
3846 /* TEST_ADDR = SP + FIRST. */
3850 /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE. */
3853 {
3857 }
3858 else
3862 /* Step 3: the loop
3864 do
3865 {
3866 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
3867 probe at TEST_ADDR
3868 }
3869 while (TEST_ADDR != LAST_ADDR)
3871 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
3872 until it is equal to ROUNDED_SIZE. */
3877 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
3878 that SIZE is equal to ROUNDED_SIZE. */
3881 {
3885 {
3890 }
3891 else
3893 }
3894 }
3896 /* Make sure nothing is scheduled before we are done. */
3898 }
3900 /* Probe a range of stack addresses from REG1 to REG2 inclusive. These are
3901 absolute addresses. */
3905 {
3912 /* Loop. */
3915 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
3920 /* Probe at TEST_ADDR. */
3923 /* Test if TEST_ADDR == LAST_ADDR. */
3927 /* Branch. */
3933 }
3935 /* Determine whether a frame chain needs to be generated. */
3936 static bool
3938 {
3939 /* Force a frame chain for EH returns so the return address is at FP+8. */
3943 /* A leaf function cannot have calls or write LR. */
3946 /* Don't use a frame chain in leaf functions if leaf frame pointers
3947 are disabled. */
3952 }
3954 /* Mark the registers that need to be saved by the callee and calculate
3955 the size of the callee-saved registers area and frame record (both FP
3956 and LR may be omitted). */
3957 static void
3959 {
3968 #define SLOT_NOT_REQUIRED (-2)
3969 #define SLOT_REQUIRED (-1)
3974 /* First mark all the registers that really need to be saved... */
3981 /* ... that includes the eh data registers (if needed)... */
3987 /* ... and any callee saved register that dataflow says is live. */
3997 {
4000 }
4003 {
4004 /* FP and LR are placed in the linkage record. */
4010 }
4012 /* Now assign stack slots for them. */
4015 {
4022 }
4030 {
4031 /* If there is an alignment gap between integer and fp callee-saves,
4032 allocate the last fp register to it if possible. */
4034 {
4037 }
4046 }
4052 HOST_WIDE_INT varargs_and_saved_regs_size
4060 /* Both these values are already aligned. */
4084 {
4085 /* Simple, small frame with no outgoing arguments:
4086 stp reg1, reg2, [sp, -frame_size]!
4087 stp reg3, reg4, [sp, 16] */
4089 }
4094 max_push_offset)))
4095 {
4096 /* Frame with small outgoing arguments:
4097 sub sp, sp, frame_size
4098 stp reg1, reg2, [sp, outgoing_args_size]
4099 stp reg3, reg4, [sp, outgoing_args_size + 16] */
4103 }
4106 {
4107 /* Frame with large outgoing arguments but a small local area:
4108 stp reg1, reg2, [sp, -hard_fp_offset]!
4109 stp reg3, reg4, [sp, 16]
4110 sub sp, sp, outgoing_args_size */
4114 }
4115 else
4116 {
4117 /* Frame with large local area and outgoing arguments using frame pointer:
4118 sub sp, sp, hard_fp_offset
4119 stp x29, x30, [sp, 0]
4120 add x29, sp, 0
4121 stp reg3, reg4, [sp, 16]
4122 sub sp, sp, outgoing_args_size */
4126 }
4129 }
4131 /* Return true if the register REGNO is saved on entry to
4132 the current function. */
4134 static bool
4136 {
4138 }
4140 /* Return the next register up from REGNO up to LIMIT for the callee
4141 to save. */
4143 static unsigned
4145 {
4147 regno ++;
4149 }
4151 /* Push the register number REGNO of mode MODE to the stack with write-back
4152 adjusting the stack by ADJUSTMENT. */
4154 static void
4156 HOST_WIDE_INT adjustment)
4157 {
4168 }
4170 /* Generate and return an instruction to store the pair of registers
4171 REG and REG2 of mode MODE to location BASE with write-back adjusting
4172 the stack location BASE by ADJUSTMENT. */
4174 static rtx
4176 HOST_WIDE_INT adjustment)
4177 {
4179 {
4190 }
4191 }
4193 /* Push registers numbered REGNO1 and REGNO2 to the stack, adjusting the
4194 stack pointer by ADJUSTMENT. */
4196 static void
4198 {
4213 }
4215 /* Load the pair of register REG, REG2 of mode MODE from stack location BASE,
4216 adjusting it by ADJUSTMENT afterwards. */
4218 static rtx
4220 HOST_WIDE_INT adjustment)
4221 {
4223 {
4232 }
4233 }
4235 /* Pop the two registers numbered REGNO1, REGNO2 from the stack, adjusting it
4236 afterwards by ADJUSTMENT and writing the appropriate REG_CFA_RESTORE notes
4237 into CFI_OPS. */
4239 static void
4242 {
4249 {
4253 }
4254 else
4255 {
4260 }
4261 }
4263 /* Generate and return a store pair instruction of mode MODE to store
4264 register REG1 to MEM1 and register REG2 to MEM2. */
4266 static rtx
4268 rtx reg2)
4269 {
4271 {
4280 }
4281 }
4283 /* Generate and regurn a load pair isntruction of mode MODE to load register
4284 REG1 from MEM1 and register REG2 from MEM2. */
4286 static rtx
4288 rtx mem2)
4289 {
4291 {
4300 }
4301 }
4303 /* Return TRUE if return address signing should be enabled for the current
4304 function, otherwise return FALSE. */
4306 bool
4308 {
4309 /* This function should only be called after frame laid out. */
4312 /* If signing scope is AARCH64_FUNCTION_NON_LEAF, we only sign a leaf function
4313 if it's LR is pushed onto stack. */
4317 }
4319 /* Emit code to save the callee-saved registers from register number START
4320 to LIMIT to the stack at the location starting at offset START_OFFSET,
4321 skipping any write-back candidates if SKIP_WB is true. */
4323 static void
4326 {
4334 {
4336 poly_int64 offset;
4349 offset));
4358 {
4360 rtx mem2;
4364 offset));
4366 reg2));
4368 /* The first part of a frame-related parallel insn is
4369 always assumed to be relevant to the frame
4370 calculations; subsequent parts, are only
4371 frame-related if explicitly marked. */
4374 }
4375 else
4379 }
4380 }
4382 /* Emit code to restore the callee registers of mode MODE from register
4383 number START up to and including LIMIT. Restore from the stack offset
4384 START_OFFSET, skipping any write-back candidates if SKIP_WB is true.
4385 Write the appropriate REG_CFA_RESTORE notes into CFI_OPS. */
4387 static void
4391 {
4395 poly_int64 offset;
4400 {
4421 {
4423 rtx mem2;
4431 }
4432 else
4435 }
4436 }
4438 /* Return true if OFFSET is a signed 4-bit value multiplied by the size
4439 of MODE. */
4443 {
4444 HOST_WIDE_INT multiple;
4447 }
4449 /* Return true if OFFSET is a unsigned 6-bit value multiplied by the size
4450 of MODE. */
4454 {
4455 HOST_WIDE_INT multiple;
4458 }
4460 /* Return true if OFFSET is a signed 7-bit value multiplied by the size
4461 of MODE. */
4463 bool
4465 {
4466 HOST_WIDE_INT multiple;
4469 }
4471 /* Return true if OFFSET is a signed 9-bit value. */
4475 poly_int64 offset)
4476 {
4477 HOST_WIDE_INT const_offset;
4480 }
4482 /* Return true if OFFSET is a signed 9-bit value multiplied by the size
4483 of MODE. */
4487 {
4488 HOST_WIDE_INT multiple;
4491 }
4493 /* Return true if OFFSET is an unsigned 12-bit value multiplied by the size
4494 of MODE. */
4498 {
4499 HOST_WIDE_INT multiple;
4502 }
4504 /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
4506 static sbitmap
4508 {
4514 /* The registers we need saved to the frame. */
4517 {
4522 /* Check that we can access the stack slot of the register with one
4523 direct load with no adjustments needed. */
4526 }
4528 /* Don't mess with the hard frame pointer. */
4534 /* If aarch64_layout_frame has chosen registers to store/restore with
4535 writeback don't interfere with them to avoid having to output explicit
4536 stack adjustment instructions. */
4546 }
4548 /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
4550 static sbitmap
4552 {
4560 /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
4566 {
4570 /* If there is a callee-save at an adjacent offset, add it too
4571 to increase the use of LDP/STP. */
4576 {
4580 }
4581 }
4584 }
4586 /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS.
4587 Nothing to do for aarch64. */
4589 static void
4591 {
4592 }
4594 /* Return the next set bit in BMP from START onwards. Return the total number
4595 of bits in BMP if no set bit is found at or after START. */
4597 static unsigned int
4599 {
4610 }
4612 /* Do the work for aarch64_emit_prologue_components and
4613 aarch64_emit_epilogue_components. COMPONENTS is the bitmap of registers
4614 to save/restore, PROLOGUE_P indicates whether to emit the prologue sequence
4615 for these components or the epilogue sequence. That is, it determines
4616 whether we should emit stores or loads and what kind of CFA notes to attach
4617 to the insns. Otherwise the logic for the two sequences is very
4618 similar. */
4620 static void
4622 {
4624 ? HARD_FRAME_POINTER_REGNUM
4632 {
4633 /* AAPCS64 section 5.1.2 requires only the bottom 64 bits to be saved
4634 so DFmode for the vector registers is enough. */
4646 /* No more registers to handle after REGNO.
4647 Emit a single save/restore and exit. */
4649 {
4654 else
4657 }
4660 /* The next register is not of the same class or its offset is not
4661 mergeable with the current one into a pair. */
4666 {
4671 else
4676 }
4678 /* REGNO2 can be saved/restored in a pair with REGNO. */
4690 else
4695 {
4698 }
4699 else
4700 {
4703 }
4706 }
4707 }
4709 /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
4711 static void
4713 {
4715 }
4717 /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
4719 static void
4721 {
4723 }
4725 /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
4727 static void
4729 {
4733 }
4735 /* Add a REG_CFA_EXPRESSION note to INSN to say that register REG
4736 is saved at BASE + OFFSET. */
4738 static void
4741 {
4745 }
4747 /* AArch64 stack frames generated by this compiler look like:
4749 +-------------------------------+
4750 | |
4751 | incoming stack arguments |
4752 | |
4753 +-------------------------------+
4754 | | <-- incoming stack pointer (aligned)
4755 | callee-allocated save area |
4756 | for register varargs |
4757 | |
4758 +-------------------------------+
4759 | local variables | <-- frame_pointer_rtx
4760 | |
4761 +-------------------------------+
4762 | padding0 | \
4763 +-------------------------------+ |
4764 | callee-saved registers | | frame.saved_regs_size
4765 +-------------------------------+ |
4766 | LR' | |
4767 +-------------------------------+ |
4768 | FP' | / <- hard_frame_pointer_rtx (aligned)
4769 +-------------------------------+
4770 | dynamic allocation |
4771 +-------------------------------+
4772 | padding |
4773 +-------------------------------+
4774 | outgoing stack arguments | <-- arg_pointer
4775 | |
4776 +-------------------------------+
4777 | | <-- stack_pointer_rtx (aligned)
4779 Dynamic stack allocations via alloca() decrease stack_pointer_rtx
4780 but leave frame_pointer_rtx and hard_frame_pointer_rtx
4781 unchanged. */
4783 /* Generate the prologue instructions for entry into a function.
4784 Establish the stack frame by decreasing the stack pointer with a
4785 properly calculated size and, if necessary, create a frame record
4786 filled with the values of LR and previous frame pointer. The
4787 current FP is also set up if it is in use. */
4789 void
4791 {
4804 /* Sign return address for functions. */
4806 {
4810 }
4816 {
4818 {
4822 (frame_size
4824 }
4827 }
4838 {
4841 {
4846 }
4851 {
4852 /* Variable-sized frames need to describe the save slot
4853 address using DW_CFA_expression rather than DW_CFA_offset.
4854 This means that, without taking further action, the
4855 locations of the registers that we've already saved would
4856 remain based on the stack pointer even after we redefine
4857 the CFA based on the frame pointer. We therefore need new
4858 DW_CFA_expressions to re-express the save slots with addresses
4859 based on the frame pointer. */
4863 /* Add an explicit CFA definition if this was previously
4864 implicit. */
4866 {
4868 callee_offset);
4871 }
4873 /* Change the save slot expressions for the registers that
4874 we've already saved. */
4879 reg_offset);
4880 }
4882 }
4889 }
4891 /* Return TRUE if we can use a simple_return insn.
4893 This function checks whether the callee saved stack is empty, which
4894 means no restore actions are need. The pro_and_epilogue will use
4895 this to check whether shrink-wrapping opt is feasible. */
4897 bool
4899 {
4909 }
4911 /* Generate the epilogue instructions for returning from a function.
4912 This is almost exactly the reverse of the prolog sequence, except
4913 that we need to insert barriers to avoid scheduling loads that read
4914 from a deallocated stack, and we optimize the unwind records by
4915 emitting them all together if possible. */
4916 void
4918 {
4929 /* A stack clash protection prologue may not have left IP0_REGNUM or
4930 IP1_REGNUM in a usable state. The same is true for allocations
4931 with an SVE component, since we then need both temporary registers
4932 for each allocation. */
4937 /* We need to add memory barrier to prevent read from deallocated stack. */
4938 bool need_barrier_p
4942 /* Emit a barrier to prevent loads from a deallocated stack. */
4946 {
4949 }
4951 /* Restore the stack pointer from the frame pointer if it may not
4952 be the same as the stack pointer. */
4957 /* If writeback is used when restoring callee-saves, the CFA
4958 is restored on the instruction doing the writeback. */
4962 else
4978 {
4979 /* Emit delayed restores and set the CFA to be SP + initial_adjust. */
4985 }
4991 {
4992 /* Emit delayed restores and reset the CFA to be SP. */
4997 }
4999 /* We prefer to emit the combined return/authenticate instruction RETAA,
5000 however there are three cases in which we must instead emit an explicit
5001 authentication instruction.
5003 1) Sibcalls don't return in a normal way, so if we're about to call one
5004 we must authenticate.
5006 2) The RETAA instruction is not available before ARMv8.3-A, so if we are
5007 generating code for !TARGET_ARMV8_3 we can't use it and must
5008 explicitly authenticate.
5010 3) On an eh_return path we make extra stack adjustments to update the
5011 canonical frame address to be the exception handler's CFA. We want
5012 to authenticate using the CFA of the function which calls eh_return.
5013 */
5016 {
5020 }
5022 /* Stack adjustment for exception handler. */
5024 {
5025 /* We need to unwind the stack by the offset computed by
5026 EH_RETURN_STACKADJ_RTX. We have already reset the CFA
5027 to be SP; letting the CFA move during this adjustment
5028 is just as correct as retaining the CFA from the body
5029 of the function. Therefore, do nothing special. */
5031 }
5036 }
5038 /* Implement EH_RETURN_HANDLER_RTX. EH returns need to either return
5039 normally or return to a previous frame after unwinding.
5041 An EH return uses a single shared return sequence. The epilogue is
5042 exactly like a normal epilogue except that it has an extra input
5043 register (EH_RETURN_STACKADJ_RTX) which contains the stack adjustment
5044 that must be applied after the frame has been destroyed. An extra label
5045 is inserted before the epilogue which initializes this register to zero,
5046 and this is the entry point for a normal return.
5048 An actual EH return updates the return address, initializes the stack
5049 adjustment and jumps directly into the epilogue (bypassing the zeroing
5050 of the adjustment). Since the return address is typically saved on the
5051 stack when a function makes a call, the saved LR must be updated outside
5052 the epilogue.
5054 This poses problems as the store is generated well before the epilogue,
5055 so the offset of LR is not known yet. Also optimizations will remove the
5056 store as it appears dead, even after the epilogue is generated (as the
5057 base or offset for loading LR is different in many cases).
5059 To avoid these problems this implementation forces the frame pointer
5060 in eh_return functions so that the location of LR is fixed and known early.
5061 It also marks the store volatile, so no optimization is permitted to
5062 remove the store. */
5063 rtx
5065 {
5069 /* Mark the store volatile, so no optimization is permitted to remove it. */
5072 }
5074 /* Output code to add DELTA to the first argument, and then jump
5075 to FUNCTION. Used for C++ multiple inheritance. */
5076 static void
5078 HOST_WIDE_INT delta,
5079 HOST_WIDE_INT vcall_offset,
5080 tree function)
5081 {
5082 /* The this pointer is always in x0. Note that this differs from
5083 Arm where the this pointer maybe bumped to r1 if r0 is required
5084 to return a pointer to an aggregate. On AArch64 a result value
5085 pointer will be in x8. */
5099 else
5100 {
5105 {
5109 else
5112 }
5116 else
5123 else
5124 {
5126 Pmode);
5128 }
5132 else
5138 }
5140 /* Generate a tail call to the target function. */
5142 {
5145 }
5157 /* Stop pretending to be a post-reload pass. */
5159 }
5161 static bool
5163 {
5168 {
5172 /* Don't recurse into UNSPEC_TLS looking for TLS symbols; these are
5173 TLS offsets, not real symbol references. */
5176 }
5178 }
5181 /* Return true if val can be encoded as a 12-bit unsigned immediate with
5182 a left shift of 0 or 12 bits. */
5183 bool
5185 {
5188 );
5189 }
5192 /* Return true if val is an immediate that can be loaded into a
5193 register by a MOVZ instruction. */
5194 static bool
5196 {
5198 {
5202 }
5203 else
5204 {
5205 /* Ignore sign extension. */
5207 }
5210 }
5212 /* VAL is a value with the inner mode of MODE. Replicate it to fill a
5213 64-bit (DImode) integer. */
5215 static unsigned HOST_WIDE_INT
5217 {
5220 {
5224 }
5226 }
5228 /* Multipliers for repeating bitmasks of width 32, 16, 8, 4, and 2. */
5231 {
5237 };
5240 /* Return true if val is a valid bitmask immediate. */
5242 bool
5244 {
5248 /* Check for a single sequence of one bits and return quickly if so.
5249 The special cases of all ones and all zeroes returns false. */
5256 /* Replicate 32-bit immediates so we can treat them as 64-bit. */
5260 /* Invert if the immediate doesn't start with a zero bit - this means we
5261 only need to search for sequences of one bits. */
5265 /* Find the first set bit and set tmp to val with the first sequence of one
5266 bits removed. Return success if there is a single sequence of ones. */
5273 /* Find the next set bit and compute the difference in bit position. */
5278 /* Check the bit position difference is a power of 2, and that the first
5279 sequence of one bits fits within 'bits' bits. */
5283 /* Check the sequence of one bits is repeated 64/bits times. */
5285 }
5287 /* Create mask of ones, covering the lowest to highest bits set in VAL_IN.
5288 Assumed precondition: VAL_IN Is not zero. */
5290 unsigned HOST_WIDE_INT
5292 {
5299 }
5301 /* Create constant where bits outside of lowest bit set to highest bit set
5302 are set to 1. */
5304 unsigned HOST_WIDE_INT
5306 {
5308 }
5310 /* Return true if VAL_IN is a valid 'and' bitmask immediate. */
5312 bool
5314 {
5315 scalar_int_mode int_mode;
5328 }
5330 /* Return true if val is an immediate that can be loaded into a
5331 register in a single instruction. */
5332 bool
5334 {
5335 scalar_int_mode int_mode;
5342 }
5344 static bool
5346 {
5352 /* There's no way to calculate VL-based values using relocations. */
5360 {
5364 else
5365 /* Avoid generating a 64-bit relocation in ILP32; leave
5366 to aarch64_expand_mov_immediate to handle it properly. */
5368 }
5371 }
5373 /* Implement TARGET_CASE_VALUES_THRESHOLD.
5374 The expansion for a table switch is quite expensive due to the number
5375 of instructions, the table lookup and hard to predict indirect jump.
5376 When optimizing for speed, and -O3 enabled, use the per-core tuning if
5377 set, otherwise use tables for > 16 cases as a tradeoff between size and
5378 performance. When optimizing for size, use the default setting. */
5380 static unsigned int
5382 {
5383 /* Use the specified limit for the number of cases before using jump
5384 tables at higher optimization levels. */
5388 else
5390 }
5392 /* Return true if register REGNO is a valid index register.
5393 STRICT_P is true if REG_OK_STRICT is in effect. */
5395 bool
5397 {
5399 {
5407 }
5409 }
5411 /* Return true if register REGNO is a valid base register for mode MODE.
5412 STRICT_P is true if REG_OK_STRICT is in effect. */
5414 bool
5416 {
5418 {
5426 }
5428 /* The fake registers will be eliminated to either the stack or
5429 hard frame pointer, both of which are usually valid base registers.
5430 Reload deals with the cases where the eliminated form isn't valid. */
5435 }
5437 /* Return true if X is a valid base register for mode MODE.
5438 STRICT_P is true if REG_OK_STRICT is in effect. */
5440 static bool
5442 {
5449 }
5451 /* Return true if address offset is a valid index. If it is, fill in INFO
5452 appropriately. STRICT_P is true if REG_OK_STRICT is in effect. */
5454 static bool
5457 {
5459 rtx index;
5462 /* (reg:P) */
5465 {
5469 }
5470 /* (sign_extend:DI (reg:SI)) */
5475 {
5480 }
5481 /* (mult:DI (sign_extend:DI (reg:SI)) (const_int scale)) */
5488 {
5493 }
5494 /* (ashift:DI (sign_extend:DI (reg:SI)) (const_int shift)) */
5501 {
5506 }
5507 /* (sign_extract:DI (mult:DI (reg:DI) (const_int scale)) 32+shift 0) */
5514 {
5522 }
5523 /* (and:DI (mult:DI (reg:DI) (const_int scale))
5524 (const_int 0xffffffff<<shift)) */
5531 {
5537 }
5538 /* (sign_extract:DI (ashift:DI (reg:DI) (const_int shift)) 32+shift 0) */
5545 {
5553 }
5554 /* (and:DI (ashift:DI (reg:DI) (const_int shift))
5555 (const_int 0xffffffff<<shift)) */
5562 {
5568 }
5569 /* (mult:P (reg:P) (const_int scale)) */
5574 {
5578 }
5579 /* (ashift:P (reg:P) (const_int shift)) */
5584 {
5588 }
5589 else
5598 {
5602 }
5603 else
5604 {
5609 }
5613 {
5618 }
5621 }
5623 /* Return true if MODE is one of the modes for which we
5624 support LDP/STP operations. */
5626 static bool
5628 {
5636 }
5638 /* Return true if REGNO is a virtual pointer register, or an eliminable
5639 "soft" frame register. Like REGNO_PTR_FRAME_P except that we don't
5640 include stack_pointer or hard_frame_pointer. */
5641 static bool
5643 {
5648 }
5650 /* Return true if X is a valid address of type TYPE for machine mode MODE.
5651 If it is, fill in INFO appropriately. STRICT_P is true if
5652 REG_OK_STRICT is in effect. */
5654 static bool
5658 {
5661 poly_int64 offset;
5663 HOST_WIDE_INT const_size;
5665 /* On BE, we use load/store pair for all large int mode load/stores.
5666 TI/TFmode may also use a load/store pair. */
5675 /* If we are dealing with ADDR_QUERY_LDP_STP_N that means the incoming mode
5676 corresponds to the actual size of the memory being loaded/stored and the
5677 mode of the corresponding addressing mode is half of that. */
5687 /* For SVE, only accept [Rn], [Rn, Rm, LSL #shift] and
5688 [Rn, #offset, MUL VL]. */
5693 /* On LE, for AdvSIMD, don't support anything other than POST_INC or
5694 REG addressing. */
5696 && !BYTES_BIG_ENDIAN
5704 {
5721 {
5728 }
5733 {
5739 /* TImode and TFmode values are allowed in both pairs of X
5740 registers and individual Q registers. The available
5741 address modes are:
5742 X,X: 7-bit signed scaled offset
5743 Q: 9-bit signed offset
5744 We conservatively require an offset representable in either mode.
5745 When performing the check for pairs of X registers i.e. LDP/STP
5746 pass down DImode since that is the natural size of the LDP/STP
5747 instruction memory accesses. */
5753 /* A 7bit offset check because OImode will emit a ldp/stp
5754 instruction (only big endian will get here).
5755 For ldp/stp instructions, the offset is scaled for the size of a
5756 single element of the pair. */
5760 /* Three 9/12 bit offsets checks because CImode will emit three
5761 ldr/str instructions (only big endian will get here). */
5768 /* Two 7bit offsets checks because XImode will emit two ldp/stp
5769 instructions (only big endian will get here). */
5775 /* Make "m" use the LD1 offset range for SVE data modes, so
5776 that pre-RTL optimizers like ivopts will work to that
5777 instead of the wider LDR/STR range. */
5784 {
5785 poly_int64 end_offset = (offset
5792 end_offset)));
5793 }
5803 else
5806 }
5809 {
5810 /* Look for base + (scaled/extended) index register. */
5813 {
5816 }
5819 {
5822 }
5823 }
5844 {
5848 /* TImode and TFmode values are allowed in both pairs of X
5849 registers and individual Q registers. The available
5850 address modes are:
5851 X,X: 7-bit signed scaled offset
5852 Q: 9-bit signed offset
5853 We conservatively require an offset representable in either mode.
5854 */
5864 else
5866 }
5872 /* load literal: pc-relative constant pool entry. Only supported
5873 for SI mode or larger. */
5879 {
5887 }
5896 {
5902 {
5903 /* The symbol and offset must be aligned to the access size. */
5909 {
5913 }
5919 else
5928 }
5929 }
5934 }
5935 }
5937 /* Return true if the address X is valid for a PRFM instruction.
5938 STRICT_P is true if we should do strict checking with
5939 aarch64_classify_address. */
5941 bool
5943 {
5946 /* PRFM accepts the same addresses as DImode... */
5951 /* ... except writeback forms. */
5953 }
5955 bool
5957 {
5958 rtx offset;
5962 }
5964 /* Classify the base of symbolic expression X. */
5966 enum aarch64_symbol_type
5968 {
5969 rtx offset;
5973 }
5976 /* Return TRUE if X is a legitimate address for accessing memory in
5977 mode MODE. */
5978 static bool
5980 {
5984 }
5986 /* Return TRUE if X is a legitimate address of type TYPE for accessing
5987 memory in mode MODE. STRICT_P is true if REG_OK_STRICT is in effect. */
5988 bool
5990 aarch64_addr_query_type type)
5991 {
5995 }
5997 /* Implement TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT. */
5999 static bool
6001 poly_int64 orig_offset,
6002 machine_mode mode)
6003 {
6004 HOST_WIDE_INT size;
6006 {
6009 /* A general SVE offset is A * VQ + B. Remove the A component from
6010 coefficient 0 in order to get the constant B. */
6013 /* Split an out-of-range address displacement into a base and
6014 offset. Use 4KB range for 1- and 2-byte accesses and a 16KB
6015 range otherwise to increase opportunities for sharing the base
6016 address of different sizes. Unaligned accesses use the signed
6017 9-bit range, TImode/TFmode use the intersection of signed
6018 scaled 7-bit and signed 9-bit offset. */
6023 else
6029 /* Split the offset into second_offset and the rest. */
6033 }
6034 else
6035 {
6036 /* Get the mode we should use as the basis of the range. For structure
6037 modes this is the mode of one vector. */
6039 machine_mode step_mode
6042 /* Get the "mul vl" multiplier we'd like to use. */
6046 /* LDR supports a 9-bit range, but the move patterns for
6047 structure modes require all vectors to be in range of the
6048 same base. The simplest way of accomodating that while still
6049 promoting reuse of anchor points between different modes is
6050 to use an 8-bit range unconditionally. */
6052 else
6053 /* Predicates are only handled singly, so we might as well use
6054 the full range. */
6059 /* Convert the "mul vl" multiplier into a byte offset. */
6064 /* Split the offset into second_offset and the rest. */
6068 }
6069 }
6071 /* Return the binary representation of floating point constant VALUE in INTVAL.
6072 If the value cannot be converted, return false without setting INTVAL.
6073 The conversion is done in the given MODE. */
6074 bool
6076 {
6078 /* We make a general exception for 0. */
6080 {
6083 }
6085 scalar_float_mode mode;
6089 /* Only support up to DF mode. */
6101 {
6105 }
6106 else
6111 }
6113 /* Return TRUE if rtx X is an immediate constant that can be moved using a
6114 single MOV(+MOVK) followed by an FMOV. */
6115 bool
6117 {
6122 /* Determine whether it's cheaper to write float constants as
6123 mov/movk pairs over ldr/adrp pairs. */
6129 {
6131 ? SImode
6136 }
6139 }
6141 /* Return TRUE if rtx X is immediate constant 0.0 */
6142 bool
6144 {
6151 }
6153 /* Return TRUE if rtx X is immediate constant that fits in a single
6154 MOVI immediate operation. */
6155 bool
6157 {
6161 machine_mode vmode;
6162 scalar_int_mode imode;
6167 {
6171 /* We make a general exception for 0. */
6176 }
6180 else
6183 /* use a 64 bit mode for everything except for DI/DF mode, where we use
6184 a 128 bit vector mode. */
6191 }
6194 /* Return the fixed registers used for condition codes. */
6196 static bool
6198 {
6202 }
6204 /* This function is used by the call expanders of the machine description.
6205 RESULT is the register in which the result is returned. It's NULL for
6206 "call" and "sibcall".
6207 MEM is the location of the function call.
6208 SIBCALL indicates whether this function call is normal call or sibling call.
6209 It will generate different pattern accordingly. */
6211 void
6213 {
6215 rtvec vec;
6216 machine_mode mode;
6223 /* Decide if we should generate indirect calls by loading the
6224 address of the callee into a register before performing
6225 the branch-and-link. */
6239 else
6246 }
6248 /* Emit call insn with PAT and do aarch64-specific handling. */
6250 void
6252 {
6258 }
6260 machine_mode
6262 {
6263 /* All floating point compares return CCFP if it is an equality
6264 comparison, and CCFPE otherwise. */
6266 {
6268 {
6289 }
6290 }
6292 /* Equality comparisons of short modes against zero can be performed
6293 using the TST instruction with the appropriate bitmask. */
6299 /* Similarly, comparisons of zero_extends from shorter modes can
6300 be performed using an ANDS with an immediate mask. */
6316 /* A compare with a shifted operand. Because of canonicalization,
6317 the comparison will have to be swapped when we emit the assembly
6318 code. */
6326 /* Similarly for a negated operand, but we can only do this for
6327 equalities. */
6334 /* A test for unsigned overflow. */
6341 /* A test for signed overflow. */
6348 /* For everything else, return CCmode. */
6350 }
6352 static int
6355 int
6357 {
6364 }
6366 static int
6368 {
6370 {
6374 {
6388 }
6393 {
6405 }
6410 {
6422 }
6427 {
6433 }
6438 {
6442 }
6447 {
6451 }
6456 {
6460 }
6465 }
6468 }
6470 bool
6472 HOST_WIDE_INT minval,
6473 HOST_WIDE_INT maxval)
6474 {
6475 rtx elt;
6479 }
6481 bool
6483 {
6485 }
6487 /* Return true if VEC is a constant in which every element is in the range
6488 [MINVAL, MAXVAL]. The elements do not need to have the same value. */
6490 static bool
6492 HOST_WIDE_INT minval,
6493 HOST_WIDE_INT maxval)
6494 {
6506 {
6511 }
6513 }
6515 /* N Z C V. */
6516 #define AARCH64_CC_V 1
6517 #define AARCH64_CC_C (1 << 1)
6518 #define AARCH64_CC_Z (1 << 2)
6519 #define AARCH64_CC_N (1 << 3)
6521 /* N Z C V flags for ccmp. Indexed by AARCH64_COND_CODE. */
6523 {
6540 };
6542 /* Print floating-point vector immediate operand X to F, negating it
6543 first if NEGATE is true. Return true on success, false if it isn't
6544 a constant we can handle. */
6546 static bool
6548 {
6549 rtx elt;
6558 /* We only handle the SVE single-bit immediates here. */
6565 else
6569 }
6571 /* Return the equivalent letter for size. */
6572 static char
6574 {
6576 {
6582 }
6583 }
6585 /* Print operand X to file F in a target specific manner according to CODE.
6586 The acceptable formatting commands given by CODE are:
6587 'c': An integer or symbol address without a preceding #
6588 sign.
6589 'C': Take the duplicated element in a vector constant
6590 and print it in hex.
6591 'D': Take the duplicated element in a vector constant
6592 and print it as an unsigned integer, in decimal.
6593 'e': Print the sign/zero-extend size as a character 8->b,
6594 16->h, 32->w.
6595 'p': Prints N such that 2^N == X (X must be power of 2 and
6596 const int).
6597 'P': Print the number of non-zero bits in X (a const_int).
6598 'H': Print the higher numbered register of a pair (TImode)
6599 of regs.
6600 'm': Print a condition (eq, ne, etc).
6601 'M': Same as 'm', but invert condition.
6602 'N': Take the duplicated element in a vector constant
6603 and print the negative of it in decimal.
6604 'b/h/s/d/q': Print a scalar FP/SIMD register name.
6605 'S/T/U/V': Print a FP/SIMD register name for a register list.
6606 The register printed is the FP/SIMD register name
6607 of X + 0/1/2/3 for S/T/U/V.
6608 'R': Print a scalar FP/SIMD register name + 1.
6609 'X': Print bottom 16 bits of integer constant in hex.
6610 'w/x': Print a general register name or the zero register
6611 (32-bit or 64-bit).
6612 '0': Print a normal operand, if it's a general register,
6613 then we assume DImode.
6614 'k': Print NZCV for conditional compare instructions.
6615 'A': Output address constant representing the first
6616 argument of X, specifying a relocation offset
6617 if appropriate.
6618 'L': Output constant address specified by X
6619 with a relocation offset if appropriate.
6620 'G': Prints address of X, specifying a PC relative
6621 relocation mode if appropriate.
6622 'y': Output address of LDP or STP - this is used for
6623 some LDP/STPs which don't use a PARALLEL in their
6624 pattern (so the mode needs to be adjusted).
6625 'z': Output address of a typical LDP or STP. */
6627 static void
6629 {
6630 rtx elt;
6632 {
6635 {
6647 {
6650 }
6651 /* Fall through. */
6655 }
6659 {
6664 {
6667 }
6670 {
6683 }
6684 }
6688 {
6692 {
6695 }
6698 }
6703 {
6706 }
6713 {
6716 }
6719 {
6722 }
6729 {
6731 /* CONST_TRUE_RTX means al/nv (al is the default, don't print it). */
6733 {
6737 }
6740 {
6743 }
6750 }
6755 {
6758 }
6764 ;
6765 else
6766 {
6769 }
6778 {
6779 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6781 }
6790 {
6791 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6793 }
6801 {
6802 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6804 }
6810 {
6813 }
6818 {
6819 /* Print a replicated constant in hex. */
6821 {
6824 }
6827 }
6831 {
6832 /* Print a replicated constant in decimal, treating it as
6833 unsigned. */
6835 {
6838 }
6841 }
6848 {
6851 }
6854 {
6857 }
6860 {
6863 }
6865 /* Fall through */
6869 {
6872 }
6875 {
6878 {
6881 else
6882 {
6883 char suffix
6888 }
6889 }
6890 else
6909 {
6912 }
6913 /* fall through */
6917 {
6920 }
6926 ;
6927 else
6928 {
6931 }
6935 /* Since we define TARGET_SUPPORTS_WIDE_INT we shouldn't ever
6936 be getting CONST_DOUBLEs holding integers. */
6939 {
6942 }
6944 {
6945 #define buf_size 20
6953 #undef buf_size
6954 }
6960 }
6968 {
6995 }
7001 {
7036 }
7042 {
7048 }
7053 {
7054 HOST_WIDE_INT cond_code;
7057 {
7060 }
7065 }
7070 {
7075 {
7078 }
7082 ? ADDR_QUERY_LDP_STP_N
7085 }
7091 }
7092 }
7094 /* Print address 'x' of a memory access with mode 'mode'.
7095 'op' is the context required by aarch64_classify_address. It can either be
7096 MEM for a normal memory access or PARALLEL for LDP/STP. */
7097 static bool
7099 aarch64_addr_query_type type)
7100 {
7104 /* Check all addresses are Pmode - including ILP32. */
7110 {
7115 {
7116 HOST_WIDE_INT vnum
7121 }
7123 {
7124 HOST_WIDE_INT vnum
7129 }
7130 else
7139 else
7148 else
7157 else
7163 /* Writeback is only supported for fixed-width modes. */
7166 {
7189 }
7201 }
7204 }
7206 /* Print address 'x' of a memory access with mode 'mode'. */
7207 static void
7209 {
7212 }
7214 bool
7216 {
7223 /* UNSPEC_TLS entries for a symbol include a LABEL_REF for the
7224 referencing instruction, but they are constant offsets, not
7225 symbols. */
7231 {
7233 {
7239 }
7242 }
7245 }
7247 /* Implement REGNO_REG_CLASS. */
7249 enum reg_class
7251 {
7269 }
7271 /* OFFSET is an address offset for mode MODE, which has SIZE bytes.
7272 If OFFSET is out of range, return an offset of an anchor point
7273 that is in range. Return 0 otherwise. */
7275 static HOST_WIDE_INT
7277 machine_mode mode)
7278 {
7279 /* Does it look like we'll need a 16-byte load/store-pair operation? */
7283 /* For offsets that aren't a multiple of the access size, the limit is
7284 -256...255. */
7286 {
7287 /* BLKmode typically uses LDP of X-registers. */
7291 }
7293 /* Small negative offsets are supported. */
7300 /* Use 12-bit offset by access size. */
7302 }
7304 static rtx
7306 {
7307 /* Try to split X+CONST into Y=X+(CONST & ~mask), Y+(CONST&mask),
7308 where mask is selected by alignment and size of the offset.
7309 We try to pick as large a range for the offset as possible to
7310 maximize the chance of a CSE. However, for aligned addresses
7311 we limit the range to 4k so that structures with different sized
7312 elements are likely to use the same base. We need to be careful
7313 not to split a CONST for some forms of address expression, otherwise
7314 it will generate sub-optimal code. */
7317 {
7323 {
7327 /* Force any scaling into a temp for CSE. */
7331 /* Let the pointer register be in op0. */
7335 /* If the pointer is virtual or frame related, then we know that
7336 virtual register instantiation or register elimination is going
7337 to apply a second constant. We want the two constants folded
7338 together easily. Therefore, emit as (OP0 + CONST) + OP1. */
7340 {
7344 }
7346 /* Otherwise, in order to encourage CSE (and thence loop strength
7347 reduce) scaled addresses, emit as (OP0 + OP1) + CONST. */
7351 }
7353 HOST_WIDE_INT size;
7355 {
7357 mode);
7359 {
7363 }
7364 }
7365 }
7368 }
7370 static reg_class_t
7372 reg_class_t rclass,
7373 machine_mode mode,
7375 {
7376 /* Use aarch64_sve_reload_be for SVE reloads that cannot be handled
7377 directly by the *aarch64_sve_mov<mode>_be move pattern. See the
7378 comment at the head of aarch64-sve.md for more details about the
7379 big-endian handling. */
7385 {
7388 }
7390 /* If we have to disable direct literal pool loads and stores because the
7391 function is too big, then we need a scratch register. */
7396 {
7399 }
7401 /* Without the TARGET_SIMD instructions we cannot move a Q register
7402 to a Q register directly. We need a scratch. */
7406 {
7409 }
7411 /* A TFmode or TImode memory access should be handled via an FP_REGS
7412 because AArch64 has richer addressing modes for LDR/STR instructions
7413 than LDP/STP instructions. */
7422 }
7424 static bool
7426 {
7429 /* If we need a frame pointer, ARG_POINTER_REGNUM and FRAME_POINTER_REGNUM
7430 can only eliminate to HARD_FRAME_POINTER_REGNUM. */
7434 }
7436 poly_int64
7438 {
7442 {
7449 }
7452 {
7456 }
7459 }
7461 /* Implement RETURN_ADDR_RTX. We do not support moving back to a
7462 previous frame. */
7464 rtx
7466 {
7470 }
7473 static void
7475 {
7477 {
7480 }
7481 else
7482 {
7485 }
7490 }
7492 static void
7494 {
7498 /* Don't need to copy the trailing D-words, we fill those in below. */
7510 /* XXX We should really define a "clear_cache" pattern and use
7511 gen_clear_cache(). */
7516 ptr_mode);
7517 }
7519 static unsigned char
7521 {
7522 /* ??? Logically we should only need to provide a value when
7523 HARD_REGNO_MODE_OK says that at least one register in REGCLASS
7524 can hold MODE, but at the moment we need to handle all modes.
7525 Just ignore any runtime parts for registers that can't store them. */
7529 {
7555 }
7557 }
7559 static reg_class_t
7561 {
7566 {
7572 }
7574 /* Register eliminiation can result in a request for
7575 SP+constant->FP_REGS. We cannot support such operations which
7576 use SP as source and an FP_REG as destination, so reject out
7577 right now. */
7579 {
7582 /* Look through a possible SUBREG introduced by ILP32. */
7588 POINTER_REGS));
7590 }
7593 }
7595 void
7597 {
7599 }
7601 static void
7603 {
7606 else
7607 {
7609 /* While priority is known to be in range [0, 65535], so 18 bytes
7610 would be enough, the compiler might not know that. To avoid
7611 -Wformat-truncation false positive, use a larger size. */
7618 }
7619 }
7621 static void
7623 {
7626 else
7627 {
7629 /* While priority is known to be in range [0, 65535], so 18 bytes
7630 would be enough, the compiler might not know that. To avoid
7631 -Wformat-truncation false positive, use a larger size. */
7638 }
7639 }
7643 {
7649 {
7650 {
7653 },
7654 {
7657 },
7658 {
7661 },
7662 /* We assume that DImode is only generated when not optimizing and
7663 that we don't really need 64-bit address offsets. That would
7664 imply an object file with 8GB of code in a single function! */
7665 {
7668 }
7669 };
7678 /* Need to implement table size reduction, by chaning the code below. */
7688 }
7691 /* Return size in bits of an arithmetic operand which is shifted/scaled and
7692 masked such that it is suitable for a UXTB, UXTH, or UXTW extend
7693 operator. */
7695 int
7697 {
7699 {
7702 {
7706 }
7707 }
7709 }
7711 /* Constant pools are per function only when PC relative
7712 literal loads are true or we are in the large memory
7713 model. */
7717 {
7720 }
7722 static bool
7724 {
7725 /* We can't use blocks for constants when we're using a per-function
7726 constant pool. */
7728 }
7730 /* Select appropriate section for constants depending
7731 on where we place literal pools. */
7735 rtx x,
7737 {
7742 }
7744 /* Implement ASM_OUTPUT_POOL_EPILOGUE. */
7745 void
7747 HOST_WIDE_INT offset)
7748 {
7749 /* When using per-function literal pools, we must ensure that any code
7750 section is aligned to the minimal instruction length, lest we get
7751 errors from the assembler re "unaligned instructions". */
7754 }
7756 /* Costs. */
7758 /* Helper function for rtx cost calculation. Strip a shift expression
7759 from X. Returns the inner operand if successful, or the original
7760 expression on failure. */
7761 static rtx
7763 {
7766 /* We accept both ROTATERT and ROTATE: since the RHS must be a constant
7767 we can convert both to ROR during final output. */
7782 }
7784 /* Helper function for rtx cost calculation. Strip an extend
7785 expression from X. Returns the inner operand if successful, or the
7786 original expression on failure. We deal with a number of possible
7787 canonicalization variations here. If STRIP_SHIFT is true, then
7788 we can strip off a shift also. */
7789 static rtx
7791 {
7792 scalar_int_mode mode;
7798 /* Zero and sign extraction of a widened value. */
7806 /* It can also be represented (for zero-extend) as an AND with an
7807 immediate. */
7816 /* Now handle extended register, as this may also have an optional
7817 left shift by 1..4. */
7832 }
7834 /* Return true iff CODE is a shift supported in combination
7835 with arithmetic instructions. */
7837 static bool
7839 {
7841 }
7844 /* Return true iff X is a cheap shift without a sign extend. */
7846 static bool
7848 {
7874 }
7876 /* Helper function for rtx cost calculation. Calculate the cost of
7877 a MULT or ASHIFT, which may be part of a compound PLUS/MINUS rtx.
7878 Return the calculated cost of the expression, recursing manually in to
7879 operands where needed. */
7881 static int
7883 {
7899 /* Integer multiply/fma. */
7901 {
7902 /* The multiply will be canonicalized as a shift, cost it as such. */
7906 {
7910 {
7912 {
7913 /* If the shift is considered cheap,
7914 then don't add any cost. */
7916 ;
7918 /* ARITH + shift-by-register. */
7921 /* ARITH + extended register. We don't have a cost field
7922 for ARITH+EXTEND+SHIFT, so use extend_arith here. */
7924 else
7925 /* ARITH + shift-by-immediate. */
7927 }
7928 else
7929 /* LSL (immediate). */
7932 }
7933 /* Strip extends as we will have costed them in the case above. */
7940 }
7942 /* MNEG or [US]MNEGL. Extract the NEG operand and indicate that it's a
7943 compound and let the below cases handle it. After all, MNEG is a
7944 special-case alias of MSUB. */
7946 {
7949 }
7951 /* Integer multiplies or FMAs have zero/sign extending variants. */
7956 {
7961 {
7963 /* SMADDL/UMADDL/UMSUBL/SMSUBL. */
7965 else
7966 /* MUL/SMULL/UMULL. */
7968 }
7971 }
7973 /* This is either an integer multiply or a MADD. In both cases
7974 we want to recurse and cost the operands. */
7979 {
7981 /* MADD/MSUB. */
7983 else
7984 /* MUL. */
7986 }
7989 }
7990 else
7991 {
7993 {
7994 /* Floating-point FMA/FMUL can also support negations of the
7995 operands, unless the rounding mode is upward or downward in
7996 which case FNMUL is different than FMUL with operand negation. */
8000 {
8005 }
8008 /* FMADD/FNMADD/FNMSUB/FMSUB. */
8010 else
8011 /* FMUL/FNMUL. */
8013 }
8018 }
8019 }
8021 static int
8023 machine_mode mode,
8024 addr_space_t as ATTRIBUTE_UNUSED,
8026 {
8034 {
8036 {
8037 /* This is a CONST or SYMBOL ref which will be split
8038 in a different way depending on the code model in use.
8039 Cost it through the generic infrastructure. */
8041 /* Divide through by the cost of one instruction to
8042 bring it to the same units as the address costs. */
8044 /* The cost is then the cost of preparing the address,
8045 followed by an immediate (possibly 0) offset. */
8047 }
8048 else
8049 {
8050 /* This is most likely a jump table from a case
8051 statement. */
8053 }
8054 }
8057 {
8069 else
8088 }
8092 {
8093 /* For the sake of calculating the cost of the shifted register
8094 component, we can treat same sized modes in the same way. */
8101 else
8102 /* We can't tell, or this is a 128-bit vector. */
8104 }
8107 }
8109 /* Return the cost of a branch. If SPEED_P is true then the compiler is
8110 optimizing for speed. If PREDICTABLE_P is true then the branch is predicted
8111 to be taken. */
8113 int
8115 {
8116 /* When optimizing for speed, use the cost of unpredictable branches. */
8122 else
8124 }
8126 /* Return true if the RTX X in mode MODE is a zero or sign extract
8127 usable in an ADD or SUB (extended register) instruction. */
8128 static bool
8130 {
8131 /* Catch add with a sign extract.
8132 This is add_<optab><mode>_multp2. */
8135 {
8146 op1))
8147 {
8149 }
8150 }
8151 /* The simple case <ARITH>, XD, XN, XM, [us]xt.
8152 No shift. */
8158 }
8160 static bool
8162 {
8164 {
8176 }
8177 }
8179 /* Return true iff X is an rtx that will match an extr instruction
8180 i.e. as described in the *extr<mode>5_insn family of patterns.
8181 OP0 and OP1 will be set to the operands of the shifts involved
8182 on success and will be NULL_RTX otherwise. */
8184 static bool
8186 {
8188 scalar_int_mode mode;
8203 {
8204 /* Canonicalise locally to ashift in op0, lshiftrt in op1. */
8216 {
8220 }
8221 }
8224 }
8226 /* Calculate the cost of calculating (if_then_else (OP0) (OP1) (OP2)),
8227 storing it in *COST. Result is true if the total cost of the operation
8228 has now been calculated. */
8229 static bool
8231 {
8232 rtx inner;
8233 rtx comparator;
8237 {
8241 }
8242 else
8243 {
8247 }
8250 {
8251 /* Conditional branch. */
8254 else
8255 {
8257 {
8259 {
8260 /* TBZ/TBNZ/CBZ/CBNZ. */
8262 /* TBZ/TBNZ. */
8265 else
8266 /* CBZ/CBNZ. */
8270 }
8271 }
8273 {
8274 /* TBZ/TBNZ. */
8277 }
8278 }
8279 }
8281 {
8282 /* CCMP. */
8284 {
8285 /* Increase cost of CCMP reg, 0, imm, CC to prefer CMP reg, 0. */
8289 {
8296 else
8298 }
8300 }
8302 /* It's a conditional operation based on the status flags,
8303 so it must be some flavor of CSEL. */
8305 /* CSNEG, CSINV, and CSINC are handled for free as part of CSEL. */
8311 {
8312 /* CSEL with zero-extension (*cmovdi_insn_uxtw). */
8315 }
8320 }
8322 /* We don't know what this is, cost all operands. */
8324 }
8326 /* Check whether X is a bitfield operation of the form shift + extend that
8327 maps down to a UBFIZ/SBFIZ/UBFX/SBFX instruction. If so, return the
8328 operand to which the bitfield operation is applied. Otherwise return
8329 NULL_RTX. */
8331 static rtx
8333 {
8347 {
8365 }
8368 }
8370 /* Return true if the mask and a shift amount from an RTX of the form
8371 (x << SHFT_AMNT) & MASK are valid to combine into a UBFIZ instruction of
8372 mode MODE. See the *andim_ashift<mode>_bfiz pattern. */
8374 bool
8376 rtx shft_amnt)
8377 {
8382 }
8384 /* Calculate the cost of calculating X, storing it in *COST. Result
8385 is true if the total cost of the operation has now been calculated. */
8386 static bool
8389 {
8394 scalar_int_mode int_mode;
8396 /* By default, assume that everything has equivalent cost to the
8397 cheapest instruction. Any additional costs are applied as a delta
8398 above this default. */
8402 {
8404 /* The cost depends entirely on the operands to SET. */
8410 {
8413 {
8427 }
8436 /* Fall through. */
8438 /* The cost is one per vector-register copied. */
8440 {
8443 }
8444 /* const0_rtx is in general free, but we will use an
8445 instruction to set a register to 0. */
8447 {
8448 /* The cost is 1 per register copied. */
8451 }
8452 else
8453 /* Cost is just the cost of the RHS of the set. */
8459 /* Bit-field insertion. Strip any redundant widening of
8460 the RHS to meet the width of the target. */
8471 {
8472 /* MOV immediate is assumed to always be cheap. */
8474 }
8475 else
8476 {
8477 /* BFM. */
8481 }
8486 /* We can't make sense of this, assume default cost. */
8489 }
8493 /* If an instruction can incorporate a constant within the
8494 instruction, the instruction's expression avoids calling
8495 rtx_cost() on the constant. If rtx_cost() is called on a
8496 constant, then it is usually because the constant must be
8497 moved into a register by one or more instructions.
8499 The exception is constant 0, which can be expressed
8500 as XZR/WZR and is therefore free. The exception to this is
8501 if we have (set (reg) (const0_rtx)) in which case we must cost
8502 the move. However, we can catch that when we cost the SET, so
8503 we don't need to consider that here. */
8506 else
8507 {
8508 /* To an approximation, building any other constant is
8509 proportionally expensive to the number of instructions
8510 required to build that constant. This is true whether we
8511 are compiling for SPEED or otherwise. */
8516 }
8521 /* First determine number of instructions to do the move
8522 as an integer constant. */
8526 {
8532 ? SImode
8538 }
8541 {
8542 /* mov[df,sf]_aarch64. */
8544 /* FMOV (scalar immediate). */
8547 {
8548 /* This will be a load from memory. */
8551 else
8553 }
8554 else
8555 /* Otherwise this is +0.0. We get this using MOVI d0, #0
8556 or MOV v0.s[0], wzr - neither of which are modeled by the
8557 cost tables. Just use the default cost. */
8558 {
8559 }
8560 }
8566 {
8567 /* For loads we want the base cost of a load, plus an
8568 approximation for the additional cost of the addressing
8569 mode. */
8583 }
8591 {
8593 {
8594 /* FNEG. */
8596 }
8598 }
8601 {
8604 {
8605 /* CSETM. */
8608 }
8610 /* Cost this as SUB wzr, X. */
8614 }
8617 {
8618 /* Support (neg(fma...)) as a single instruction only if
8619 sign of zeros is unimportant. This matches the decision
8620 making in aarch64.md. */
8622 {
8623 /* FNMADD. */
8626 }
8628 {
8629 /* FNMUL. */
8632 }
8634 /* FNEG. */
8637 }
8644 {
8647 else
8649 }
8659 {
8663 }
8666 {
8667 /* TODO: A write to the CC flags possibly costs extra, this
8668 needs encoding in the cost tables. */
8671 /* ANDS. */
8673 {
8676 }
8679 {
8680 /* ADDS (and CMN alias). */
8683 }
8686 {
8687 /* SUBS. */
8690 }
8695 {
8696 /* COMPARE of ZERO_EXTRACT form of TST-immediate.
8697 Handle it here directly rather than going to cost_logic
8698 since we know the immediate generated for the TST is valid
8699 so we can avoid creating an intermediate rtx for it only
8700 for costing purposes. */
8707 }
8710 {
8711 /* CMN. */
8718 }
8720 /* CMP.
8722 Compare can freely swap the order of operands, and
8723 canonicalization puts the more complex operation first.
8724 But the integer MINUS logic expects the shift/extend
8725 operation in op1. */
8728 {
8731 }
8733 }
8736 {
8737 /* FCMP. */
8742 {
8744 /* FCMP supports constant 0.0 for no extra cost. */
8746 }
8748 }
8751 {
8752 /* Vector compare. */
8757 {
8758 /* Vector cm (eq|ge|gt|lt|le) supports constant 0.0 for no extra
8759 cost. */
8761 }
8763 }
8767 {
8771 cost_minus:
8774 /* Detect valid immediates. */
8780 {
8782 /* SUB(S) (immediate). */
8785 }
8787 /* Look for SUB (extended register). */
8790 {
8798 }
8802 /* Cost this as an FMA-alike operation. */
8806 {
8809 speed);
8811 }
8816 {
8818 {
8819 /* Vector SUB. */
8821 }
8823 {
8824 /* SUB(S). */
8826 }
8828 {
8829 /* FSUB. */
8831 }
8832 }
8834 }
8837 {
8838 rtx new_op0;
8843 cost_plus:
8846 {
8847 /* CSINC. */
8851 }
8856 {
8860 /* ADD (immediate). */
8863 }
8867 /* Look for ADD (extended register). */
8870 {
8878 }
8880 /* Strip any extend, leave shifts behind as we will
8881 cost them through mult_cost. */
8886 {
8888 speed);
8890 }
8895 {
8897 {
8898 /* Vector ADD. */
8900 }
8902 {
8903 /* ADD. */
8905 }
8907 {
8908 /* FADD. */
8910 }
8911 }
8913 }
8919 {
8922 else
8924 }
8929 {
8933 {
8936 else
8938 }
8940 }
8943 {
8950 }
8951 /* Fall through. */
8954 cost_logic:
8959 {
8963 }
8971 {
8972 /* This is a UBFM/SBFM. */
8977 }
8980 {
8982 {
8983 /* We have a mask + shift version of a UBFIZ
8984 i.e. the *andim_ashift<mode>_bfiz pattern. */
8988 {
8995 }
8997 {
8998 /* We possibly get the immediate for free, this is not
8999 modelled. */
9006 }
9007 }
9008 else
9009 {
9012 /* Handle ORN, EON, or BIC. */
9018 /* If we had a shift on op0 then this is a logical-shift-
9019 by-register/immediate operation. Otherwise, this is just
9020 a logical operation. */
9022 {
9024 {
9025 /* Shift by immediate. */
9028 else
9030 }
9031 else
9033 }
9035 /* In both cases we want to cost both operands. */
9042 }
9043 }
9051 {
9052 /* Vector NOT. */
9055 }
9057 /* MVN-shifted-reg. */
9059 {
9066 }
9067 /* EON can have two forms: (xor (not a) b) but also (not (xor a b)).
9068 Handle the second form here taking care that 'a' in the above can
9069 be a shift. */
9071 {
9080 {
9083 else
9085 }
9088 }
9089 /* MVN. */
9098 /* If a value is written in SI mode, then zero extended to DI
9099 mode, the operation will in general be free as a write to
9100 a 'w' register implicitly zeroes the upper bits of an 'x'
9101 register. However, if this is
9103 (set (reg) (zero_extend (reg)))
9105 we must cost the explicit register move. */
9109 {
9112 /* If OP_COST is non-zero, then the cost of the zero extend
9113 is effectively the cost of the inner operation. Otherwise
9114 we have a MOV instruction and we take the cost from the MOV
9115 itself. This is true independently of whether we are
9116 optimizing for space or time. */
9121 }
9123 {
9124 /* All loads can zero extend to any size for free. */
9127 }
9131 {
9136 }
9139 {
9141 {
9142 /* UMOV. */
9144 }
9145 else
9146 {
9147 /* We generate an AND instead of UXTB/UXTH. */
9149 }
9150 }
9155 {
9156 /* LDRSH. */
9158 {
9165 }
9167 }
9171 {
9176 }
9179 {
9182 else
9184 }
9192 {
9194 {
9196 {
9197 /* Vector shift (immediate). */
9199 }
9200 else
9201 {
9202 /* LSL (immediate), UBMF, UBFIZ and friends. These are all
9203 aliases. */
9205 }
9206 }
9208 /* We can incorporate zero/sign extend for free. */
9215 }
9216 else
9217 {
9219 {
9221 /* Vector shift (register). */
9223 }
9224 else
9225 {
9227 /* LSLV. */
9234 {
9236 /* We already demanded XEXP (op1, 0) to be REG_P, so
9237 don't recurse into it. */
9239 }
9240 }
9242 }
9252 {
9253 /* ASR (immediate) and friends. */
9255 {
9258 else
9260 }
9264 }
9265 else
9266 {
9268 {
9270 /* Vector shift (register). */
9272 }
9273 else
9274 {
9276 /* ASR (register) and friends. */
9283 {
9285 /* We already demanded XEXP (op1, 0) to be REG_P, so
9286 don't recurse into it. */
9288 }
9289 }
9291 }
9297 {
9298 /* LDR. */
9301 }
9304 {
9305 /* ADRP, followed by ADD. */
9309 }
9312 {
9313 /* ADR. */
9316 }
9319 {
9320 /* One extra load instruction, after accessing the GOT. */
9324 }
9329 /* ADRP/ADD (immediate). */
9336 /* UBFX/SBFX. */
9338 {
9341 else
9343 }
9345 /* We can trust that the immediates used will be correct (there
9346 are no by-register forms), so we need only cost op0. */
9352 /* aarch64_rtx_mult_cost always handles recursion to its
9353 operands. */
9357 /* We can expand signed mod by power of 2 using a NEGS, two parallel
9358 ANDs and a CSNEG. Assume here that CSNEG is the same as the cost of
9359 an unconditional negate. This case should only ever be reached through
9360 the set_smod_pow2_cheap check in expmed.c. */
9364 {
9365 /* We expand to 4 instructions. Reset the baseline. */
9373 }
9375 /* Fall-through. */
9378 {
9379 /* Slighly prefer UMOD over SMOD. */
9386 }
9393 {
9397 /* There is no integer SQRT, so only DIV and UDIV can get
9398 here. */
9400 /* Slighly prefer UDIV over SDIV. */
9402 else
9404 }
9430 {
9433 else
9435 }
9437 /* FMSUB, FNMADD, and FNMSUB are free. */
9444 /* aarch64_fnma4_elt_to_64v2df has the NEG as operand 1,
9445 and the by-element operand as operand 0. */
9449 /* Catch vector-by-element operations. The by-element operand can
9450 either be (vec_duplicate (vec_select (x))) or just
9451 (vec_select (x)), depending on whether we are multiplying by
9452 a vector or a scalar.
9454 Canonicalization is not very good in these cases, FMA4 will put the
9455 by-element operand as operand 0, FNMA4 will have it as operand 1. */
9466 /* If the remaining parameters are not registers,
9467 get the cost to put them into registers. */
9481 {
9483 {
9484 /*Vector truncate. */
9486 }
9487 else
9489 }
9494 {
9496 {
9497 /*Vector conversion. */
9499 }
9500 else
9502 }
9508 /* Strip the rounding part. They will all be implemented
9509 by the fcvt* family of instructions anyway. */
9511 {
9520 }
9523 {
9526 else
9528 }
9530 /* We can combine fmul by a power of 2 followed by a fcvt into a single
9531 fixed-point fcvt. */
9536 {
9540 }
9547 {
9548 /* ABS (vector). */
9551 }
9553 {
9556 /* FABD, which is analogous to FADD. */
9558 {
9565 }
9566 /* Simple FABS is analogous to FNEG. */
9569 }
9570 else
9571 {
9572 /* Integer ABS will either be split to
9573 two arithmetic instructions, or will be an ABS
9574 (scalar), which we don't model. */
9578 }
9584 {
9587 else
9588 {
9589 /* FMAXNM/FMINNM/FMAX/FMIN.
9590 TODO: This may not be accurate for all implementations, but
9591 we do not model this in the cost tables. */
9593 }
9594 }
9598 /* The floating point round to integer frint* instructions. */
9600 {
9605 }
9608 {
9613 }
9618 /* Decompose <su>muldi3_highpart. */
9620 mode == DImode
9621 /* (lshiftrt:TI */
9624 /* (mult:TI */
9626 /* (ANY_EXTEND:TI (reg:DI))
9627 (ANY_EXTEND:TI (reg:DI))) */
9634 /* (const_int 64) */
9637 {
9638 /* UMULH/SMULH. */
9646 }
9648 /* Fall through. */
9651 }
9659 }
9661 /* Wrapper around aarch64_rtx_costs, dumps the partial, or total cost
9662 calculated for X. This cost is stored in *COST. Returns true
9663 if the total cost of X was calculated. */
9664 static bool
9667 {
9672 {
9677 }
9680 }
9682 static int
9685 {
9691 /* Caller save and pointer regs are equivalent to GENERAL_REGS. */
9698 /* Moving between GPR and stack cost is the same as GP2GP. */
9703 /* To/From the stack register, we move via the gprs. */
9709 {
9710 /* 128-bit operations on general registers require 2 instructions. */
9718 /* When AdvSIMD instructions are disabled it is not possible to move
9719 a 128-bit value directly between Q registers. This is handled in
9720 secondary reload. A general register is used as a scratch to move
9721 the upper DI value and the lower DI value is moved directly,
9722 hence the cost is the sum of three moves. */
9727 }
9737 }
9739 static int
9741 reg_class_t rclass ATTRIBUTE_UNUSED,
9743 {
9745 }
9747 /* Return true if it is safe and beneficial to use the approximate rsqrt optabs
9748 to optimize 1.0/sqrt. */
9750 static bool
9752 {
9754 && flag_unsafe_math_optimizations
9758 }
9760 /* Function to decide when to use the approximate reciprocal square root
9761 builtin. */
9763 static tree
9765 {
9771 }
9773 /* Emit instruction sequence to compute either the approximate square root
9774 or its approximate reciprocal, depending on the flag RECP, and return
9775 whether the sequence was emitted or not. */
9777 bool
9779 {
9783 {
9786 }
9789 {
9796 || flag_trapping_math
9797 || !flag_unsafe_math_optimizations
9800 }
9801 else
9802 /* Caller assumes we cannot fail. */
9808 /* When calculating the approximate square root, compare the
9809 argument with 0.0 and create a mask. */
9815 /* Estimate the approximate reciprocal square root. */
9819 /* Iterate over the series twice for SF and thrice for DF. */
9822 /* Optionally iterate over the series once less for faster performance
9823 while sacrificing the accuracy. */
9826 iterations--;
9828 /* Iterate over the series to calculate the approximate reciprocal square
9829 root. */
9832 {
9840 }
9843 {
9844 /* Qualify the approximate reciprocal square root when the argument is
9845 0.0 by squashing the intermediary result to 0.0. */
9851 /* Calculate the approximate square root. */
9853 }
9855 /* Finalize the approximation. */
9859 }
9861 /* Emit the instruction sequence to compute the approximation for the division
9862 of NUM by DEN in QUO and return whether the sequence was emitted or not. */
9864 bool
9866 {
9877 || flag_trapping_math
9878 || !flag_unsafe_math_optimizations
9886 /* Estimate the approximate reciprocal. */
9890 /* Iterate over the series twice for SF and thrice for DF. */
9893 /* Optionally iterate over the series once less for faster performance,
9894 while sacrificing the accuracy. */
9896 iterations--;
9898 /* Iterate over the series to calculate the approximate reciprocal. */
9901 {
9906 }
9909 {
9910 /* As the approximate reciprocal of DEN is already calculated, only
9911 calculate the approximate division when NUM is not 1.0. */
9914 }
9916 /* Finalize the approximation. */
9919 }
9921 /* Return the number of instructions that can be issued per cycle. */
9922 static int
9924 {
9926 }
9928 static int
9930 {
9934 }
9937 /* Implement TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD as
9938 autopref_multipass_dfa_lookahead_guard from haifa-sched.c. It only
9939 has an effect if PARAM_SCHED_AUTOPREF_QUEUE_DEPTH > 0. */
9941 static int
9944 {
9946 }
9949 /* Vectorizer cost model target hooks. */
9951 /* Implement targetm.vectorize.builtin_vectorization_cost. */
9952 static int
9954 tree vectype,
9956 {
9965 {
10016 }
10017 }
10019 /* Implement targetm.vectorize.add_stmt_cost. */
10020 static unsigned
10024 {
10029 {
10034 /* Statements in an inner loop relative to the loop being
10035 vectorized are weighted more heavily. The value here is
10036 arbitrary and could potentially be improved with analysis. */
10042 }
10045 }
10049 /* Parse the TO_PARSE string and put the architecture struct that it
10050 selects into RES and the architectural features into ISA_FLAGS.
10051 Return an aarch64_parse_opt_result describing the parse result.
10052 If there is an error parsing, RES and ISA_FLAGS are left unchanged. */
10054 static enum aarch64_parse_opt_result
10057 {
10069 else
10076 /* Loop through the list of supported ARCHes to find a match. */
10078 {
10080 {
10084 {
10085 /* TO_PARSE string contains at least one extension. */
10086 enum aarch64_parse_opt_result ext_res
10091 }
10092 /* Extension parsing was successful. Confirm the result
10093 arch and ISA flags. */
10097 }
10098 }
10100 /* ARCH name not found in list. */
10102 }
10104 /* Parse the TO_PARSE string and put the result tuning in RES and the
10105 architecture flags in ISA_FLAGS. Return an aarch64_parse_opt_result
10106 describing the parse result. If there is an error parsing, RES and
10107 ISA_FLAGS are left unchanged. */
10109 static enum aarch64_parse_opt_result
10112 {
10124 else
10131 /* Loop through the list of supported CPUs to find a match. */
10133 {
10135 {
10140 {
10141 /* TO_PARSE string contains at least one extension. */
10142 enum aarch64_parse_opt_result ext_res
10147 }
10148 /* Extension parsing was successfull. Confirm the result
10149 cpu and ISA flags. */
10153 }
10154 }
10156 /* CPU name not found in list. */
10158 }
10160 /* Parse the TO_PARSE string and put the cpu it selects into RES.
10161 Return an aarch64_parse_opt_result describing the parse result.
10162 If the parsing fails the RES does not change. */
10164 static enum aarch64_parse_opt_result
10166 {
10172 /* Loop through the list of supported CPUs to find a match. */
10174 {
10176 {
10179 }
10180 }
10182 /* CPU name not found in list. */
10184 }
10186 /* Parse TOKEN, which has length LENGTH to see if it is an option
10187 described in FLAG. If it is, return the index bit for that fusion type.
10188 If not, error (printing OPTION_NAME) and return zero. */
10190 static unsigned int
10195 {
10197 {
10201 }
10205 }
10207 /* Parse OPTION which is a comma-separated list of flags to enable.
10208 FLAGS gives the list of flags we understand, INITIAL_STATE gives any
10209 default state we inherit from the CPU tuning structures. OPTION_NAME
10210 gives the top-level option we are parsing in the -moverride string,
10211 for use in error messages. */
10213 static unsigned int
10218 {
10225 {
10228 token_length,
10229 flags,
10230 option_name);
10231 /* If we find "none" (or, for simplicity's sake, an error) anywhere
10232 in the token stream, reset the supported operations. So:
10234 adrp+add.cmp+branch.none.adrp+add
10236 would have the result of turning on only adrp+add fusion. */
10242 }
10244 /* We ended with a comma, print something. */
10246 {
10249 }
10251 /* We still have one more token to parse. */
10254 token_length,
10255 flags,
10256 option_name);
10262 }
10264 /* Support for overriding instruction fusion. */
10266 static void
10269 {
10271 aarch64_fusible_pairs,
10274 }
10276 /* Support for overriding other tuning flags. */
10278 static void
10281 {
10282 tune->extra_tuning_flags
10284 aarch64_tuning_flags,
10287 }
10289 /* Parse TOKEN, which has length LENGTH to see if it is a tuning option
10290 we understand. If it is, extract the option string and handoff to
10291 the appropriate function. */
10293 void
10297 {
10303 {
10306 }
10308 /* Get the length of the option name. */
10310 /* Skip the '=' to get to the option string. */
10311 option_part++;
10314 {
10316 {
10319 }
10320 }
10324 }
10326 /* A checking mechanism for the implementation of the tls size. */
10328 static void
10330 {
10335 {
10337 /* Both the default and maximum TLS size allowed under tiny is 1M which
10338 needs two instructions to address, so we clamp the size to 24. */
10343 /* The maximum TLS size allowed under small is 4G. */
10348 /* The maximum TLS size allowed under large is 16E.
10349 FIXME: 16E should be 64bit, we only support 48bit offset now. */
10355 }
10358 }
10360 /* Parse STRING looking for options in the format:
10361 string :: option:string
10362 option :: name=substring
10363 name :: {a-z}
10364 substring :: defined by option. */
10366 static void
10369 {
10380 {
10382 /* Make this substring look like a string. */
10386 }
10388 /* One last option to parse. */
10391 }
10394 static void
10396 {
10397 /* PR 70044: We have to be careful about being called multiple times for the
10398 same function. This means all changes should be repeatable. */
10400 /* Set aarch64_use_frame_pointer based on -fno-omit-frame-pointer.
10401 Disable the frame pointer flag so the mid-end will not use a frame
10402 pointer in leaf functions in order to support -fomit-leaf-frame-pointer.
10403 Set x_flag_omit_frame_pointer to the special value 2 to differentiate
10404 between -fomit-frame-pointer (1) and -fno-omit-frame-pointer (2). */
10409 /* If not optimizing for size, set the default
10410 alignment to what the target wants. */
10412 {
10419 }
10421 /* We default to no pc-relative literal loads. */
10425 /* If -mpc-relative-literal-loads is set on the command line, this
10426 implies that the user asked for PC relative literal loads. */
10430 /* In the tiny memory model it makes no sense to disallow PC relative
10431 literal pool loads. */
10436 /* When enabling the lower precision Newton series for the square root, also
10437 enable it for the reciprocal square root, since the latter is an
10438 intermediary step for the former. */
10441 }
10443 /* 'Unpack' up the internal tuning structs and update the options
10444 in OPTS. The caller must have set up selected_tune and selected_arch
10445 as all the other target-specific codegen decisions are
10446 derived from them. */
10448 void
10450 {
10453 /* Make a copy of the tuning parameters attached to the core, which
10454 we may later overwrite. */
10462 /* This target defaults to strict volatile bitfields. */
10471 {
10483 }
10485 /* We don't mind passing in global_options_set here as we don't use
10486 the *options_set structs anyway. */
10488 queue_depth,
10492 /* Set up parameters to be used in prefetching algorithm. Do not
10493 override the defaults unless we are tuning for a core we have
10494 researched values for. */
10526 /* Use the alternative scheduling-pressure algorithm by default. */
10531 /* Enable sw prefetching at specified optimization level for
10532 CPUS that have prefetch. Lower optimization level threshold by 1
10533 when profiling is enabled. */
10541 }
10543 /* Print a hint with a suggestion for a core or architecture name that
10544 most closely resembles what the user passed in STR. ARCH is true if
10545 the user is asking for an architecture name. ARCH is false if the user
10546 is asking for a core name. */
10548 static void
10550 {
10556 #ifdef HAVE_LOCAL_CPU_DETECT
10557 /* Add also "native" as possible value. */
10560 #endif
10567 else
10571 }
10573 /* Print a hint with a suggestion for a core name that most closely resembles
10574 what the user passed in STR. */
10578 {
10580 }
10582 /* Print a hint with a suggestion for an architecture name that most closely
10583 resembles what the user passed in STR. */
10587 {
10589 }
10591 /* Validate a command-line -mcpu option. Parse the cpu and extensions (if any)
10592 specified in STR and throw errors if appropriate. Put the results if
10593 they are valid in RES and ISA_FLAGS. Return whether the option is
10594 valid. */
10596 static bool
10599 {
10600 enum aarch64_parse_opt_result parse_res
10607 {
10620 }
10623 }
10625 /* Validate a command-line -march option. Parse the arch and extensions
10626 (if any) specified in STR and throw errors if appropriate. Put the
10627 results, if they are valid, in RES and ISA_FLAGS. Return whether the
10628 option is valid. */
10630 static bool
10633 {
10634 enum aarch64_parse_opt_result parse_res
10641 {
10654 }
10657 }
10659 /* Validate a command-line -mtune option. Parse the cpu
10660 specified in STR and throw errors if appropriate. Put the
10661 result, if it is valid, in RES. Return whether the option is
10662 valid. */
10664 static bool
10666 {
10667 enum aarch64_parse_opt_result parse_res
10674 {
10684 }
10686 }
10688 /* Return the CPU corresponding to the enum CPU.
10689 If it doesn't specify a cpu, return the default. */
10693 {
10697 /* The & 0x3f is to extract the bottom 6 bits that encode the
10698 default cpu as selected by the --with-cpu GCC configure option
10699 in config.gcc.
10700 ???: The whole TARGET_CPU_DEFAULT and AARCH64_CPU_DEFAULT_FLAGS
10701 flags mechanism should be reworked to make it more sane. */
10703 }
10705 /* Return the architecture corresponding to the enum ARCH.
10706 If it doesn't specify a valid architecture, return the default. */
10710 {
10717 }
10719 /* Return the VG value associated with -msve-vector-bits= value VALUE. */
10721 static poly_uint16
10723 {
10724 /* For now generate vector-length agnostic code for -msve-vector-bits=128.
10725 This ensures we can clearly distinguish SVE and Advanced SIMD modes when
10726 deciding which .md file patterns to use and when deciding whether
10727 something is a legitimate address or constant. */
10730 else
10732 }
10734 /* Implement TARGET_OPTION_OVERRIDE. This is called once in the beginning
10735 and is used to parse the -m{cpu,tune,arch} strings and setup the initial
10736 tuning structs. In particular it must set selected_tune and
10737 aarch64_isa_flags that define the available ISA features and tuning
10738 decisions. It must also set selected_arch as this will be used to
10739 output the .arch asm tags for each function. */
10741 static void
10743 {
10756 /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
10757 If either of -march or -mtune is given, they override their
10758 respective component of -mcpu. */
10770 /* If the user did not specify a processor, choose the default
10771 one for them. This will be the CPU set during configuration using
10772 --with-cpu, otherwise it is "generic". */
10774 {
10776 {
10780 }
10781 else
10782 {
10783 /* Get default configure-time CPU. */
10786 }
10790 }
10791 /* If both -mcpu and -march are specified check that they are architecturally
10792 compatible, warn if they're not and prefer the -march ISA flags. */
10794 {
10796 {
10800 }
10805 }
10806 else
10807 {
10808 /* -mcpu but no -march. */
10815 }
10817 /* Set the arch as well as we will need it when outputing
10818 the .arch directive in assembly. */
10820 {
10823 }
10828 #ifndef HAVE_AS_MABI_OPTION
10829 /* The compiler may have been configured with 2.23.* binutils, which does
10830 not have support for ILP32. */
10833 #endif
10835 /* Convert -msve-vector-bits to a VG count. */
10841 /* Make sure we properly set up the explicit options. */
10852 /* Save these options as the default ones in case we push and pop them later
10853 while processing functions with potential target attributes. */
10854 target_option_default_node = target_option_current_node
10856 }
10858 /* Implement targetm.override_options_after_change. */
10860 static void
10862 {
10864 }
10868 {
10872 }
10874 void
10876 {
10878 }
10880 /* A checking mechanism for the implementation of the various code models. */
10881 static void
10883 {
10885 {
10887 {
10892 #ifdef HAVE_AS_SMALL_PIC_RELOCS
10894 ? AARCH64_CMODEL_SMALL_PIC
10896 #else
10898 #endif
10906 }
10907 }
10908 else
10910 }
10912 /* Implement TARGET_OPTION_SAVE. */
10914 static void
10916 {
10918 }
10920 /* Implements TARGET_OPTION_RESTORE. Restore the backend codegen decisions
10921 using the information saved in PTR. */
10923 static void
10925 {
10933 }
10935 /* Implement TARGET_OPTION_PRINT. */
10937 static void
10939 {
10950 }
10954 void
10956 {
10958 }
10960 /* Restore or save the TREE_TARGET_GLOBALS from or to NEW_TREE.
10961 Used by aarch64_set_current_function and aarch64_pragma_target_parse to
10962 make sure optab availability predicates are recomputed when necessary. */
10964 void
10966 {
10971 else
10973 }
10975 /* Implement TARGET_SET_CURRENT_FUNCTION. Unpack the codegen decisions
10976 like tuning and ISA features from the DECL_FUNCTION_SPECIFIC_TARGET
10977 of the function, if such exists. This function may be called multiple
10978 times on a single function so use aarch64_previous_fndecl to avoid
10979 setting up identical state. */
10981 static void
10983 {
10987 tree old_tree = (aarch64_previous_fndecl
10993 /* If current function has no attributes but the previous one did,
10994 use the default node. */
10998 /* If nothing to do, return. #pragma GCC reset or #pragma GCC pop to
10999 the default have been handled by aarch64_save_restore_target_globals from
11000 aarch64_pragma_target_parse. */
11006 /* First set the target options. */
11010 }
11012 /* Enum describing the various ways we can handle attributes.
11013 In many cases we can reuse the generic option handling machinery. */
11015 enum aarch64_attr_opt_type
11016 {
11020 aarch64_attr_custom /* Attribute requires a custom handling function. */
11021 };
11023 /* All the information needed to handle a target attribute.
11024 NAME is the name of the attribute.
11025 ATTR_TYPE specifies the type of behavior of the attribute as described
11026 in the definition of enum aarch64_attr_opt_type.
11027 ALLOW_NEG is true if the attribute supports a "no-" form.
11028 HANDLER is the function that takes the attribute string as an argument
11029 It is needed only when the ATTR_TYPE is aarch64_attr_custom.
11030 OPT_NUM is the enum specifying the option that the attribute modifies.
11031 This is needed for attributes that mirror the behavior of a command-line
11032 option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
11033 aarch64_attr_enum. */
11035 struct aarch64_attribute_info
11036 {
11042 };
11044 /* Handle the ARCH_STR argument to the arch= target attribute. */
11046 static bool
11048 {
11050 enum aarch64_parse_opt_result parse_res
11054 {
11059 }
11062 {
11075 }
11078 }
11080 /* Handle the argument CPU_STR to the cpu= target attribute. */
11082 static bool
11084 {
11086 enum aarch64_parse_opt_result parse_res
11090 {
11098 }
11101 {
11114 }
11117 }
11119 /* Handle the argument STR to the tune= target attribute. */
11121 static bool
11123 {
11125 enum aarch64_parse_opt_result parse_res
11129 {
11134 }
11137 {
11144 }
11147 }
11149 /* Parse an architecture extensions target attribute string specified in STR.
11150 For example "+fp+nosimd". Show any errors if needed. Return TRUE
11151 if successful. Update aarch64_isa_flags to reflect the ISA features
11152 modified. */
11154 static bool
11156 {
11160 /* We allow "+nothing" in the beginning to clear out all architectural
11161 features if the user wants to handpick specific features. */
11163 {
11166 }
11171 {
11174 }
11177 {
11188 }
11191 }
11193 /* The target attributes that we support. On top of these we also support just
11194 ISA extensions, like __attribute__ ((target ("+crc"))), but that case is
11195 handled explicitly in aarch64_process_one_target_attr. */
11198 {
11200 OPT_mgeneral_regs_only },
11202 OPT_mfix_cortex_a53_835769 },
11204 OPT_mfix_cortex_a53_843419 },
11208 OPT_momit_leaf_frame_pointer },
11211 OPT_march_ },
11214 OPT_mtune_ },
11216 OPT_msign_return_address_ },
11218 };
11220 /* Parse ARG_STR which contains the definition of one target attribute.
11221 Show appropriate errors if any or return true if the attribute is valid. */
11223 static bool
11225 {
11231 {
11234 }
11239 /* Skip leading whitespace. */
11241 str_to_check++;
11243 /* We have something like __attribute__ ((target ("+fp+nosimd"))).
11244 It is easier to detect and handle it explicitly here rather than going
11245 through the machinery for the rest of the target attributes in this
11246 function. */
11251 {
11254 }
11257 /* If we found opt=foo then terminate STR_TO_CHECK at the '='
11258 and point ARG to "foo". */
11260 {
11262 arg++;
11263 }
11267 {
11268 /* If the names don't match up, or the user has given an argument
11269 to an attribute that doesn't accept one, or didn't give an argument
11270 to an attribute that expects one, fail to match. */
11279 {
11282 }
11284 /* If the name matches but the attribute does not allow "no-" versions
11285 then we can't match. */
11287 {
11290 }
11293 {
11294 /* Has a custom handler registered.
11295 For example, cpu=, arch=, tune=. */
11302 /* Either set or unset a boolean option. */
11304 {
11312 }
11313 /* Set or unset a bit in the target_flags. aarch64_handle_option
11314 should know what mask to apply given the option number. */
11316 {
11318 /* We only need to specify the option number.
11319 aarch64_handle_option will know which mask to apply. */
11325 }
11326 /* Use the option setting machinery to set an option to an enum. */
11328 {
11335 {
11338 global_dc);
11339 }
11340 else
11341 {
11343 }
11345 }
11348 }
11349 }
11351 /* If we reached here we either have found an attribute and validated
11352 it or didn't match any. If we matched an attribute but its arguments
11353 were malformed we will have returned false already. */
11355 }
11357 /* Count how many times the character C appears in
11358 NULL-terminated string STR. */
11360 static unsigned int
11362 {
11365 {
11367 res++;
11369 str++;
11370 }
11373 }
11375 /* Parse the tree in ARGS that contains the target attribute information
11376 and update the global target options space. */
11378 bool
11380 {
11382 {
11383 do
11384 {
11387 {
11390 }
11395 }
11398 {
11401 }
11408 {
11411 }
11413 /* Used to catch empty spaces between commas i.e.
11414 attribute ((target ("attr1,,attr2"))). */
11417 /* Handle multiple target attributes separated by ','. */
11422 {
11423 num_attrs++;
11425 {
11428 }
11431 }
11434 {
11437 }
11440 }
11442 /* Implement TARGET_OPTION_VALID_ATTRIBUTE_P. This is used to
11443 process attribute ((target ("..."))). */
11445 static bool
11447 {
11450 tree old_optimize;
11454 /* If what we're processing is the current pragma string then the
11455 target option node is already stored in target_option_current_node
11456 by aarch64_pragma_target_parse in aarch64-c.c. Use that to avoid
11457 having to re-parse the string. This is especially useful to keep
11458 arm_neon.h compile times down since that header contains a lot
11459 of intrinsics enclosed in pragmas. */
11461 {
11464 }
11470 /* If the function changed the optimization levels as well as setting
11471 target options, start with the optimizations specified. */
11476 /* Save the current target options to restore at the end. */
11479 /* If fndecl already has some target attributes applied to it, unpack
11480 them so that we add this attribute on top of them, rather than
11481 overwriting them. */
11483 {
11489 }
11490 else
11496 /* Set up any additional state. */
11498 {
11500 /* Initialize SIMD builtins if we haven't already.
11501 Set current_target_pragma to NULL for the duration so that
11502 the builtin initialization code doesn't try to tag the functions
11503 being built with the attributes specified by any current pragma, thus
11504 going into an infinite recursion. */
11506 {
11511 }
11513 }
11514 else
11520 {
11525 }
11533 }
11535 /* Helper for aarch64_can_inline_p. In the case where CALLER and CALLEE are
11536 tri-bool options (yes, no, don't care) and the default value is
11537 DEF, determine whether to reject inlining. */
11539 static bool
11542 {
11543 /* If the callee doesn't care, always allow inlining. */
11547 /* If the caller doesn't care, always allow inlining. */
11551 /* Otherwise, allow inlining if either the callee and caller values
11552 agree, or if the callee is using the default value. */
11554 }
11556 /* Implement TARGET_CAN_INLINE_P. Decide whether it is valid
11557 to inline CALLEE into CALLER based on target-specific info.
11558 Make sure that the caller and callee have compatible architectural
11559 features. Then go through the other possible target attributes
11560 and see if they can block inlining. Try not to reject always_inline
11561 callees unless they are incompatible architecturally. */
11563 static bool
11565 {
11577 /* Callee's ISA flags should be a subset of the caller's. */
11582 /* Allow non-strict aligned functions inlining into strict
11583 aligned ones. */
11593 /* If the architectural features match up and the callee is always_inline
11594 then the other attributes don't matter. */
11606 /* Honour explicit requests to workaround errata. */
11619 /* If the user explicitly specified -momit-leaf-frame-pointer for the
11620 caller and calle and they don't match up, reject inlining. */
11627 /* If the callee has specific tuning overrides, respect them. */
11632 /* If the user specified tuning override strings for the
11633 caller and callee and they don't match up, reject inlining.
11634 We just do a string compare here, we don't analyze the meaning
11635 of the string, as it would be too costly for little gain. */
11643 }
11645 /* Return true if SYMBOL_REF X binds locally. */
11647 static bool
11649 {
11653 }
11655 /* Return true if SYMBOL_REF X is thread local */
11656 static bool
11658 {
11666 }
11668 /* Classify a TLS symbol into one of the TLS kinds. */
11669 enum aarch64_symbol_type
11671 {
11675 {
11682 {
11688 }
11699 else
11708 }
11709 }
11711 /* Return the correct method for accessing X + OFFSET, where X is either
11712 a SYMBOL_REF or LABEL_REF. */
11714 enum aarch64_symbol_type
11716 {
11718 {
11720 {
11735 }
11736 }
11739 {
11744 {
11746 /* When we retrieve symbol + offset address, we have to make sure
11747 the offset does not cause overflow of the final address. But
11748 we have no way of knowing the address of symbol at compile time
11749 so we can't accurately say if the distance between the PC and
11750 symbol + offset is outside the addressible range of +/-1M in the
11751 TINY code model. So we rely on images not being greater than
11752 1M and cap the offset at 1M and anything beyond 1M will have to
11753 be loaded using an alternative mechanism. Furthermore if the
11754 symbol is a weak reference to something that isn't known to
11755 resolve to a symbol in this module, then force to memory. */
11763 /* Same reasoning as the tiny code model, but the offset cap here is
11764 4G. */
11785 /* This is alright even in PIC code as the constant
11786 pool reference is always PC relative and within
11787 the same translation unit. */
11790 else
11795 }
11796 }
11798 /* By default push everything into the constant pool. */
11800 }
11802 bool
11804 {
11806 }
11808 bool
11810 {
11818 }
11820 /* Implement TARGET_LEGITIMATE_CONSTANT_P hook. Return true for constants
11821 that should be rematerialized rather than spilled. */
11823 static bool
11825 {
11826 /* Support CSE and rematerialization of common constants. */
11832 /* Do not allow vector struct mode constants for Advanced SIMD.
11833 We could support 0 and -1 easily, but they need support in
11834 aarch64-simd.md. */
11839 /* Only accept variable-length vector constants if they can be
11840 handled directly.
11842 ??? It would be possible to handle rematerialization of other
11843 constants via secondary reloads. */
11850 /* Accept polynomial constants that can be calculated by using the
11851 destination of a move as the sole temporary. Constants that
11852 require a second temporary cannot be rematerialized (they can't be
11853 forced to memory and also aren't legitimate constants). */
11854 poly_int64 offset;
11858 /* If an offset is being added to something else, we need to allow the
11859 base to be moved into the destination register, meaning that there
11860 are no free temporaries for the offset. */
11865 /* Do not allow const (plus (anchor_symbol, const_int)). */
11869 /* Treat symbols as constants. Avoid TLS symbols as they are complex,
11870 so spilling them is better than rematerialization. */
11874 /* Label references are always constant. */
11879 }
11881 rtx
11883 {
11889 /* Can return in any reg. */
11892 }
11894 /* On AAPCS systems, this is the "struct __va_list". */
11897 /* Implement TARGET_BUILD_BUILTIN_VA_LIST.
11898 Return the type to use as __builtin_va_list.
11900 AAPCS64 \S 7.1.4 requires that va_list be a typedef for a type defined as:
11902 struct __va_list
11903 {
11904 void *__stack;
11905 void *__gr_top;
11906 void *__vr_top;
11907 int __gr_offs;
11908 int __vr_offs;
11909 }; */
11911 static tree
11913 {
11914 tree va_list_name;
11917 /* Create the type. */
11919 /* Give it the required name. */
11921 TYPE_DECL,
11923 va_list_type);
11928 /* Create the fields. */
11931 ptr_type_node);
11934 ptr_type_node);
11937 ptr_type_node);
11940 integer_type_node);
11943 integer_type_node);
11945 /* Tell tree-stdarg pass about our internal offset fields.
11946 NOTE: va_list_gpr/fpr_counter_field are only used for tree comparision
11947 purpose to identify whether the code is updating va_list internal
11948 offset fields through irregular way. */
11970 /* Compute its layout. */
11974 }
11976 /* Implement TARGET_EXPAND_BUILTIN_VA_START. */
11977 static void
11979 {
11983 tree t;
11997 {
12000 }
12009 NULL_TREE);
12011 NULL_TREE);
12013 NULL_TREE);
12015 NULL_TREE);
12017 NULL_TREE);
12019 /* Emit code to initialize STACK, which points to the next varargs stack
12020 argument. CUM->AAPCS_STACK_SIZE gives the number of stack words used
12021 by named arguments. STACK is 8-byte aligned. */
12028 /* Emit code to initialize GRTOP, the top of the GR save area.
12029 virtual_incoming_args_rtx should have been 16 byte aligned. */
12034 /* Emit code to initialize VRTOP, the top of the VR save area.
12035 This address is gr_save_area_bytes below GRTOP, rounded
12036 down to the next 16-byte boundary. */
12046 /* Emit code to initialize GROFF, the offset from GRTOP of the
12047 next GPR argument. */
12052 /* Likewise emit code to initialize VROFF, the offset from FTOP
12053 of the next VR argument. */
12057 }
12059 /* Implement TARGET_GIMPLIFY_VA_ARG_EXPR. */
12061 static tree
12064 {
12065 tree addr;
12071 machine_mode mode;
12098 type,
12102 {
12103 /* No frontends can create types with variable-sized modes, so we
12104 shouldn't be asked to pass or return them. */
12107 /* TYPE passed in fp/simd registers. */
12119 {
12122 }
12125 {
12127 }
12128 }
12129 else
12130 {
12131 /* TYPE passed in general registers. */
12144 {
12146 }
12147 }
12149 /* Get a local temporary for the field value. */
12152 /* Emit code to branch if off >= 0. */
12158 {
12159 /* Emit: offs = (offs + 15) & -16. */
12165 }
12166 else
12169 /* Update ap.__[g|v]r_offs */
12174 /* String up. */
12178 /* [cond2] if (ap.__[g|v]r_offs > 0) */
12183 /* String up: make sure the assignment happens before the use. */
12187 /* Prepare the trees handling the argument that is passed on the stack;
12188 the top level node will store in ON_STACK. */
12191 {
12192 /* if (alignof(type) > 8) (arg = arg + 15) & -16; */
12197 }
12198 else
12200 /* Advance ap.__stack */
12205 /* String up roundup and advance. */
12208 /* String up with arg */
12210 /* Big-endianness related address adjustment. */
12213 {
12217 }
12222 /* Adjustment to OFFSET in the case of BIG_ENDIAN. */
12232 {
12233 /* type ha; // treat as "struct {ftype field[n];}"
12234 ... [computing offs]
12235 for (i = 0; i <nregs; ++i, offs += 16)
12236 ha.field[i] = *((ftype *)(ap.__vr_top + offs));
12237 return ha; */
12241 /* Declare a local variable. */
12245 /* Establish the base type. */
12247 {
12266 {
12270 }
12274 }
12276 /* *(field_ptr_t)&ha = *((field_ptr_t)vr_saved_area */
12284 /* ha.field[i] = *((field_ptr_t)vr_saved_area + i) */
12286 {
12296 }
12300 }
12310 }
12312 /* Implement TARGET_SETUP_INCOMING_VARARGS. */
12314 static void
12318 {
12320 CUMULATIVE_ARGS local_cum;
12324 /* The caller has advanced CUM up to, but not beyond, the last named
12325 argument. Advance a local copy of CUM past the last "real" named
12326 argument, to find out how many registers are left over. */
12330 /* Found out how many registers we need to save.
12331 Honor tree-stdvar analysis results. */
12340 {
12343 }
12346 {
12348 {
12351 /* virtual_incoming_args_rtx should have been 16-byte aligned. */
12359 }
12361 {
12362 /* We can't use move_block_from_reg, because it will use
12363 the wrong mode, storing D regs only. */
12367 /* Set OFF to the offset from virtual_incoming_args_rtx of
12368 the first vector register. The VR save area lies below
12369 the GR one, and is aligned to 16 bytes. */
12376 {
12384 }
12385 }
12386 }
12388 /* We don't save the size into *PRETEND_SIZE because we want to avoid
12389 any complication of having crtl->args.pretend_args_size changed. */
12394 }
12396 static void
12398 {
12401 {
12403 {
12406 }
12407 }
12410 {
12413 }
12415 /* When tracking speculation, we need a couple of call-clobbered registers
12416 to track the speculation state. It would be nice to just use
12417 IP0 and IP1, but currently there are numerous places that just
12418 assume these registers are free for other uses (eg pointer
12419 authentication). */
12421 {
12426 }
12427 }
12429 /* Walk down the type tree of TYPE counting consecutive base elements.
12430 If *MODEP is VOIDmode, then set it to the first valid floating point
12431 type. If a non-floating point type is found, or if a floating point
12432 type that doesn't match a non-VOIDmode *MODEP is found, then return -1,
12433 otherwise return the count in the sub-tree. */
12434 static int
12436 {
12437 machine_mode mode;
12438 HOST_WIDE_INT size;
12441 {
12471 /* Use V2SImode and V4SImode as representatives of all 64-bit
12472 and 128-bit vector types. */
12475 {
12484 }
12489 /* Vector modes are considered to be opaque: two vectors are
12490 equivalent for the purposes of being homogeneous aggregates
12491 if they are the same size. */
12498 {
12502 /* Can't handle incomplete types nor sizes that are not
12503 fixed. */
12510 || !index
12521 /* There must be no padding. */
12527 }
12530 {
12533 tree field;
12535 /* Can't handle incomplete types nor sizes that are not
12536 fixed. */
12542 {
12550 }
12552 /* There must be no padding. */
12558 }
12562 {
12563 /* These aren't very interesting except in a degenerate case. */
12566 tree field;
12568 /* Can't handle incomplete types nor sizes that are not
12569 fixed. */
12575 {
12583 }
12585 /* There must be no padding. */
12591 }
12595 }
12598 }
12600 /* Return TRUE if the type, as described by TYPE and MODE, is a short vector
12601 type as described in AAPCS64 \S 4.1.2.
12603 See the comment above aarch64_composite_type_p for the notes on MODE. */
12605 static bool
12607 machine_mode mode)
12608 {
12618 }
12620 /* Return TRUE if the type, as described by TYPE and MODE, is a composite
12621 type as described in AAPCS64 \S 4.3. This includes aggregate, union and
12622 array types. The C99 floating-point complex types are also considered
12623 as composite types, according to AAPCS64 \S 7.1.1. The complex integer
12624 types, which are GCC extensions and out of the scope of AAPCS64, are
12625 treated as composite types here as well.
12627 Note that MODE itself is not sufficient in determining whether a type
12628 is such a composite type or not. This is because
12629 stor-layout.c:compute_record_mode may have already changed the MODE
12630 (BLKmode) of a RECORD_TYPE TYPE to some other mode. For example, a
12631 structure with only one field may have its MODE set to the mode of the
12632 field. Also an integer mode whose size matches the size of the
12633 RECORD_TYPE type may be used to substitute the original mode
12634 (i.e. BLKmode) in certain circumstances. In other words, MODE cannot be
12635 solely relied on. */
12637 static bool
12639 machine_mode mode)
12640 {
12653 }
12655 /* Return TRUE if an argument, whose type is described by TYPE and MODE,
12656 shall be passed or returned in simd/fp register(s) (providing these
12657 parameter passing registers are available).
12659 Upon successful return, *COUNT returns the number of needed registers,
12660 *BASE_MODE returns the mode of the individual register and when IS_HAF
12661 is not NULL, *IS_HA indicates whether or not the argument is a homogeneous
12662 floating-point aggregate or a homogeneous short-vector aggregate. */
12664 static bool
12666 const_tree type,
12670 {
12678 {
12681 }
12683 {
12687 }
12689 {
12693 {
12696 }
12697 else
12699 }
12700 else
12705 }
12707 /* Implement TARGET_STRUCT_VALUE_RTX. */
12709 static rtx
12712 {
12714 }
12716 /* Implements target hook vector_mode_supported_p. */
12717 static bool
12719 {
12722 }
12724 /* Return appropriate SIMD container
12725 for MODE within a vector of WIDTH bits. */
12726 static machine_mode
12728 {
12731 {
12748 }
12752 {
12755 {
12772 }
12773 else
12775 {
12788 }
12789 }
12791 }
12793 /* Return 128-bit container as the preferred SIMD mode for MODE. */
12794 static machine_mode
12796 {
12799 }
12801 /* Return a list of possible vector sizes for the vectorizer
12802 to iterate over. */
12803 static void
12805 {
12810 }
12812 /* Implement TARGET_MANGLE_TYPE. */
12816 {
12817 /* The AArch64 ABI documents say that "__va_list" has to be
12818 managled as if it is in the "std" namespace. */
12822 /* Half-precision float. */
12826 /* Mangle AArch64-specific internal types. TYPE_NAME is non-NULL_TREE for
12827 builtin types. */
12831 /* Use the default mangling. */
12833 }
12835 /* Find the first rtx_insn before insn that will generate an assembly
12836 instruction. */
12840 {
12844 do
12845 {
12847 }
12851 }
12853 static bool
12855 {
12857 /* A number of these may be AArch32 only. */
12862 };
12865 {
12868 }
12871 }
12873 /* Check if there is a register dependency between a load and the insn
12874 for which we hold recog_data. */
12876 static bool
12878 {
12879 rtx load_reg;
12889 {
12895 }
12897 }
12900 /* When working around the Cortex-A53 erratum 835769,
12901 given rtx_insn INSN, return true if it is a 64-bit multiply-accumulate
12902 instruction and has a preceding memory instruction such that a NOP
12903 should be inserted between them. */
12905 bool
12907 {
12910 rtx body;
12923 /* aarch64_prev_real_insn can call recog_memoized on insns other than INSN.
12924 Restore recog state to INSN to avoid state corruption. */
12932 /* If the previous insn is a memory op and there is no dependency between
12933 it and the DImode madd, emit a NOP between them. If body is NULL then we
12934 have a complex memory operation, probably a load/store pair.
12935 Be conservative for now and emit a NOP. */
12942 }
12945 /* Implement FINAL_PRESCAN_INSN. */
12947 void
12949 {
12952 }
12955 /* Return true if BASE_OR_STEP is a valid immediate operand for an SVE INDEX
12956 instruction. */
12958 bool
12960 {
12963 }
12965 /* Return true if X is a valid immediate for the SVE ADD and SUB
12966 instructions. Negate X first if NEGATE_P is true. */
12968 bool
12970 {
12971 rtx elt;
12985 }
12987 /* Return true if X is a valid immediate operand for an SVE logical
12988 instruction such as AND. */
12990 bool
12992 {
12993 rtx elt;
12999 }
13001 /* Return true if X is a valid immediate for the SVE DUP and CPY
13002 instructions. */
13004 bool
13006 {
13007 rtx elt;
13017 }
13019 /* Return true if X is a valid immediate operand for an SVE CMP instruction.
13020 SIGNED_P says whether the operand is signed rather than unsigned. */
13022 bool
13024 {
13025 rtx elt;
13029 && (signed_p
13032 }
13034 /* Return true if X is a valid immediate operand for an SVE FADD or FSUB
13035 instruction. Negate X first if NEGATE_P is true. */
13037 bool
13039 {
13040 rtx elt;
13041 REAL_VALUE_TYPE r;
13057 }
13059 /* Return true if X is a valid immediate operand for an SVE FMUL
13060 instruction. */
13062 bool
13064 {
13065 rtx elt;
13067 /* GCC will never generate a multiply with an immediate of 2, so there is no
13068 point testing for it (even though it is a valid constant). */
13072 }
13074 /* Return true if replicating VAL32 is a valid 2-byte or 4-byte immediate
13075 for the Advanced SIMD operation described by WHICH and INSN. If INFO
13076 is nonnull, use it to describe valid immediates. */
13077 static bool
13082 {
13083 /* Try a 4-byte immediate with LSL. */
13086 {
13091 }
13093 /* Try a 2-byte immediate with LSL. */
13098 {
13103 }
13105 /* Try a 4-byte immediate with MSL, except for cases that MVN
13106 can handle. */
13109 {
13112 {
13117 }
13118 }
13121 }
13123 /* Return true if replicating VAL64 is a valid immediate for the
13124 Advanced SIMD operation described by WHICH. If INFO is nonnull,
13125 use it to describe valid immediates. */
13126 static bool
13130 {
13136 {
13147 /* Try using a replicated byte. */
13151 {
13155 }
13156 }
13158 /* Try using a bit-to-bytemask. */
13160 {
13163 {
13167 }
13169 {
13173 }
13174 }
13176 }
13178 /* Return true if replicating VAL64 gives a valid immediate for an SVE MOV
13179 instruction. If INFO is nonnull, use it to describe valid immediates. */
13181 static bool
13184 {
13188 {
13192 {
13197 }
13198 }
13201 {
13202 /* DUP with no shift. */
13206 }
13208 {
13209 /* DUP with LSL #8. */
13213 }
13215 {
13216 /* DUPM. */
13220 }
13222 }
13224 /* Return true if OP is a valid SIMD immediate for the operation
13225 described by WHICH. If INFO is nonnull, use it to describe valid
13226 immediates. */
13227 bool
13230 {
13244 {
13253 }
13257 else
13260 /* Handle PFALSE and PTRUE. */
13265 scalar_float_mode elt_float_mode;
13268 {
13272 {
13276 }
13277 }
13285 /* Expand the vector constant out into a byte vector, with the least
13286 significant byte of the register first. */
13290 {
13291 /* The vector is provided in gcc endian-neutral fashion.
13292 For aarch64_be Advanced SIMD, it must be laid out in the vector
13293 register in reverse order. */
13305 {
13308 }
13309 }
13311 /* The immediate must repeat every eight bytes. */
13317 /* Get the repeating 8-byte value as an integer. No endian correction
13318 is needed here because bytes is already in lsb-first order. */
13326 else
13328 }
13330 /* Check whether X is a VEC_SERIES-like constant that starts at 0 and
13331 has a step in the range of INDEX. Return the index expression if so,
13332 otherwise return null. */
13333 rtx
13335 {
13342 }
13344 /* Check of immediate shift constants are within range. */
13345 bool
13347 {
13351 else
13353 }
13355 /* Return the bitmask CONST_INT to select the bits required by a zero extract
13356 operation of width WIDTH at bit position POS. */
13358 rtx
13360 {
13364 unsigned HOST_WIDE_INT mask
13367 }
13369 bool
13371 {
13390 }
13392 /* Return a const_int vector of VAL. */
13393 rtx
13395 {
13398 }
13400 /* Check OP is a legal scalar immediate for the MOVI instruction. */
13402 bool
13404 {
13405 machine_mode vmode;
13410 }
13412 /* Construct and return a PARALLEL RTX vector with elements numbering the
13413 lanes of either the high (HIGH == TRUE) or low (HIGH == FALSE) half of
13414 the vector - from the perspective of the architecture. This does not
13415 line up with GCC's perspective on lane numbers, so we end up with
13416 different masks depending on our target endian-ness. The diagram
13417 below may help. We must draw the distinction when building masks
13418 which select one half of the vector. An instruction selecting
13419 architectural low-lanes for a big-endian target, must be described using
13420 a mask selecting GCC high-lanes.
13422 Big-Endian Little-Endian
13424 GCC 0 1 2 3 3 2 1 0
13425 | x | x | x | x | | x | x | x | x |
13426 Architecture 3 2 1 0 3 2 1 0
13428 Low Mask: { 2, 3 } { 0, 1 }
13429 High Mask: { 0, 1 } { 2, 3 }
13431 MODE Is the mode of the vector and NUNITS is the number of units in it. */
13433 rtx
13435 {
13440 rtx t1;
13445 else
13453 }
13455 /* Check OP for validity as a PARALLEL RTX vector with elements
13456 numbering the lanes of either the high (HIGH == TRUE) or low lanes,
13457 from the perspective of the architecture. See the diagram above
13458 aarch64_simd_vect_par_cnst_half for more details. */
13460 bool
13463 {
13477 {
13484 }
13486 }
13488 /* Bounds-check lanes. Ensure OPERAND lies between LOW (inclusive) and
13489 HIGH (exclusive). */
13490 void
13492 const_tree exp)
13493 {
13494 HOST_WIDE_INT lane;
13499 {
13502 else
13504 }
13505 }
13507 /* Peform endian correction on lane number N, which indexes a vector
13508 of mode MODE, and return the result as an SImode rtx. */
13510 rtx
13512 {
13514 }
13516 /* Return TRUE if OP is a valid vector addressing mode. */
13518 bool
13520 {
13523 }
13525 /* Return true if OP is a valid MEM operand for an SVE LD1R instruction. */
13527 bool
13529 {
13531 scalar_mode mode;
13538 }
13540 /* Return true if OP is a valid MEM operand for an SVE LDR instruction.
13541 The conditions for STR are the same. */
13542 bool
13544 {
13551 }
13553 /* Return true if OP is a valid MEM operand for an SVE_STRUCT mode.
13554 We need to be able to access the individual pieces, so the range
13555 is different from LD[234] and ST[234]. */
13556 bool
13558 {
13565 ADDR_QUERY_ANY)
13573 }
13575 /* Emit a register copy from operand to operand, taking care not to
13576 early-clobber source registers in the process.
13578 COUNT is the number of components into which the copy needs to be
13579 decomposed. */
13580 void
13583 {
13593 else
13597 }
13599 /* Compute and return the length of aarch64_simd_reglist<mode>, where <mode> is
13600 one of VSTRUCT modes: OI, CI, or XI. */
13601 int
13603 {
13604 /* This is only used (and only meaningful) for Advanced SIMD, not SVE. */
13606 }
13608 /* Implement target hook TARGET_VECTOR_ALIGNMENT. The AAPCS64 sets the maximum
13609 alignment of a vector to 128 bits. SVE predicates have an alignment of
13610 16 bits. */
13611 static HOST_WIDE_INT
13613 {
13615 /* ??? Checking the mode isn't ideal, but VECTOR_BOOLEAN_TYPE_P can
13616 be set for non-predicate vectors of booleans. Modes are the most
13617 direct way we have of identifying real SVE predicate types. */
13621 }
13623 /* Implement target hook TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT. */
13624 static HOST_WIDE_INT
13626 {
13628 {
13629 /* If the length of the vector is fixed, try to align to that length,
13630 otherwise don't try to align at all. */
13631 HOST_WIDE_INT result;
13635 }
13637 }
13639 /* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE. */
13640 static bool
13642 {
13646 /* For fixed-length vectors, check that the vectorizer will aim for
13647 full-vector alignment. This isn't true for generic GCC vectors
13648 that are wider than the ABI maximum of 128 bits. */
13654 /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned. */
13656 }
13658 /* Return true if the vector misalignment factor is supported by the
13659 target. */
13660 static bool
13664 {
13666 {
13667 /* Return if movmisalign pattern is not supported for this mode. */
13671 /* Misalignment factor is unknown at compile time. */
13674 }
13676 is_packed);
13677 }
13679 /* If VALS is a vector constant that can be loaded into a register
13680 using DUP, generate instructions to do so and return an RTX to
13681 assign to the register. Otherwise return NULL_RTX. */
13682 static rtx
13684 {
13687 rtx x;
13692 /* We can load this constant by using DUP and a constant in a
13693 single ARM register. This will be cheaper than a vector
13694 load. */
13697 }
13700 /* Generate code to load VALS, which is a PARALLEL containing only
13701 constants (for vec_init) or CONST_VECTOR, efficiently into a
13702 register. Returns an RTX to copy into the register, or NULL_RTX
13703 for a PARALLEL that can not be converted into a CONST_VECTOR. */
13704 static rtx
13706 {
13708 rtx const_dup;
13716 {
13717 /* A CONST_VECTOR must contain only CONST_INTs and
13718 CONST_DOUBLEs, but CONSTANT_P allows more (e.g. SYMBOL_REF).
13719 Only store valid constants in a CONST_VECTOR. */
13722 {
13725 n_const++;
13726 }
13729 }
13730 else
13735 /* Load using MOVI/MVNI. */
13738 /* Loaded using DUP. */
13741 /* Load from constant pool. We can not take advantage of single-cycle
13742 LD1 because we need a PC-relative addressing mode. */
13744 else
13745 /* A PARALLEL containing something not valid inside CONST_VECTOR.
13746 We can not construct an initializer. */
13748 }
13750 /* Expand a vector initialisation sequence, such that TARGET is
13751 initialised to contain VALS. */
13753 void
13755 {
13758 /* The number of vector elements. */
13760 /* The number of vector elements which are not constant. */
13763 /* The first element of vals. */
13767 /* Count the number of variable elements to initialise. */
13769 {
13773 else
13777 }
13779 /* No variable elements, hand off to aarch64_simd_make_constant which knows
13780 how best to handle this. */
13782 {
13785 {
13788 }
13789 }
13791 /* Splat a single non-constant element if we can. */
13793 {
13797 }
13802 /* If there are only variable elements, try to optimize
13803 the insertion using dup for the most common element
13804 followed by insertions. */
13806 /* The algorithm will fill matches[*][0] with the earliest matching element,
13807 and matches[X][1] with the count of duplicate elements (if X is the
13808 earliest element which has duplicates). */
13811 {
13814 {
13816 {
13818 {
13822 }
13823 }
13824 }
13829 {
13832 }
13834 /* Create a duplicate of the most common element, unless all elements
13835 are equally useless to us, in which case just immediately set the
13836 vector register using the first element. */
13839 {
13840 /* For vectors of two 64-bit elements, we can do even better. */
13845 {
13848 /* Combine can pick up this case, but handling it directly
13849 here leaves clearer RTL.
13851 This is load_pair_lanes<mode>, and also gives us a clean-up
13852 for store_pair_lanes<mode>. */
13855 && !STRICT_ALIGNMENT
13860 {
13861 rtx t;
13864 else
13868 }
13869 }
13870 /* The subreg-move sequence below will move into lane zero of the
13871 vector register. For big-endian we want that position to hold
13872 the last element of VALS. */
13876 }
13877 else
13878 {
13881 }
13883 /* Insert the rest. */
13885 {
13891 }
13893 }
13895 /* Initialise a vector which is part-variable. We want to first try
13896 to build those lanes which are constant in the most efficient way we
13897 can. */
13899 {
13902 /* Load constant part of vector. We really don't care what goes into the
13903 parts we will overwrite, but we're more likely to be able to load the
13904 constant efficiently if it has fewer, larger, repeating parts
13905 (see aarch64_simd_valid_immediate). */
13907 {
13913 {
13914 /* Look in the copied vector, as more elements are const. */
13917 {
13920 }
13921 }
13923 }
13925 }
13927 /* Insert the variable lanes directly. */
13929 {
13935 }
13936 }
13938 static unsigned HOST_WIDE_INT
13940 {
13944 }
13946 /* Select a format to encode pointers in exception handling data. */
13947 int
13949 {
13952 {
13958 /* text+got+data < 4Gb. 4-byte signed relocs are sufficient
13959 for everything. */
13963 /* No assumptions here. 8-byte relocs required. */
13966 }
13968 }
13970 /* The last .arch and .tune assembly strings that we printed. */
13974 /* Implement ASM_DECLARE_FUNCTION_NAME. Output the ISA features used
13975 by the function fndecl. */
13977 void
13979 tree fndecl)
13980 {
13986 else
13997 /* Only update the assembler .arch string if it is distinct from the last
13998 such string we printed. */
14001 {
14004 }
14006 /* Print the cpu name we're tuning for in the comments, might be
14007 useful to readers of the generated asm. Do it only when it changes
14008 from function to function and verbose assembly is requested. */
14013 {
14017 }
14019 /* Don't forget the type directive for ELF. */
14022 }
14024 /* Implements TARGET_ASM_FILE_START. Output the assembly header. */
14026 static void
14028 {
14045 }
14047 /* Emit load exclusive. */
14049 static void
14052 {
14054 }
14056 /* Emit store exclusive. */
14058 static void
14061 {
14063 }
14065 /* Mark the previous jump instruction as unlikely. */
14067 static void
14069 {
14072 }
14074 /* Expand a compare and swap pattern. */
14076 void
14078 {
14093 /* Normally the succ memory model must be stronger than fail, but in the
14094 unlikely event of fail being ACQUIRE and succ being RELEASE we need to
14095 promote succ to ACQ_REL so that we don't lose the acquire semantics. */
14102 {
14105 /* For short modes, we're going to perform the comparison in SImode,
14106 so do the zero-extension now. */
14110 /* Fall through. */
14114 /* Force the value into a register if needed. */
14121 }
14126 mod_f));
14127 else
14138 }
14140 /* Test whether the target supports using a atomic load-operate instruction.
14141 CODE is the operation and AFTER is TRUE if the data in memory after the
14142 operation should be returned and FALSE if the data before the operation
14143 should be returned. Returns FALSE if the operation isn't supported by the
14144 architecture. */
14146 bool
14148 {
14153 {
14163 }
14164 }
14166 /* Emit a barrier, that is appropriate for memory model MODEL, at the end of a
14167 sequence implementing an atomic operation. */
14169 static void
14171 {
14178 {
14180 }
14181 }
14183 /* Emit an atomic compare-and-swap operation. RVAL is the destination register
14184 for the data in memory. EXPECTED is the value expected to be in memory.
14185 DESIRED is the value to store to memory. MEM is the memory location. MODEL
14186 is the memory ordering to use. */
14188 void
14191 rtx model)
14192 {
14193 machine_mode mode;
14197 /* Move the expected value into the CAS destination register. */
14200 /* Emit the CAS. */
14203 /* Compare the expected value with the value loaded by the CAS, to establish
14204 whether the swap was made. */
14206 }
14208 /* Split a compare and swap pattern. */
14210 void
14212 {
14214 machine_mode mode;
14219 rtx model_rtx;
14231 /* When OLDVAL is zero and we want the strong version we can emit a tighter
14232 loop:
14233 .label1:
14234 LD[A]XR rval, [mem]
14235 CBNZ rval, .label2
14236 ST[L]XR scratch, newval, [mem]
14237 CBNZ scratch, .label1
14238 .label2:
14239 CMP rval, 0. */
14244 {
14247 }
14250 /* The initial load can be relaxed for a __sync operation since a final
14251 barrier will be emitted to stop code hoisting. */
14255 else
14259 {
14261 {
14262 /* Emit an explicit compare instruction, so that we can correctly
14263 track the condition codes. */
14266 }
14267 else
14273 }
14274 else
14275 {
14281 }
14286 {
14288 {
14289 /* Emit an explicit compare instruction, so that we can correctly
14290 track the condition codes. */
14293 }
14294 else
14300 }
14301 else
14302 {
14306 }
14309 /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL
14310 to set the condition flags. If this is not used it will be removed by
14311 later passes. */
14313 {
14317 }
14318 /* Emit any final barrier needed for a __sync operation. */
14321 }
14323 /* Emit a BIC instruction. */
14325 static void
14327 {
14332 {
14337 }
14340 }
14342 /* Emit an atomic swap. */
14344 static void
14347 {
14349 }
14351 /* Emit an atomic load+operate. CODE is the operation. OUT_DATA is the
14352 location to store the data read from memory. OUT_RESULT is the location to
14353 store the result of the operation. MEM is the memory location to read and
14354 modify. MODEL_RTX is the memory ordering to use. VALUE is the second
14355 operand for the operation. Either OUT_DATA or OUT_RESULT, but not both, can
14356 be NULL. */
14358 void
14361 {
14366 rtx src;
14367 rtx x;
14375 /* Make sure the value is in a register, putting it into a destination
14376 register if it needs to be manipulated. */
14379 {
14382 }
14383 else
14387 /* Preprocess the data for the operation as necessary. If the operation is
14388 a SET then emit a swap instruction and finish. */
14390 {
14396 /* Negate the value and treat it as a PLUS. */
14397 {
14398 rtx neg_src;
14400 /* Resize the value if necessary. */
14409 }
14410 /* Fall-through. */
14424 {
14425 rtx not_src;
14427 /* Resize the value if necessary. */
14436 }
14441 /* The operation can't be done with atomic instructions. */
14443 }
14448 /* If necessary, calculate the data in memory after the update by redoing the
14449 operation from values in registers. */
14454 {
14458 }
14463 {
14479 }
14484 }
14486 /* Split an atomic operation. */
14488 void
14491 {
14497 rtx x;
14499 /* Split the atomic operation into a sequence. */
14507 else
14511 /* The initial load can be relaxed for a __sync operation since a final
14512 barrier will be emitted to stop code hoisting. */
14516 else
14520 {
14534 {
14537 }
14538 /* Fall through. */
14544 }
14550 {
14551 /* Emit an explicit compare instruction, so that we can correctly
14552 track the condition codes. */
14555 }
14556 else
14563 /* Emit any final barrier needed for a __sync operation. */
14566 }
14568 static void
14570 {
14571 /* Half-precision float operations. The compiler handles all operations
14572 with NULL libfuncs by converting to SFmode. */
14574 /* Conversions. */
14578 /* Arithmetic. */
14585 /* Comparisons. */
14593 }
14595 /* Target hook for c_mode_for_suffix. */
14596 static machine_mode
14598 {
14603 }
14605 /* We can only represent floating point constants which will fit in
14606 "quarter-precision" values. These values are characterised by
14607 a sign bit, a 4-bit mantissa and a 3-bit exponent. And are given
14608 by:
14610 (-1)^s * (n/16) * 2^r
14612 Where:
14613 's' is the sign bit.
14614 'n' is an integer in the range 16 <= n <= 31.
14615 'r' is an integer in the range -3 <= r <= 4. */
14617 /* Return true iff X can be represented by a quarter-precision
14618 floating point immediate operand X. Note, we cannot represent 0.0. */
14619 bool
14621 {
14622 /* This represents our current view of how many bits
14623 make up the mantissa. */
14639 /* We cannot represent infinities, NaNs or +/-zero. We won't
14640 know if we have +zero until we analyse the mantissa, but we
14641 can reject the other invalid values. */
14646 /* Extract exponent. */
14650 /* For the mantissa, we expand into two HOST_WIDE_INTS, apart from the
14651 highest (sign) bit, with a fixed binary point at bit point_pos.
14652 m1 holds the low part of the mantissa, m2 the high part.
14653 WARNING: If we ever have a representation using more than 2 * H_W_I - 1
14654 bits for the mantissa, this can fail (low bits will be lost). */
14658 /* If the low part of the mantissa has bits set we cannot represent
14659 the value. */
14662 /* We have rejected the lower HOST_WIDE_INT, so update our
14663 understanding of how many bits lie in the mantissa and
14664 look only at the high HOST_WIDE_INT. */
14668 /* We can only represent values with a mantissa of the form 1.xxxx. */
14673 /* Having filtered unrepresentable values, we may now remove all
14674 but the highest 5 bits. */
14677 /* We cannot represent the value 0.0, so reject it. This is handled
14678 elsewhere. */
14682 /* Then, as bit 4 is always set, we can mask it off, leaving
14683 the mantissa in the range [0, 15]. */
14687 /* GCC internally does not use IEEE754-like encoding (where normalized
14688 significands are in the range [1, 2). GCC uses [0.5, 1) (see real.c).
14689 Our mantissa values are shifted 4 places to the left relative to
14690 normalized IEEE754 so we must modify the exponent returned by REAL_EXP
14691 by 5 places to correct for GCC's representation. */
14695 }
14697 /* Returns the string with the instruction for AdvSIMD MOVI, MVNI, ORR or BIC
14698 immediate with a CONST_VECTOR of MODE and WIDTH. WHICH selects whether to
14699 output MOVI/MVNI, ORR or BIC immediate. */
14703 {
14713 /* This will return true to show const_vector is legal for use as either
14714 a AdvSIMD MOVI instruction (or, implicitly, MVNI), ORR or BIC immediate.
14715 It will also update INFO to show how the immediate should be generated.
14716 WHICH selects whether to check for MOVI/MVNI, ORR or BIC. */
14724 {
14726 /* For FP zero change it to a CONST_INT 0 and use the integer SIMD
14727 move immediate path. */
14730 else
14731 {
14740 else
14744 }
14745 }
14750 {
14760 else
14764 }
14765 else
14766 {
14767 /* For AARCH64_CHECK_BIC and AARCH64_CHECK_ORR. */
14773 else
14777 }
14779 }
14783 {
14785 /* If a floating point number was passed and we desire to use it in an
14786 integer mode do the conversion to integer. */
14788 {
14793 }
14795 machine_mode vmode;
14796 /* use a 64 bit mode for everything except for DI/DF mode, where we use
14797 a 128 bit vector mode. */
14803 }
14805 /* Return the output string to use for moving immediate CONST_VECTOR
14806 into an SVE register. */
14810 {
14821 {
14826 }
14829 {
14832 else
14833 {
14843 }
14844 }
14849 }
14851 /* Return the asm format for a PTRUE instruction whose destination has
14852 mode MODE. SUFFIX is the element size suffix. */
14856 {
14861 else
14864 }
14866 /* Split operands into moves from op[1] + op[2] into op[0]. */
14868 void
14870 {
14881 {
14882 /* No-op move. Can't split to nothing; emit something. */
14885 }
14887 /* Preserve register attributes for variable tracking. */
14892 /* Special case of reversed high/low parts. */
14895 {
14899 }
14901 {
14902 /* Try to avoid unnecessary moves if part of the result
14903 is in the right place already. */
14908 }
14909 else
14910 {
14915 }
14916 }
14918 /* vec_perm support. */
14920 struct expand_vec_perm_d
14921 {
14923 vec_perm_indices perm;
14924 machine_mode vmode;
14928 };
14930 /* Generate a variable permutation. */
14932 static void
14934 {
14945 {
14947 {
14948 /* Expand the argument to a V16QI mode by duplicating it. */
14952 }
14953 else
14954 {
14956 }
14957 }
14958 else
14959 {
14960 rtx pair;
14963 {
14967 }
14968 else
14969 {
14973 }
14974 }
14975 }
14977 /* Expand a vec_perm with the operands given by TARGET, OP0, OP1 and SEL.
14978 NELT is the number of elements in the vector. */
14980 void
14983 {
14986 rtx mask;
14988 /* The TBL instruction does not use a modulo index, so we must take care
14989 of that ourselves. */
14994 /* For big-endian, we also need to reverse the index within the vector
14995 (but not which vector). */
14997 {
14998 /* If one_vector_p, mask is a vector of (nelt - 1)'s already. */
15003 }
15005 }
15007 /* Generate (set TARGET (unspec [OP0 OP1] CODE)). */
15009 static void
15011 {
15015 }
15017 /* Expand an SVE vec_perm with the given operands. */
15019 void
15021 {
15024 /* Enforced by the pattern condition. */
15027 /* Note: vec_perm indices are supposed to wrap when they go beyond the
15028 size of the two value vectors, i.e. the upper bits of the indices
15029 are effectively ignored. SVE TBL instead produces 0 for any
15030 out-of-range indices, so we need to modulo all the vec_perm indices
15031 to ensure they are all in range. */
15034 /* Check if the sel only references the first values vector. */
15037 {
15040 }
15042 /* Check if the two values vectors are the same. */
15044 {
15050 }
15052 /* Run TBL on for each value vector and combine the results. */
15059 {
15064 }
15071 else
15073 }
15075 /* Recognize patterns suitable for the TRN instructions. */
15076 static bool
15078 {
15079 HOST_WIDE_INT odd;
15087 /* Note that these are little-endian tests.
15088 We correct for big-endian later. */
15095 /* Success! */
15101 /* We don't need a big-endian lane correction for SVE; see the comment
15102 at the head of aarch64-sve.md for details. */
15104 {
15107 }
15113 }
15115 /* Recognize patterns suitable for the UZP instructions. */
15116 static bool
15118 {
15119 HOST_WIDE_INT odd;
15126 /* Note that these are little-endian tests.
15127 We correct for big-endian later. */
15133 /* Success! */
15139 /* We don't need a big-endian lane correction for SVE; see the comment
15140 at the head of aarch64-sve.md for details. */
15142 {
15145 }
15151 }
15153 /* Recognize patterns suitable for the ZIP instructions. */
15154 static bool
15156 {
15165 /* Note that these are little-endian tests.
15166 We correct for big-endian later. */
15174 /* Success! */
15180 /* We don't need a big-endian lane correction for SVE; see the comment
15181 at the head of aarch64-sve.md for details. */
15183 {
15186 }
15192 }
15194 /* Recognize patterns for the EXT insn. */
15196 static bool
15198 {
15199 HOST_WIDE_INT location;
15200 rtx offset;
15202 /* The first element always refers to the first vector.
15203 Check if the extracted indices are increasing by one. */
15209 /* Success! */
15213 /* The case where (location == 0) is a no-op for both big- and little-endian,
15214 and is removed by the mid-end at optimization levels -O1 and higher.
15216 We don't need a big-endian lane correction for SVE; see the comment
15217 at the head of aarch64-sve.md for details. */
15219 {
15220 /* After setup, we want the high elements of the first vector (stored
15221 at the LSB end of the register), and the low elements of the second
15222 vector (stored at the MSB end of the register). So swap. */
15224 /* location != 0 (above), so safe to assume (nelt - location) < nelt.
15225 to_constant () is safe since this is restricted to Advanced SIMD
15226 vectors. */
15228 }
15234 UNSPEC_EXT));
15236 }
15238 /* Recognize patterns for the REV{64,32,16} insns, which reverse elements
15239 within each 64-bit, 32-bit or 16-bit granule. */
15241 static bool
15243 {
15244 HOST_WIDE_INT diff;
15246 machine_mode pred_mode;
15255 {
15258 }
15260 {
15263 }
15265 {
15268 }
15269 else
15277 /* Success! */
15283 {
15286 UNSPEC_MERGE_PTRUE);
15287 }
15290 }
15292 /* Recognize patterns for the REV insn, which reverses elements within
15293 a full vector. */
15295 static bool
15297 {
15306 /* Success! */
15313 }
15315 static bool
15317 {
15319 rtx in0;
15320 HOST_WIDE_INT elt;
15322 rtx lane;
15332 /* Success! */
15336 /* The generic preparation in aarch64_expand_vec_perm_const_1
15337 swaps the operand order and the permute indices if it finds
15338 d->perm[0] to be in the second operand. Thus, we can always
15339 use d->op0 and need not do any extra arithmetic to get the
15340 correct lane number. */
15348 }
15350 static bool
15352 {
15356 /* Make sure that the indices are constant. */
15365 /* Generic code will try constant permutation twice. Once with the
15366 original mode and again with the elements lowered to QImode.
15367 So wait and don't do the selector expansion ourselves. */
15371 /* to_constant is safe since this routine is specific to Advanced SIMD
15372 vectors. */
15375 /* If big-endian and two vectors we end up with a weird mixed-endian
15376 mode on NEON. Reverse the index within each word but not the word
15377 itself. to_constant is safe because we checked is_constant above. */
15387 }
15389 /* Try to implement D using an SVE TBL instruction. */
15391 static bool
15393 {
15396 /* Permuting two variable-length vectors could overflow the
15397 index range. */
15408 }
15410 static bool
15412 {
15413 /* The pattern matching functions above are written to look for a small
15414 number to begin the sequence (0, 1, N/2). If we begin with an index
15415 from the second operand, we can swap the operands. */
15418 {
15421 }
15427 {
15446 }
15448 }
15450 /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
15452 static bool
15455 {
15458 /* Check whether the mask can be applied to a single vector. */
15462 {
15465 }
15467 {
15470 }
15471 else
15491 }
15493 /* Generate a byte permute mask for a register of mode MODE,
15494 which has NUNITS units. */
15496 rtx
15498 {
15499 /* We have to reverse each vector because we dont have
15500 a permuted load that can reverse-load according to ABI rules. */
15501 rtx mask;
15514 }
15516 /* Return true if X is a valid second operand for the SVE instruction
15517 that implements integer comparison OP_CODE. */
15519 static bool
15521 {
15526 {
15541 }
15542 }
15544 /* Use predicated SVE instructions to implement the equivalent of:
15546 (set TARGET OP)
15548 given that PTRUE is an all-true predicate of the appropriate mode. */
15550 static void
15552 {
15555 UNSPEC_MERGE_PTRUE);
15558 }
15560 /* Likewise, but also clobber the condition codes. */
15562 static void
15564 {
15567 UNSPEC_MERGE_PTRUE);
15570 }
15572 /* Return the UNSPEC_COND_* code for comparison CODE. */
15574 static unsigned int
15576 {
15578 {
15593 }
15594 }
15596 /* Emit:
15598 (set TARGET (unspec [PRED OP0 OP1] UNSPEC_COND_<X>))
15600 where <X> is the operation associated with comparison CODE. This form
15601 of instruction is used when (and (CODE OP0 OP1) PRED) would have different
15602 semantics, such as when PRED might not be all-true and when comparing
15603 inactive lanes could have side effects. */
15605 static void
15608 {
15613 }
15615 /* Expand an SVE integer comparison using the SVE equivalent of:
15617 (set TARGET (CODE OP0 OP1)). */
15619 void
15621 {
15631 }
15633 /* Emit the SVE equivalent of:
15635 (set TMP1 (CODE1 OP0 OP1))
15636 (set TMP2 (CODE2 OP0 OP1))
15637 (set TARGET (ior:PRED_MODE TMP1 TMP2))
15639 PTRUE is an all-true predicate with the same mode as TARGET. */
15641 static void
15644 {
15653 }
15655 /* Emit the SVE equivalent of:
15657 (set TMP (CODE OP0 OP1))
15658 (set TARGET (not TMP))
15660 PTRUE is an all-true predicate with the same mode as TARGET. */
15662 static void
15665 {
15671 }
15673 /* Expand an SVE floating-point comparison using the SVE equivalent of:
15675 (set TARGET (CODE OP0 OP1))
15677 If CAN_INVERT_P is true, the caller can also handle inverted results;
15678 return true if the result is in fact inverted. */
15680 bool
15683 {
15689 {
15691 /* UNORDERED has no immediate form. */
15693 /* fall through */
15700 {
15701 /* There is native support for the comparison. */
15705 }
15708 /* This is a trapping operation (LT or GT). */
15714 {
15715 /* This would trap for signaling NaNs. */
15719 }
15720 /* fall through */
15726 {
15727 /* Work out which elements are ordered. */
15732 /* Test the opposite condition for the ordered elements,
15733 then invert the result. */
15736 else
15739 {
15743 }
15748 }
15752 /* ORDERED has no immediate form. */
15758 }
15760 /* There is native support for the inverse comparison. */
15763 {
15767 }
15770 }
15772 /* Expand an SVE vcond pattern with operands OPS. DATA_MODE is the mode
15773 of the data being selected and CMP_MODE is the mode of the values being
15774 compared. */
15776 void
15779 {
15780 machine_mode pred_mode
15785 {
15789 }
15790 else
15795 }
15797 /* Implement TARGET_MODES_TIEABLE_P. In principle we should always return
15798 true. However due to issues with register allocation it is preferable
15799 to avoid tieing integer scalar and FP scalar modes. Executing integer
15800 operations in general registers is better than treating them as scalar
15801 vector operations. This reduces latency and avoids redundant int<->FP
15802 moves. So tie modes if they are either the same class, or vector modes
15803 with other vector modes, vector structs or any scalar mode. */
15805 static bool
15807 {
15811 /* We specifically want to allow elements of "structure" modes to
15812 be tieable to the structure. This more general condition allows
15813 other rarer situations too. The reason we don't extend this to
15814 predicate modes is that there are no predicate structure modes
15815 nor any specific instructions for extracting part of a predicate
15816 register. */
15821 /* Also allow any scalar modes with vectors. */
15827 }
15829 /* Return a new RTX holding the result of moving POINTER forward by
15830 AMOUNT bytes. */
15832 static rtx
15834 {
15839 }
15841 /* Return a new RTX holding the result of moving POINTER forward by the
15842 size of the mode it points to. */
15844 static rtx
15846 {
15848 }
15850 /* Copy one MODE sized block from SRC to DST, then progress SRC and DST by
15851 MODE bytes. */
15853 static void
15855 machine_mode mode)
15856 {
15859 /* "Cast" the pointers to the correct mode. */
15862 /* Emit the memcpy. */
15865 /* Move the pointers forward. */
15868 }
15870 /* Expand movmem, as if from a __builtin_memcpy. Return true if
15871 we succeed, otherwise return false. */
15873 bool
15875 {
15879 rtx base;
15883 /* When optimizing for size, give a better estimate of the length of a
15884 memcpy call, but use the default otherwise. Moves larger than 8 bytes
15885 will always require an even number of instructions to do now. And each
15886 operation requires both a load+store, so devide the max number by 2. */
15889 /* We can't do anything smart if the amount to copy is not constant. */
15895 /* Try to keep the number of instructions low. For all cases we will do at
15896 most two moves for the residual amount, since we'll always overlap the
15897 remainder. */
15907 /* Convert n to bits to make the rest of the code simpler. */
15911 {
15912 /* Find the largest mode in which to do the copy in without over reading
15913 or writing. */
15914 opt_scalar_int_mode mode_iter;
15926 /* Do certain trailing copies as overlapping if it's going to be
15927 cheaper. i.e. less instructions to do so. For instance doing a 15
15928 byte copy it's more efficient to do two overlapping 8 byte copies than
15929 8 + 6 + 1. */
15933 {
15937 }
15938 }
15941 }
15943 /* Split a DImode store of a CONST_INT SRC to MEM DST as two
15944 SImode stores. Handle the case when the constant has identical
15945 bottom and top halves. This is beneficial when the two stores can be
15946 merged into an STP and we avoid synthesising potentially expensive
15947 immediates twice. Return true if such a split is possible. */
15949 bool
15951 {
15960 unsigned int orig_cost
15962 unsigned int lo_cost
15965 /* We want to transform:
15966 MOV x1, 49370
15967 MOVK x1, 0x140, lsl 16
15968 MOVK x1, 0xc0da, lsl 32
15969 MOVK x1, 0x140, lsl 48
15970 STR x1, [x0]
15971 into:
15972 MOV w1, 49370
15973 MOVK w1, 0x140, lsl 16
15974 STP w1, w1, [x0]
15975 So we want to perform this only when we save two instructions
15976 or more. When optimizing for size, however, accept any code size
15977 savings we can. */
15992 /* Don't emit an explicit store pair as this may not be always profitable.
15993 Let the sched-fusion logic decide whether to merge them. */
15998 }
16000 /* Generate RTL for a conditional branch with rtx comparison CODE in
16001 mode CC_MODE. The destination of the unlikely conditional branch
16002 is LABEL_REF. */
16004 void
16006 rtx label_ref)
16007 {
16008 rtx x;
16011 const0_rtx);
16015 pc_rtx);
16017 }
16019 /* Generate DImode scratch registers for 128-bit (TImode) addition.
16021 OP1 represents the TImode destination operand 1
16022 OP2 represents the TImode destination operand 2
16023 LOW_DEST represents the low half (DImode) of TImode operand 0
16024 LOW_IN1 represents the low half (DImode) of TImode operand 1
16025 LOW_IN2 represents the low half (DImode) of TImode operand 2
16026 HIGH_DEST represents the high half (DImode) of TImode operand 0
16027 HIGH_IN1 represents the high half (DImode) of TImode operand 1
16028 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
16030 void
16035 {
16044 }
16046 /* Generate DImode scratch registers for 128-bit (TImode) subtraction.
16048 This function differs from 'arch64_addti_scratch_regs' in that
16049 OP1 can be an immediate constant (zero). We must call
16050 subreg_highpart_offset with DImode and TImode arguments, otherwise
16051 VOIDmode will be used for the const_int which generates an internal
16052 error from subreg_size_highpart_offset which does not expect a size of zero.
16054 OP1 represents the TImode destination operand 1
16055 OP2 represents the TImode destination operand 2
16056 LOW_DEST represents the low half (DImode) of TImode operand 0
16057 LOW_IN1 represents the low half (DImode) of TImode operand 1
16058 LOW_IN2 represents the low half (DImode) of TImode operand 2
16059 HIGH_DEST represents the high half (DImode) of TImode operand 0
16060 HIGH_IN1 represents the high half (DImode) of TImode operand 1
16061 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
16064 void
16069 {
16082 }
16084 /* Generate RTL for 128-bit (TImode) subtraction with overflow.
16086 OP0 represents the TImode destination operand 0
16087 LOW_DEST represents the low half (DImode) of TImode operand 0
16088 LOW_IN1 represents the low half (DImode) of TImode operand 1
16089 LOW_IN2 represents the low half (DImode) of TImode operand 2
16090 HIGH_DEST represents the high half (DImode) of TImode operand 0
16091 HIGH_IN1 represents the high half (DImode) of TImode operand 1
16092 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
16094 void
16097 rtx high_in2)
16098 {
16100 {
16104 }
16105 else
16106 {
16108 {
16112 }
16113 else
16117 high_in2));
16118 }
16123 }
16125 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
16127 static unsigned HOST_WIDE_INT
16129 {
16131 }
16133 static rtx
16136 {
16140 insn_code icode;
16151 {
16179 }
16184 {
16187 }
16196 {
16199 }
16205 }
16207 static rtx
16210 {
16214 insn_code icode;
16226 {
16254 }
16259 {
16262 }
16270 {
16275 }
16286 {
16289 }
16295 }
16297 #undef TARGET_GEN_CCMP_FIRST
16298 #define TARGET_GEN_CCMP_FIRST aarch64_gen_ccmp_first
16300 #undef TARGET_GEN_CCMP_NEXT
16301 #define TARGET_GEN_CCMP_NEXT aarch64_gen_ccmp_next
16303 /* Implement TARGET_SCHED_MACRO_FUSION_P. Return true if target supports
16304 instruction fusion of some sort. */
16306 static bool
16308 {
16310 }
16313 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P. Return true if PREV and CURR
16314 should be kept together during scheduling. */
16316 static bool
16318 {
16319 rtx set_dest;
16322 /* prev and curr are simple SET insns i.e. no flag setting or branching. */
16329 {
16330 /* We are trying to match:
16331 prev (mov) == (set (reg r0) (const_int imm16))
16332 curr (movk) == (set (zero_extract (reg r0)
16333 (const_int 16)
16334 (const_int 16))
16335 (const_int imm16_1)) */
16347 {
16349 }
16350 }
16353 {
16355 /* We're trying to match:
16356 prev (adrp) == (set (reg r1)
16357 (high (symbol_ref ("SYM"))))
16358 curr (add) == (set (reg r0)
16359 (lo_sum (reg r1)
16360 (symbol_ref ("SYM"))))
16361 Note that r0 need not necessarily be the same as r1, especially
16362 during pre-regalloc scheduling. */
16366 {
16374 }
16375 }
16378 {
16380 /* We're trying to match:
16381 prev (movk) == (set (zero_extract (reg r0)
16382 (const_int 16)
16383 (const_int 32))
16384 (const_int imm16_1))
16385 curr (movk) == (set (zero_extract (reg r0)
16386 (const_int 16)
16387 (const_int 48))
16388 (const_int imm16_2)) */
16404 }
16406 {
16407 /* We're trying to match:
16408 prev (adrp) == (set (reg r0)
16409 (high (symbol_ref ("SYM"))))
16410 curr (ldr) == (set (reg r1)
16411 (mem (lo_sum (reg r0)
16412 (symbol_ref ("SYM")))))
16413 or
16414 curr (ldr) == (set (reg r1)
16415 (zero_extend (mem
16416 (lo_sum (reg r0)
16417 (symbol_ref ("SYM")))))) */
16420 {
16433 }
16434 }
16442 {
16446 rtx cc_reg_1;
16451 && prev
16453 {
16454 /* FIXME: this misses some which is considered simple arthematic
16455 instructions for ThunderX. Simple shifts are missed here. */
16461 }
16462 }
16465 && curr_set
16468 {
16469 /* We're trying to match:
16470 prev (alu_insn) == (set (r0) plus ((r0) (r1/imm)))
16471 curr (cbz) == (set (pc) (if_then_else (eq/ne) (r0)
16472 (const_int 0))
16473 (label_ref ("SYM"))
16474 (pc)) */
16481 {
16482 /* Fuse ALU operations followed by conditional branch instruction. */
16484 {
16505 }
16506 }
16507 }
16510 }
16512 /* Return true iff the instruction fusion described by OP is enabled. */
16514 bool
16516 {
16518 }
16520 /* If MEM is in the form of [base+offset], extract the two parts
16521 of address and set to BASE and OFFSET, otherwise return false
16522 after clearing BASE and OFFSET. */
16524 bool
16526 {
16527 rtx addr;
16534 {
16538 }
16542 {
16546 }
16552 }
16554 /* Types for scheduling fusion. */
16555 enum sched_fusion_type
16556 {
16558 SCHED_FUSION_LD_SIGN_EXTEND,
16559 SCHED_FUSION_LD_ZERO_EXTEND,
16560 SCHED_FUSION_LD,
16561 SCHED_FUSION_ST,
16562 SCHED_FUSION_NUM
16563 };
16565 /* If INSN is a load or store of address in the form of [base+offset],
16566 extract the two parts and set to BASE and OFFSET. Return scheduling
16567 fusion type this INSN is. */
16569 static enum sched_fusion_type
16571 {
16589 {
16594 }
16596 {
16601 }
16606 {
16609 }
16610 else
16617 }
16619 /* Implement the TARGET_SCHED_FUSION_PRIORITY hook.
16621 Currently we only support to fuse ldr or str instructions, so FUSION_PRI
16622 and PRI are only calculated for these instructions. For other instruction,
16623 FUSION_PRI and PRI are simply set to MAX_PRI - 1. In the future, other
16624 type instruction fusion can be added by returning different priorities.
16626 It's important that irrelevant instructions get the largest FUSION_PRI. */
16628 static void
16631 {
16641 {
16645 }
16647 /* Set FUSION_PRI according to fusion type and base register. */
16650 /* Calculate PRI. */
16653 /* INSN with smaller offset goes first. */
16657 else
16662 }
16664 /* Implement the TARGET_SCHED_ADJUST_PRIORITY hook.
16665 Adjust priority of sha1h instructions so they are scheduled before
16666 other SHA1 instructions. */
16668 static int
16670 {
16674 {
16679 }
16682 }
16684 /* Given OPERANDS of consecutive load/store, check if we can merge
16685 them into ldp/stp. LOAD is true if they are load instructions.
16686 MODE is the mode of memory operands. */
16688 bool
16690 machine_mode mode)
16691 {
16697 {
16705 }
16706 else
16707 {
16712 }
16714 /* The mems cannot be volatile. */
16718 /* If we have SImode and slow unaligned ldp,
16719 check the alignment to be at least 8 byte. */
16723 && !optimize_size
16727 /* Check if the addresses are in the form of [base+offset]. */
16735 /* Check if the bases are same. */
16739 /* The operands must be of the same size. */
16745 /* We should only be trying this for fixed-sized modes. There is no
16746 SVE LDP/STP instruction. */
16748 /* Check if the offsets are consecutive. */
16752 /* Check if the addresses are clobbered by load. */
16754 {
16758 /* In increasing order, the last load can clobber the address. */
16761 }
16763 /* One of the memory accesses must be a mempair operand.
16764 If it is not the first one, they need to be swapped by the
16765 peephole. */
16772 else
16777 else
16780 /* Check if the registers are of same class. */
16785 }
16787 /* Given OPERANDS of consecutive load/store that can be merged,
16788 swap them if they are not in ascending order. */
16789 void
16791 {
16796 {
16799 }
16800 else
16801 {
16804 }
16813 {
16814 /* Irrespective of whether this is a load or a store,
16815 we do the same swap. */
16818 }
16819 }
16821 /* Taking X and Y to be HOST_WIDE_INT pointers, return the result of a
16822 comparison between the two. */
16823 int
16825 {
16828 }
16830 /* Taking X and Y to be pairs of RTX, one pointing to a MEM rtx and the
16831 other pointing to a REG rtx containing an offset, compare the offsets
16832 of the two pairs.
16834 Return:
16836 1 iff offset (X) > offset (Y)
16837 0 iff offset (X) == offset (Y)
16838 -1 iff offset (X) < offset (Y) */
16839 int
16841 {
16848 else
16853 else
16856 /* Extract the offsets. */
16863 }
16865 /* Given OPERANDS of consecutive load/store, check if we can merge
16866 them into ldp/stp by adjusting the offset. LOAD is true if they
16867 are load instructions. MODE is the mode of memory operands.
16869 Given below consecutive stores:
16871 str w1, [xb, 0x100]
16872 str w1, [xb, 0x104]
16873 str w1, [xb, 0x108]
16874 str w1, [xb, 0x10c]
16876 Though the offsets are out of the range supported by stp, we can
16877 still pair them after adjusting the offset, like:
16879 add scratch, xb, 0x100
16880 stp w1, w1, [scratch]
16881 stp w1, w1, [scratch, 0x8]
16883 The peephole patterns detecting this opportunity should guarantee
16884 the scratch register is avaliable. */
16886 bool
16888 scalar_mode mode)
16889 {
16896 {
16898 {
16903 }
16905 /* Do not attempt to merge the loads if the loads clobber each other. */
16910 }
16911 else
16913 {
16916 }
16918 /* Skip if memory operand is by itself valid for ldp/stp. */
16923 {
16924 /* The mems cannot be volatile. */
16928 /* Check if the addresses are in the form of [base+offset]. */
16932 }
16934 /* Check if the registers are of same class. */
16940 {
16943 }
16944 else
16945 {
16948 }
16950 /* Only the last register in the order in which they occur
16951 may be clobbered by the load. */
16957 /* Check if the bases are same. */
16967 /* Check if the offsets can be put in the right order to do a ldp/stp. */
16969 aarch64_host_wide_int_compare);
16975 /* Check that offsets are within range of each other. The ldp/stp
16976 instructions have 7 bit immediate offsets, so use 0x80. */
16980 /* The offsets must be aligned with respect to each other. */
16984 /* If we have SImode and slow unaligned ldp,
16985 check the alignment to be at least 8 byte. */
16989 && !optimize_size
16994 }
16996 /* Given OPERANDS of consecutive load/store, this function pairs them
16997 into LDP/STP after adjusting the offset. It depends on the fact
16998 that the operands can be sorted so the offsets are correct for STP.
16999 MODE is the mode of memory operands. CODE is the rtl operator
17000 which should be applied to all memory operands, it's SIGN_EXTEND,
17001 ZERO_EXTEND or UNKNOWN. */
17003 bool
17006 {
17013 /* We make changes on a copy as we may still bail out. */
17017 /* Sort the operands. */
17021 {
17026 }
17027 else
17028 {
17034 }
17041 /* Adjust offset so it can fit in LDP/STP instruction. */
17049 /* The base offset is optimally half way between the two STP/LDP offsets. */
17052 else
17053 /* However, due to issues with negative LDP/STP offset generation for
17054 larger modes, for DF, DI and vector modes. we must not use negative
17055 addresses smaller than 9 signed unadjusted bits can store. This
17056 provides the most range in this case. */
17059 /* Adjust the base so that it is aligned with the addresses but still
17060 optimal. */
17062 /* Fix the offset, bearing in mind we want to make it bigger not
17063 smaller. */
17066 /* The negative range of LDP/STP is one larger than the positive range. */
17069 /* Check if base offset is too big or too small. We can attempt to resolve
17070 this issue by setting it to the maximum value and seeing if the offsets
17071 still fit. */
17073 {
17075 /* We must still make sure that the base offset is aligned with respect
17076 to the address. But it may may not be made any bigger. */
17078 }
17080 /* Likewise for the case where the base is too small. */
17082 {
17085 }
17087 /* Offset of the first STP/LDP. */
17090 /* Offset of the second STP/LDP. */
17093 /* The offsets must be within the range of the LDP/STP instructions. */
17112 {
17117 }
17119 {
17124 }
17127 {
17136 }
17137 else
17138 {
17147 }
17149 /* Emit adjusting instruction. */
17151 /* Emit ldp/stp instructions. */
17159 }
17161 /* Implement TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE. Assume for now that
17162 it isn't worth branching around empty masked ops (including masked
17163 stores). */
17165 static bool
17167 {
17169 }
17171 /* Return 1 if pseudo register should be created and used to hold
17172 GOT address for PIC code. */
17174 bool
17176 {
17178 }
17180 /* Implement TARGET_UNSPEC_MAY_TRAP_P. */
17182 static int
17184 {
17186 {
17193 }
17196 }
17199 /* If X is a positive CONST_DOUBLE with a value that is a power of 2
17200 return the log2 of that value. Otherwise return -1. */
17202 int
17204 {
17219 }
17221 /* If X is a vector of equal CONST_DOUBLE values and that value is
17222 Y, return the aarch64_fpconst_pow_of_2 of Y. Otherwise return -1. */
17224 int
17226 {
17244 }
17246 /* Implement TARGET_PROMOTED_TYPE to promote 16-bit floating point types
17247 to float.
17249 __fp16 always promotes through this hook.
17250 _Float16 may promote if TARGET_FLT_EVAL_METHOD is 16, but we do that
17251 through the generic excess precision logic rather than here. */
17253 static tree
17255 {
17261 }
17263 /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
17265 static bool
17267 optimization_type opt_type)
17268 {
17270 {
17276 }
17277 }
17279 /* Implement the TARGET_DWARF_POLY_INDETERMINATE_VALUE hook. */
17281 static unsigned int
17284 {
17285 /* Polynomial invariant 1 == (VG / 2) - 1. */
17290 }
17292 /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
17293 if MODE is HFmode, and punt to the generic implementation otherwise. */
17295 static bool
17297 {
17301 }
17303 /* Implement TARGET_SCALAR_MODE_SUPPORTED_P - return TRUE
17304 if MODE is HFmode, and punt to the generic implementation otherwise. */
17306 static bool
17308 {
17312 }
17314 /* Set the value of FLT_EVAL_METHOD.
17315 ISO/IEC TS 18661-3 defines two values that we'd like to make use of:
17317 0: evaluate all operations and constants, whose semantic type has at
17318 most the range and precision of type float, to the range and
17319 precision of float; evaluate all other operations and constants to
17320 the range and precision of the semantic type;
17322 N, where _FloatN is a supported interchange floating type
17323 evaluate all operations and constants, whose semantic type has at
17324 most the range and precision of _FloatN type, to the range and
17325 precision of the _FloatN type; evaluate all other operations and
17326 constants to the range and precision of the semantic type;
17328 If we have the ARMv8.2-A extensions then we support _Float16 in native
17329 precision, so we should set this to 16. Otherwise, we support the type,
17330 but want to evaluate expressions in float precision, so set this to
17331 0. */
17333 static enum flt_eval_method
17335 {
17337 {
17340 /* We can calculate either in 16-bit range and precision or
17341 32-bit range and precision. Make that decision based on whether
17342 we have native support for the ARMv8.2-A 16-bit floating-point
17343 instructions or not. */
17345 ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
17351 }
17353 }
17355 /* Implement TARGET_SCHED_CAN_SPECULATE_INSN. Return true if INSN can be
17356 scheduled for speculative execution. Reject the long-running division
17357 and square-root instructions. */
17359 static bool
17361 {
17363 {
17381 }
17382 }
17384 /* Implement TARGET_COMPUTE_PRESSURE_CLASSES. */
17386 static int
17388 {
17392 /* PR_REGS isn't a useful pressure class because many predicate pseudo
17393 registers need to go in PR_LO_REGS at some point during their
17394 lifetime. Splitting it into two halves has the effect of making
17395 all predicates count against PR_LO_REGS, so that we try whenever
17396 possible to restrict the number of live predicates to 8. This
17397 greatly reduces the amount of spilling in certain loops. */
17401 }
17403 /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
17405 static bool
17408 {
17410 {
17414 /* Don't allow changes between SVE data modes and non-SVE modes.
17415 See the comment at the head of aarch64-sve.md for details. */
17419 /* Don't allow changes in element size: lane 0 of the new vector
17420 would not then be lane 0 of the old vector. See the comment
17421 above aarch64_maybe_expand_sve_subreg_move for a more detailed
17422 description.
17424 In the worst case, this forces a register to be spilled in
17425 one mode and reloaded in the other, which handles the
17426 endianness correctly. */
17429 }
17431 }
17433 /* Implement TARGET_EARLY_REMAT_MODES. */
17435 static void
17437 {
17438 /* SVE values are not normally live across a call, so it should be
17439 worth doing early rematerialization even in VL-specific mode. */
17441 {
17446 }
17447 }
17449 /* Override the default target speculation_safe_value. */
17450 static rtx
17453 {
17454 /* Maybe we should warn if falling back to hard barriers. They are
17455 likely to be noticably more expensive than the alternative below. */
17466 {
17484 }
17486 }
17488 /* Target-specific selftests. */
17490 #if CHECKING_P
17494 /* Selftest for the RTL loader.
17495 Verify that the RTL loader copes with a dump from
17496 print_rtx_function. This is essentially just a test that class
17497 function_reader can handle a real dump, but it also verifies
17498 that lookup_reg_by_dump_name correctly handles hard regs.
17499 The presence of hard reg names in the dump means that the test is
17500 target-specific, hence it is in this file. */
17502 static void
17504 {
17516 /* Verify crtl->return_rtx. */
17520 }
17522 /* Run all target-specific selftests. */
17524 static void
17526 {
17528 }
17534 #undef TARGET_ADDRESS_COST
17535 #define TARGET_ADDRESS_COST aarch64_address_cost
17537 /* This hook will determines whether unnamed bitfields affect the alignment
17538 of the containing structure. The hook returns true if the structure
17539 should inherit the alignment requirements of an unnamed bitfield's
17540 type. */
17541 #undef TARGET_ALIGN_ANON_BITFIELD
17542 #define TARGET_ALIGN_ANON_BITFIELD hook_bool_void_true
17544 #undef TARGET_ASM_ALIGNED_DI_OP
17547 #undef TARGET_ASM_ALIGNED_HI_OP
17550 #undef TARGET_ASM_ALIGNED_SI_OP
17553 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
17554 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK \
17555 hook_bool_const_tree_hwi_hwi_const_tree_true
17557 #undef TARGET_ASM_FILE_START
17558 #define TARGET_ASM_FILE_START aarch64_start_file
17560 #undef TARGET_ASM_OUTPUT_MI_THUNK
17561 #define TARGET_ASM_OUTPUT_MI_THUNK aarch64_output_mi_thunk
17563 #undef TARGET_ASM_SELECT_RTX_SECTION
17564 #define TARGET_ASM_SELECT_RTX_SECTION aarch64_select_rtx_section
17566 #undef TARGET_ASM_TRAMPOLINE_TEMPLATE
17567 #define TARGET_ASM_TRAMPOLINE_TEMPLATE aarch64_asm_trampoline_template
17569 #undef TARGET_BUILD_BUILTIN_VA_LIST
17570 #define TARGET_BUILD_BUILTIN_VA_LIST aarch64_build_builtin_va_list
17572 #undef TARGET_CALLEE_COPIES
17573 #define TARGET_CALLEE_COPIES hook_bool_CUMULATIVE_ARGS_mode_tree_bool_false
17575 #undef TARGET_CAN_ELIMINATE
17576 #define TARGET_CAN_ELIMINATE aarch64_can_eliminate
17578 #undef TARGET_CAN_INLINE_P
17579 #define TARGET_CAN_INLINE_P aarch64_can_inline_p
17581 #undef TARGET_CANNOT_FORCE_CONST_MEM
17582 #define TARGET_CANNOT_FORCE_CONST_MEM aarch64_cannot_force_const_mem
17584 #undef TARGET_CASE_VALUES_THRESHOLD
17585 #define TARGET_CASE_VALUES_THRESHOLD aarch64_case_values_threshold
17587 #undef TARGET_CONDITIONAL_REGISTER_USAGE
17588 #define TARGET_CONDITIONAL_REGISTER_USAGE aarch64_conditional_register_usage
17590 /* Only the least significant bit is used for initialization guard
17591 variables. */
17592 #undef TARGET_CXX_GUARD_MASK_BIT
17593 #define TARGET_CXX_GUARD_MASK_BIT hook_bool_void_true
17595 #undef TARGET_C_MODE_FOR_SUFFIX
17596 #define TARGET_C_MODE_FOR_SUFFIX aarch64_c_mode_for_suffix
17598 #ifdef TARGET_BIG_ENDIAN_DEFAULT
17599 #undef TARGET_DEFAULT_TARGET_FLAGS
17600 #define TARGET_DEFAULT_TARGET_FLAGS (MASK_BIG_END)
17601 #endif
17603 #undef TARGET_CLASS_MAX_NREGS
17604 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
17606 #undef TARGET_BUILTIN_DECL
17607 #define TARGET_BUILTIN_DECL aarch64_builtin_decl
17609 #undef TARGET_BUILTIN_RECIPROCAL
17610 #define TARGET_BUILTIN_RECIPROCAL aarch64_builtin_reciprocal
17612 #undef TARGET_C_EXCESS_PRECISION
17613 #define TARGET_C_EXCESS_PRECISION aarch64_excess_precision
17615 #undef TARGET_EXPAND_BUILTIN
17616 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
17618 #undef TARGET_EXPAND_BUILTIN_VA_START
17619 #define TARGET_EXPAND_BUILTIN_VA_START aarch64_expand_builtin_va_start
17621 #undef TARGET_FOLD_BUILTIN
17622 #define TARGET_FOLD_BUILTIN aarch64_fold_builtin
17624 #undef TARGET_FUNCTION_ARG
17625 #define TARGET_FUNCTION_ARG aarch64_function_arg
17627 #undef TARGET_FUNCTION_ARG_ADVANCE
17628 #define TARGET_FUNCTION_ARG_ADVANCE aarch64_function_arg_advance
17630 #undef TARGET_FUNCTION_ARG_BOUNDARY
17631 #define TARGET_FUNCTION_ARG_BOUNDARY aarch64_function_arg_boundary
17633 #undef TARGET_FUNCTION_ARG_PADDING
17634 #define TARGET_FUNCTION_ARG_PADDING aarch64_function_arg_padding
17636 #undef TARGET_GET_RAW_RESULT_MODE
17637 #define TARGET_GET_RAW_RESULT_MODE aarch64_get_reg_raw_mode
17638 #undef TARGET_GET_RAW_ARG_MODE
17639 #define TARGET_GET_RAW_ARG_MODE aarch64_get_reg_raw_mode
17641 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
17642 #define TARGET_FUNCTION_OK_FOR_SIBCALL aarch64_function_ok_for_sibcall
17644 #undef TARGET_FUNCTION_VALUE
17645 #define TARGET_FUNCTION_VALUE aarch64_function_value
17647 #undef TARGET_FUNCTION_VALUE_REGNO_P
17648 #define TARGET_FUNCTION_VALUE_REGNO_P aarch64_function_value_regno_p
17650 #undef TARGET_GIMPLE_FOLD_BUILTIN
17651 #define TARGET_GIMPLE_FOLD_BUILTIN aarch64_gimple_fold_builtin
17653 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
17654 #define TARGET_GIMPLIFY_VA_ARG_EXPR aarch64_gimplify_va_arg_expr
17656 #undef TARGET_INIT_BUILTINS
17657 #define TARGET_INIT_BUILTINS aarch64_init_builtins
17659 #undef TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS
17660 #define TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS \
17661 aarch64_ira_change_pseudo_allocno_class
17663 #undef TARGET_LEGITIMATE_ADDRESS_P
17664 #define TARGET_LEGITIMATE_ADDRESS_P aarch64_legitimate_address_hook_p
17666 #undef TARGET_LEGITIMATE_CONSTANT_P
17667 #define TARGET_LEGITIMATE_CONSTANT_P aarch64_legitimate_constant_p
17669 #undef TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT
17670 #define TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT \
17671 aarch64_legitimize_address_displacement
17673 #undef TARGET_LIBGCC_CMP_RETURN_MODE
17674 #define TARGET_LIBGCC_CMP_RETURN_MODE aarch64_libgcc_cmp_return_mode
17676 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
17677 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
17678 aarch64_libgcc_floating_mode_supported_p
17680 #undef TARGET_MANGLE_TYPE
17681 #define TARGET_MANGLE_TYPE aarch64_mangle_type
17683 #undef TARGET_MEMORY_MOVE_COST
17684 #define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
17686 #undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
17687 #define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
17689 #undef TARGET_MUST_PASS_IN_STACK
17690 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
17692 /* This target hook should return true if accesses to volatile bitfields
17693 should use the narrowest mode possible. It should return false if these
17694 accesses should use the bitfield container type. */
17695 #undef TARGET_NARROW_VOLATILE_BITFIELD
17696 #define TARGET_NARROW_VOLATILE_BITFIELD hook_bool_void_false
17698 #undef TARGET_OPTION_OVERRIDE
17699 #define TARGET_OPTION_OVERRIDE aarch64_override_options
17701 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
17702 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE \
17703 aarch64_override_options_after_change
17705 #undef TARGET_OPTION_SAVE
17706 #define TARGET_OPTION_SAVE aarch64_option_save
17708 #undef TARGET_OPTION_RESTORE
17709 #define TARGET_OPTION_RESTORE aarch64_option_restore
17711 #undef TARGET_OPTION_PRINT
17712 #define TARGET_OPTION_PRINT aarch64_option_print
17714 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
17715 #define TARGET_OPTION_VALID_ATTRIBUTE_P aarch64_option_valid_attribute_p
17717 #undef TARGET_SET_CURRENT_FUNCTION
17718 #define TARGET_SET_CURRENT_FUNCTION aarch64_set_current_function
17720 #undef TARGET_PASS_BY_REFERENCE
17721 #define TARGET_PASS_BY_REFERENCE aarch64_pass_by_reference
17723 #undef TARGET_PREFERRED_RELOAD_CLASS
17724 #define TARGET_PREFERRED_RELOAD_CLASS aarch64_preferred_reload_class
17726 #undef TARGET_SCHED_REASSOCIATION_WIDTH
17727 #define TARGET_SCHED_REASSOCIATION_WIDTH aarch64_reassociation_width
17729 #undef TARGET_PROMOTED_TYPE
17730 #define TARGET_PROMOTED_TYPE aarch64_promoted_type
17732 #undef TARGET_SECONDARY_RELOAD
17733 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
17735 #undef TARGET_SHIFT_TRUNCATION_MASK
17736 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
17738 #undef TARGET_SETUP_INCOMING_VARARGS
17739 #define TARGET_SETUP_INCOMING_VARARGS aarch64_setup_incoming_varargs
17741 #undef TARGET_STRUCT_VALUE_RTX
17742 #define TARGET_STRUCT_VALUE_RTX aarch64_struct_value_rtx
17744 #undef TARGET_REGISTER_MOVE_COST
17745 #define TARGET_REGISTER_MOVE_COST aarch64_register_move_cost
17747 #undef TARGET_RETURN_IN_MEMORY
17748 #define TARGET_RETURN_IN_MEMORY aarch64_return_in_memory
17750 #undef TARGET_RETURN_IN_MSB
17751 #define TARGET_RETURN_IN_MSB aarch64_return_in_msb
17753 #undef TARGET_RTX_COSTS
17754 #define TARGET_RTX_COSTS aarch64_rtx_costs_wrapper
17756 #undef TARGET_SCALAR_MODE_SUPPORTED_P
17757 #define TARGET_SCALAR_MODE_SUPPORTED_P aarch64_scalar_mode_supported_p
17759 #undef TARGET_SCHED_ISSUE_RATE
17760 #define TARGET_SCHED_ISSUE_RATE aarch64_sched_issue_rate
17762 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
17763 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
17764 aarch64_sched_first_cycle_multipass_dfa_lookahead
17766 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
17767 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD \
17768 aarch64_first_cycle_multipass_dfa_lookahead_guard
17770 #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
17771 #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS \
17772 aarch64_get_separate_components
17774 #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
17775 #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB \
17776 aarch64_components_for_bb
17778 #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
17779 #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS \
17780 aarch64_disqualify_components
17782 #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
17783 #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
17784 aarch64_emit_prologue_components
17786 #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
17787 #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
17788 aarch64_emit_epilogue_components
17790 #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
17791 #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS \
17792 aarch64_set_handled_components
17794 #undef TARGET_TRAMPOLINE_INIT
17795 #define TARGET_TRAMPOLINE_INIT aarch64_trampoline_init
17797 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
17798 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P aarch64_use_blocks_for_constant_p
17800 #undef TARGET_VECTOR_MODE_SUPPORTED_P
17801 #define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
17803 #undef TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT
17804 #define TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT \
17805 aarch64_builtin_support_vector_misalignment
17807 #undef TARGET_ARRAY_MODE
17808 #define TARGET_ARRAY_MODE aarch64_array_mode
17810 #undef TARGET_ARRAY_MODE_SUPPORTED_P
17811 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
17813 #undef TARGET_VECTORIZE_ADD_STMT_COST
17814 #define TARGET_VECTORIZE_ADD_STMT_COST aarch64_add_stmt_cost
17816 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
17817 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
17818 aarch64_builtin_vectorization_cost
17820 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
17821 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
17823 #undef TARGET_VECTORIZE_BUILTINS
17824 #define TARGET_VECTORIZE_BUILTINS
17826 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
17827 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
17828 aarch64_builtin_vectorized_function
17830 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
17831 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
17832 aarch64_autovectorize_vector_sizes
17834 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
17835 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV \
17836 aarch64_atomic_assign_expand_fenv
17838 /* Section anchor support. */
17840 #undef TARGET_MIN_ANCHOR_OFFSET
17841 #define TARGET_MIN_ANCHOR_OFFSET -256
17843 /* Limit the maximum anchor offset to 4k-1, since that's the limit for a
17844 byte offset; we can do much more for larger data types, but have no way
17845 to determine the size of the access. We assume accesses are aligned. */
17846 #undef TARGET_MAX_ANCHOR_OFFSET
17847 #define TARGET_MAX_ANCHOR_OFFSET 4095
17849 #undef TARGET_VECTOR_ALIGNMENT
17850 #define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
17852 #undef TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
17853 #define TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT \
17854 aarch64_vectorize_preferred_vector_alignment
17855 #undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
17856 #define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
17857 aarch64_simd_vector_alignment_reachable
17859 /* vec_perm support. */
17861 #undef TARGET_VECTORIZE_VEC_PERM_CONST
17862 #define TARGET_VECTORIZE_VEC_PERM_CONST \
17863 aarch64_vectorize_vec_perm_const
17865 #undef TARGET_VECTORIZE_GET_MASK_MODE
17866 #define TARGET_VECTORIZE_GET_MASK_MODE aarch64_get_mask_mode
17867 #undef TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE
17868 #define TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE \
17869 aarch64_empty_mask_is_expensive
17870 #undef TARGET_PREFERRED_ELSE_VALUE
17871 #define TARGET_PREFERRED_ELSE_VALUE \
17872 aarch64_preferred_else_value
17874 #undef TARGET_INIT_LIBFUNCS
17875 #define TARGET_INIT_LIBFUNCS aarch64_init_libfuncs
17877 #undef TARGET_FIXED_CONDITION_CODE_REGS
17878 #define TARGET_FIXED_CONDITION_CODE_REGS aarch64_fixed_condition_code_regs
17880 #undef TARGET_FLAGS_REGNUM
17881 #define TARGET_FLAGS_REGNUM CC_REGNUM
17883 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
17884 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
17886 #undef TARGET_ASAN_SHADOW_OFFSET
17887 #define TARGET_ASAN_SHADOW_OFFSET aarch64_asan_shadow_offset
17889 #undef TARGET_LEGITIMIZE_ADDRESS
17890 #define TARGET_LEGITIMIZE_ADDRESS aarch64_legitimize_address
17892 #undef TARGET_SCHED_CAN_SPECULATE_INSN
17893 #define TARGET_SCHED_CAN_SPECULATE_INSN aarch64_sched_can_speculate_insn
17895 #undef TARGET_CAN_USE_DOLOOP_P
17896 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
17898 #undef TARGET_SCHED_ADJUST_PRIORITY
17899 #define TARGET_SCHED_ADJUST_PRIORITY aarch64_sched_adjust_priority
17901 #undef TARGET_SCHED_MACRO_FUSION_P
17902 #define TARGET_SCHED_MACRO_FUSION_P aarch64_macro_fusion_p
17904 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
17905 #define TARGET_SCHED_MACRO_FUSION_PAIR_P aarch_macro_fusion_pair_p
17907 #undef TARGET_SCHED_FUSION_PRIORITY
17908 #define TARGET_SCHED_FUSION_PRIORITY aarch64_sched_fusion_priority
17910 #undef TARGET_UNSPEC_MAY_TRAP_P
17911 #define TARGET_UNSPEC_MAY_TRAP_P aarch64_unspec_may_trap_p
17913 #undef TARGET_USE_PSEUDO_PIC_REG
17914 #define TARGET_USE_PSEUDO_PIC_REG aarch64_use_pseudo_pic_reg
17916 #undef TARGET_PRINT_OPERAND
17917 #define TARGET_PRINT_OPERAND aarch64_print_operand
17919 #undef TARGET_PRINT_OPERAND_ADDRESS
17920 #define TARGET_PRINT_OPERAND_ADDRESS aarch64_print_operand_address
17922 #undef TARGET_OPTAB_SUPPORTED_P
17923 #define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p
17925 #undef TARGET_OMIT_STRUCT_RETURN_REG
17926 #define TARGET_OMIT_STRUCT_RETURN_REG true
17928 #undef TARGET_DWARF_POLY_INDETERMINATE_VALUE
17929 #define TARGET_DWARF_POLY_INDETERMINATE_VALUE \
17930 aarch64_dwarf_poly_indeterminate_value
17932 /* The architecture reserves bits 0 and 1 so use bit 2 for descriptors. */
17933 #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
17934 #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 4
17936 #undef TARGET_HARD_REGNO_NREGS
17937 #define TARGET_HARD_REGNO_NREGS aarch64_hard_regno_nregs
17938 #undef TARGET_HARD_REGNO_MODE_OK
17939 #define TARGET_HARD_REGNO_MODE_OK aarch64_hard_regno_mode_ok
17941 #undef TARGET_MODES_TIEABLE_P
17942 #define TARGET_MODES_TIEABLE_P aarch64_modes_tieable_p
17944 #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
17945 #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
17946 aarch64_hard_regno_call_part_clobbered
17948 #undef TARGET_CONSTANT_ALIGNMENT
17949 #define TARGET_CONSTANT_ALIGNMENT aarch64_constant_alignment
17951 #undef TARGET_COMPUTE_PRESSURE_CLASSES
17952 #define TARGET_COMPUTE_PRESSURE_CLASSES aarch64_compute_pressure_classes
17954 #undef TARGET_CAN_CHANGE_MODE_CLASS
17955 #define TARGET_CAN_CHANGE_MODE_CLASS aarch64_can_change_mode_class
17957 #undef TARGET_SELECT_EARLY_REMAT_MODES
17958 #define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
17960 #undef TARGET_SPECULATION_SAFE_VALUE
17961 #define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value
17963 #if CHECKING_P
17964 #undef TARGET_RUN_TARGET_SELFTESTS
17965 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests