| blob | 98ef45714fd62678a39a10f705f89d322c083c5b |
1 /* Machine description for AArch64 architecture.
2 Copyright (C) 2009-2018 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
24 #define INCLUDE_STRING
75 /* This file should be included last. */
78 /* Defined for convenience. */
79 #define POINTER_BYTES (POINTER_SIZE / BITS_PER_UNIT)
81 /* Classifies an address.
83 ADDRESS_REG_IMM
84 A simple base register plus immediate offset.
86 ADDRESS_REG_WB
87 A base register indexed by immediate offset with writeback.
89 ADDRESS_REG_REG
90 A base register indexed by (optionally scaled) register.
92 ADDRESS_REG_UXTW
93 A base register indexed by (optionally scaled) zero-extended register.
95 ADDRESS_REG_SXTW
96 A base register indexed by (optionally scaled) sign-extended register.
98 ADDRESS_LO_SUM
99 A LO_SUM rtx with a base register and "LO12" symbol relocation.
101 ADDRESS_SYMBOLIC:
102 A constant symbolic address, in pc-relative literal pool. */
105 ADDRESS_REG_IMM,
106 ADDRESS_REG_WB,
107 ADDRESS_REG_REG,
108 ADDRESS_REG_UXTW,
109 ADDRESS_REG_SXTW,
110 ADDRESS_LO_SUM,
111 ADDRESS_SYMBOLIC
112 };
116 rtx base;
117 rtx offset;
118 poly_int64 const_offset;
121 };
123 /* Information about a legitimate vector immediate operand. */
124 struct simd_immediate_info
125 {
136 /* The mode of the elements. */
137 scalar_mode elt_mode;
139 /* The value of each element if all elements are the same, or the
140 first value if the constant is a series. */
141 rtx value;
143 /* The value of the step if the constant is a series, null otherwise. */
144 rtx step;
146 /* The instruction to use to move the immediate into a vector. */
147 insn_type insn;
149 /* The kind of shift modifier to use, and the number of bits to shift.
150 This is (LSL, 0) if no shift is needed. */
151 modifier_type modifier;
153 };
155 /* Construct a floating-point immediate in which each element has mode
156 ELT_MODE_IN and value VALUE_IN. */
157 inline simd_immediate_info
161 {}
163 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
164 and value VALUE_IN. The other parameters are as for the structure
165 fields. */
166 inline simd_immediate_info
173 {}
175 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
176 and where element I is equal to VALUE_IN + I * STEP_IN. */
177 inline simd_immediate_info
181 {}
183 /* The current code model. */
186 /* The number of 64-bit elements in an SVE vector. */
187 poly_uint16 aarch64_sve_vg;
189 #ifdef HAVE_AS_TLS
190 #undef TARGET_HAVE_TLS
191 #define TARGET_HAVE_TLS 1
192 #endif
196 const_tree,
205 const_tree type,
211 /* Major revision number of the ARM Architecture implemented by the target. */
214 /* The processor for which instructions should be scheduled. */
217 /* Mask to specify which instruction scheduling options should be used. */
220 /* Global flag for PC relative loads. */
223 /* Global flag for whether frame pointer is enabled. */
226 /* Support for command line parsing of boolean flags in the tuning
227 structures. */
228 struct aarch64_flag_desc
229 {
232 };
234 #define AARCH64_FUSION_PAIR(name, internal_name) \
235 { name, AARCH64_FUSE_##internal_name },
237 {
242 };
244 #define AARCH64_EXTRA_TUNING_OPTION(name, internal_name) \
245 { name, AARCH64_EXTRA_TUNE_##internal_name },
247 {
252 };
254 /* Tuning parameters. */
257 {
258 {
263 },
270 };
273 {
274 {
279 },
286 };
289 {
290 {
295 },
302 };
305 {
306 {
311 },
318 };
321 {
322 {
327 },
334 };
337 {
339 /* Avoid the use of slow int<->fp moves for spilling by setting
340 their cost higher than memmov_cost. */
344 };
347 {
349 /* Avoid the use of slow int<->fp moves for spilling by setting
350 their cost higher than memmov_cost. */
354 };
357 {
359 /* Avoid the use of slow int<->fp moves for spilling by setting
360 their cost higher than memmov_cost. */
364 };
367 {
369 /* Avoid the use of slow int<->fp moves for spilling by setting
370 their cost higher than memmov_cost (actual, 4 and 9). */
374 };
377 {
382 };
385 {
387 /* Avoid the use of slow int<->fp moves for spilling by setting
388 their cost higher than memmov_cost. */
392 };
395 {
397 /* Avoid the use of int<->fp moves for spilling. */
401 };
404 {
406 /* Avoid the use of int<->fp moves for spilling. */
410 };
412 /* Generic costs for vector insn classes. */
414 {
430 };
432 /* ThunderX costs for vector insn classes. */
434 {
450 };
452 /* Generic costs for vector insn classes. */
454 {
470 };
473 {
489 };
491 /* Generic costs for vector insn classes. */
493 {
509 };
511 /* Costs for vector insn classes for Vulcan. */
513 {
529 };
531 /* Generic costs for branch instructions. */
533 {
536 };
538 /* Generic approximation modes. */
540 {
543 AARCH64_APPROX_NONE /* recip_sqrt */
544 };
546 /* Approximation modes for Exynos M1. */
548 {
551 AARCH64_APPROX_ALL /* recip_sqrt */
552 };
554 /* Approximation modes for X-Gene 1. */
556 {
559 AARCH64_APPROX_ALL /* recip_sqrt */
560 };
562 /* Generic prefetch settings (which disable prefetch). */
564 {
572 };
575 {
583 };
586 {
594 };
597 {
605 };
608 {
616 };
619 {
627 };
630 {
651 &generic_prefetch_tune
652 };
655 {
677 &generic_prefetch_tune
678 };
681 {
703 &generic_prefetch_tune
704 };
707 {
729 &generic_prefetch_tune
730 };
733 {
755 &generic_prefetch_tune
756 };
759 {
781 &generic_prefetch_tune
782 };
787 {
808 &exynosm1_prefetch_tune
809 };
812 {
833 &thunderxt88_prefetch_tune
834 };
837 {
857 (AARCH64_EXTRA_TUNE_SLOW_UNALIGNED_LDPW
859 &thunderx_prefetch_tune
860 };
863 {
884 &generic_prefetch_tune
885 };
888 {
910 &qdf24xx_prefetch_tune
911 };
913 /* Tuning structure for the Qualcomm Saphira core. Default to falkor values
914 for now. */
916 {
938 &generic_prefetch_tune
939 };
942 {
964 &thunderx2t99_prefetch_tune
965 };
967 /* Support for fine-grained override of the tuning structures. */
968 struct aarch64_tuning_override_function
969 {
972 };
977 static const struct aarch64_tuning_override_function
978 aarch64_tuning_override_functions[] =
979 {
983 };
985 /* A processor implementing AArch64. */
986 struct processor
987 {
995 };
997 /* Architectures implementing AArch64. */
999 {
1000 #define AARCH64_ARCH(NAME, CORE, ARCH_IDENT, ARCH_REV, FLAGS) \
1001 {NAME, CORE, CORE, AARCH64_ARCH_##ARCH_IDENT, ARCH_REV, FLAGS, NULL},
1004 };
1006 /* Processor cores implementing AArch64. */
1008 {
1009 #define AARCH64_CORE(NAME, IDENT, SCHED, ARCH, FLAGS, COSTS, IMP, PART, VARIANT) \
1010 {NAME, IDENT, SCHED, AARCH64_ARCH_##ARCH, \
1011 all_architectures[AARCH64_ARCH_##ARCH].architecture_version, \
1012 FLAGS, &COSTS##_tunings},
1017 };
1020 /* Target specification. These are populated by the -march, -mtune, -mcpu
1021 handling code or by target attributes. */
1026 /* The current tuning set. */
1029 #define AARCH64_CPU_DEFAULT_FLAGS ((selected_cpu) ? selected_cpu->flags : 0)
1031 /* An ISA extension in the co-processor and main instruction set space. */
1032 struct aarch64_option_extension
1033 {
1037 };
1040 {
1044 }
1045 aarch64_cc;
1047 #define AARCH64_INVERSE_CONDITION_CODE(X) ((aarch64_cc) (((int) X) ^ 1))
1049 /* The condition codes of the processor, and the inverse function. */
1051 {
1054 };
1056 /* Generate code to enable conditional branches in functions over 1 MiB. */
1060 {
1077 }
1079 void
1081 {
1085 else
1087 }
1089 /* Implement TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS.
1090 The register allocator chooses POINTER_AND_FP_REGS if FP_REGS and
1091 GENERAL_REGS have the same cost - even if POINTER_AND_FP_REGS has a much
1092 higher cost. POINTER_AND_FP_REGS is also used if the cost of both FP_REGS
1093 and GENERAL_REGS is lower than the memory cost (in this case the best class
1094 is the lowest cost one). Using POINTER_AND_FP_REGS irrespectively of its
1095 cost results in bad allocations with many redundant int<->FP moves which
1096 are expensive on various cores.
1097 To avoid this we don't allow POINTER_AND_FP_REGS as the allocno class, but
1098 force a decision between FP_REGS and GENERAL_REGS. We use the allocno class
1099 if it isn't POINTER_AND_FP_REGS. Similarly, use the best class if it isn't
1100 POINTER_AND_FP_REGS. Otherwise set the allocno class depending on the mode.
1101 The result of this is that it is no longer inefficient to have a higher
1102 memory move cost than the register move cost.
1103 */
1105 static reg_class_t
1107 reg_class_t best_class)
1108 {
1109 machine_mode mode;
1121 }
1123 static unsigned int
1125 {
1129 }
1131 /* Return the reassociation width of treeop OPC with mode MODE. */
1132 static int
1134 {
1139 /* Avoid reassociating floating point addition so we emit more FMAs. */
1143 }
1145 /* Provide a mapping from gcc register numbers to dwarf register numbers. */
1146 unsigned
1148 {
1160 /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
1161 equivalent DWARF register. */
1163 }
1165 /* Return true if MODE is any of the Advanced SIMD structure modes. */
1166 static bool
1168 {
1171 }
1173 /* Return true if MODE is an SVE predicate mode. */
1174 static bool
1176 {
1182 }
1184 /* Three mutually-exclusive flags describing a vector or predicate type. */
1188 /* Can be used in combination with VEC_ADVSIMD or VEC_SVE_DATA to indicate
1189 a structure of 2, 3 or 4 vectors. */
1191 /* Useful combinations of the above. */
1195 /* Return a set of flags describing the vector properties of mode MODE.
1196 Ignore modes that are not supported by the current target. */
1197 static unsigned int
1199 {
1215 {
1217 {
1224 }
1226 /* This includes V1DF but not V1DI (which doesn't exist). */
1231 }
1234 }
1236 /* Return true if MODE is any of the data vector modes, including
1237 structure modes. */
1238 static bool
1240 {
1242 }
1244 /* Return true if MODE is an SVE data vector mode; either a single vector
1245 or a structure of vectors. */
1246 static bool
1248 {
1250 }
1252 /* Implement target hook TARGET_ARRAY_MODE. */
1253 static opt_machine_mode
1255 {
1262 }
1264 /* Implement target hook TARGET_ARRAY_MODE_SUPPORTED_P. */
1265 static bool
1268 {
1276 }
1278 /* Return the SVE predicate mode to use for elements that have
1279 ELEM_NBYTES bytes, if such a mode exists. */
1281 opt_machine_mode
1283 {
1285 {
1294 }
1296 }
1298 /* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
1300 static opt_machine_mode
1302 {
1304 {
1306 machine_mode pred_mode;
1309 }
1312 }
1314 /* Implement TARGET_HARD_REGNO_NREGS. */
1316 static unsigned int
1318 {
1319 /* ??? Logically we should only need to provide a value when
1320 HARD_REGNO_MODE_OK says that the combination is valid,
1321 but at the moment we need to handle all modes. Just ignore
1322 any runtime parts for registers that can't store them. */
1325 {
1338 }
1340 }
1342 /* Implement TARGET_HARD_REGNO_MODE_OK. */
1344 static bool
1346 {
1351 /* This must have the same size as _Unwind_Word. */
1362 /* The purpose of comparing with ptr_mode is to support the
1363 global register variable associated with the stack pointer
1364 register via the syntax of asm ("wsp") in ILP32. */
1374 {
1377 else
1379 }
1382 }
1384 /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The callee only saves
1385 the lower 64 bits of a 128-bit register. Tell the compiler the callee
1386 clobbers the top 64 bits when restoring the bottom 64 bits. */
1388 static bool
1390 {
1392 }
1394 /* Implement REGMODE_NATURAL_SIZE. */
1395 poly_uint64
1397 {
1398 /* The natural size for SVE data modes is one SVE data vector,
1399 and similarly for predicates. We can't independently modify
1400 anything smaller than that. */
1401 /* ??? For now, only do this for variable-width SVE registers.
1402 Doing it for constant-sized registers breaks lower-subreg.c. */
1403 /* ??? And once that's fixed, we should probably have similar
1404 code for Advanced SIMD. */
1406 {
1412 }
1414 }
1416 /* Implement HARD_REGNO_CALLER_SAVE_MODE. */
1417 machine_mode
1419 machine_mode mode)
1420 {
1421 /* The predicate mode determines which bits are significant and
1422 which are "don't care". Decreasing the number of lanes would
1423 lose data while increasing the number of lanes would make bits
1424 unnecessarily significant. */
1429 else
1431 }
1433 /* Implement TARGET_CONSTANT_ALIGNMENT. Make strings word-aligned so
1434 that strcpy from constants will be faster. */
1436 static HOST_WIDE_INT
1438 {
1442 }
1444 /* Return true if calls to DECL should be treated as
1445 long-calls (ie called via a register). */
1446 static bool
1448 {
1450 }
1452 /* Return true if calls to symbol-ref SYM should be treated as
1453 long-calls (ie called via a register). */
1454 bool
1456 {
1458 }
1460 /* Return true if calls to symbol-ref SYM should not go through
1461 plt stubs. */
1463 bool
1465 {
1469 && decl
1470 && (!flag_plt
1476 }
1478 /* Return true if the offsets to a zero/sign-extract operation
1479 represent an expression that matches an extend operation. The
1480 operands represent the paramters from
1482 (extract:MODE (mult (reg) (MULT_IMM)) (EXTRACT_IMM) (const_int 0)). */
1483 bool
1485 rtx extract_imm)
1486 {
1503 }
1505 /* Emit an insn that's a simple single-set. Both the operands must be
1506 known to be valid. */
1509 {
1511 }
1513 /* X and Y are two things to compare using CODE. Emit the compare insn and
1514 return the rtx for register 0 in the proper mode. */
1515 rtx
1517 {
1523 }
1525 /* Build the SYMBOL_REF for __tls_get_addr. */
1529 rtx
1531 {
1535 }
1537 /* Return the TLS model to use for ADDR. */
1539 static enum tls_model
1541 {
1544 {
1545 poly_int64 addend;
1549 }
1554 }
1556 /* We'll allow lo_sum's in addresses in our legitimate addresses
1557 so that combine would take care of combining addresses where
1558 necessary, but for generation purposes, we'll generate the address
1559 as :
1560 RTL Absolute
1561 tmp = hi (symbol_ref); adrp x1, foo
1562 dest = lo_sum (tmp, symbol_ref); add dest, x1, :lo_12:foo
1563 nop
1565 PIC TLS
1566 adrp x1, :got:foo adrp tmp, :tlsgd:foo
1567 ldr x1, [:got_lo12:foo] add dest, tmp, :tlsgd_lo12:foo
1568 bl __tls_get_addr
1569 nop
1571 Load TLS symbol, depending on TLS mechanism and TLS access model.
1573 Global Dynamic - Traditional TLS:
1574 adrp tmp, :tlsgd:imm
1575 add dest, tmp, #:tlsgd_lo12:imm
1576 bl __tls_get_addr
1578 Global Dynamic - TLS Descriptors:
1579 adrp dest, :tlsdesc:imm
1580 ldr tmp, [dest, #:tlsdesc_lo12:imm]
1581 add dest, dest, #:tlsdesc_lo12:imm
1582 blr tmp
1583 mrs tp, tpidr_el0
1584 add dest, dest, tp
1586 Initial Exec:
1587 mrs tp, tpidr_el0
1588 adrp tmp, :gottprel:imm
1589 ldr dest, [tmp, #:gottprel_lo12:imm]
1590 add dest, dest, tp
1592 Local Exec:
1593 mrs tp, tpidr_el0
1594 add t0, tp, #:tprel_hi12:imm, lsl #12
1595 add t0, t0, #:tprel_lo12_nc:imm
1596 */
1598 static void
1601 {
1603 {
1605 {
1606 /* In ILP32, the mode of dest can be either SImode or DImode. */
1618 }
1625 {
1628 rtx insn;
1629 rtx mem;
1631 /* NOTE: pic_offset_table_rtx can be NULL_RTX, because we can reach
1632 here before rtl expand. Tree IVOPT will generate rtl pattern to
1633 decide rtx costs, in which case pic_offset_table_rtx is not
1634 initialized. For that case no need to generate the first adrp
1635 instruction as the final cost for global variable access is
1636 one instruction. */
1638 {
1639 /* -fpic for -mcmodel=small allow 32K GOT table size (but we are
1640 using the page base as GOT base, the first page may be wasted,
1641 in the worst scenario, there is only 28K space for GOT).
1643 The generate instruction sequence for accessing global variable
1644 is:
1646 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym]
1648 Only one instruction needed. But we must initialize
1649 pic_offset_table_rtx properly. We generate initialize insn for
1650 every global access, and allow CSE to remove all redundant.
1652 The final instruction sequences will look like the following
1653 for multiply global variables access.
1655 adrp pic_offset_table_rtx, _GLOBAL_OFFSET_TABLE_
1657 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym1]
1658 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym2]
1659 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym3]
1660 ... */
1669 }
1672 {
1675 else
1679 }
1680 else
1681 {
1686 }
1688 /* The operand is expected to be MEM. Whenever the related insn
1689 pattern changed, above code which calculate mem should be
1690 updated. */
1696 }
1699 {
1700 /* In ILP32, the mode of dest can be either SImode or DImode,
1701 while the got entry is always of SImode size. The mode of
1702 dest depends on how dest is used: if dest is assigned to a
1703 pointer (e.g. in the memory), it has SImode; it may have
1704 DImode if dest is dereferenced to access the memeory.
1705 This is why we have to handle three different ldr_got_small
1706 patterns here (two patterns for ILP32). */
1708 rtx insn;
1709 rtx mem;
1718 {
1721 else
1725 }
1726 else
1727 {
1732 }
1739 }
1742 {
1750 else
1758 }
1761 {
1764 rtx tp;
1768 /* In ILP32, the got entry is always of SImode size. Unlike
1769 small GOT, the dest is fixed at reg 0. */
1772 else
1783 }
1786 {
1787 /* In ILP32, the mode of dest can be either SImode or DImode,
1788 while the got entry is always of SImode size. The mode of
1789 dest depends on how dest is used: if dest is assigned to a
1790 pointer (e.g. in the memory), it has SImode; it may have
1791 DImode if dest is dereferenced to access the memeory.
1792 This is why we have to handle three different tlsie_small
1793 patterns here (two patterns for ILP32). */
1799 {
1802 else
1803 {
1806 }
1807 }
1808 else
1809 {
1812 }
1818 }
1824 {
1832 {
1855 }
1860 }
1867 {
1872 {
1875 else
1876 {
1879 }
1880 }
1881 else
1882 {
1885 }
1890 }
1894 }
1895 }
1897 /* Emit a move from SRC to DEST. Assume that the move expanders can
1898 handle all moves if !can_create_pseudo_p (). The distinction is
1899 important because, unlike emit_move_insn, the move expanders know
1900 how to force Pmode objects into the constant pool even when the
1901 constant pool address is not itself legitimate. */
1902 static rtx
1904 {
1908 }
1910 /* Apply UNOPTAB to OP and store the result in DEST. */
1912 static void
1914 {
1918 }
1920 /* Apply BINOPTAB to OP0 and OP1 and store the result in DEST. */
1922 static void
1924 {
1926 OPTAB_DIRECT);
1929 }
1931 /* Split a 128-bit move operation into two 64-bit move operations,
1932 taking care to handle partial overlap of register to register
1933 copies. Special cases are needed when moving between GP regs and
1934 FP regs. SRC can be a register, constant or memory; DST a register
1935 or memory. If either operand is memory it must not have any side
1936 effects. */
1937 void
1939 {
1950 {
1954 /* Handle FP <-> GP regs. */
1956 {
1961 {
1964 }
1965 else
1966 {
1969 }
1971 }
1973 {
1978 {
1981 }
1982 else
1983 {
1986 }
1988 }
1989 }
1996 /* At most one pairing may overlap. */
1998 {
2001 }
2002 else
2003 {
2006 }
2007 }
2009 bool
2011 {
2014 }
2016 /* Split a complex SIMD combine. */
2018 void
2020 {
2032 {
2056 }
2060 }
2062 /* Split a complex SIMD move. */
2064 void
2066 {
2073 {
2079 {
2103 }
2107 }
2108 }
2110 bool
2113 {
2117 }
2120 static rtx
2122 {
2125 else
2126 {
2130 }
2131 }
2133 /* Return true if we can move VALUE into a register using a single
2134 CNT[BHWD] instruction. */
2136 static bool
2138 {
2140 /* The coefficient must be [1, 16] * {2, 4, 8, 16}. */
2145 }
2147 /* Likewise for rtx X. */
2149 bool
2151 {
2152 poly_int64 value;
2154 }
2156 /* Return the asm string for an instruction with a CNT-like vector size
2157 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2158 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2159 first part of the operands template (the part that comes before the
2160 vector size itself). FACTOR is the number of quadwords.
2161 NELTS_PER_VQ, if nonzero, is the number of elements in each quadword.
2162 If it is zero, we can use any element size. */
2168 {
2172 /* There is some overlap in the ranges of the four CNT instructions.
2173 Here we always use the smallest possible element size, so that the
2174 multiplier is 1 whereever possible. */
2185 else
2190 }
2192 /* Return the asm string for an instruction with a CNT-like vector size
2193 operand (a vector pattern followed by a multiplier in the range [1, 16]).
2194 PREFIX is the mnemonic without the size suffix and OPERANDS is the
2195 first part of the operands template (the part that comes before the
2196 vector size itself). X is the value of the vector size operand,
2197 as a polynomial integer rtx. */
2201 rtx x)
2202 {
2207 }
2209 /* Return true if we can add VALUE to a register using a single ADDVL
2210 or ADDPL instruction. */
2212 static bool
2214 {
2218 /* FACTOR counts VG / 2, so a value of 2 is one predicate width
2219 and a value of 16 is one vector width. */
2222 }
2224 /* Likewise for rtx X. */
2226 bool
2228 {
2229 poly_int64 value;
2232 }
2234 /* Return the asm string for adding ADDVL or ADDPL immediate X to operand 1
2235 and storing the result in operand 0. */
2239 {
2244 /* Use INC or DEC if possible. */
2246 {
2253 }
2258 else
2261 }
2263 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2264 instruction. If it is, store the number of elements in each vector
2265 quadword in *NELTS_PER_VQ_OUT (if nonnull) and store the multiplication
2266 factor in *FACTOR_OUT (if nonnull). */
2268 bool
2271 {
2272 rtx elt;
2273 poly_int64 value;
2281 /* There's no vector INCB. */
2288 /* The coefficient must be [1, 16] * NELTS_PER_VQ. */
2298 }
2300 /* Return true if X is a valid immediate for an SVE vector INC or DEC
2301 instruction. */
2303 bool
2305 {
2307 }
2309 /* Return the asm template for an SVE vector INC or DEC instruction.
2310 OPERANDS gives the operands before the vector count and X is the
2311 value of the vector count operand itself. */
2315 {
2322 nelts_per_vq);
2323 else
2325 nelts_per_vq);
2326 }
2328 static int
2330 scalar_int_mode mode)
2331 {
2340 {
2344 }
2346 /* Check to see if the low 32 bits are either 0xffffXXXX or 0xXXXXffff
2347 (with XXXX non-zero). In that case check to see if the move can be done in
2348 a smaller mode. */
2353 {
2357 /* Check if we have to emit a second instruction by checking to see
2358 if any of the upper 32 bits of the original DI mode value is set. */
2369 }
2372 {
2374 {
2379 else
2382 }
2384 }
2386 /* Remaining cases are all for DImode. */
2395 {
2396 /* Try emitting a bitmask immediate with a movk replacing 16 bits.
2397 For a 64-bit bitmask try whether changing 16 bits to all ones or
2398 zeroes creates a valid bitmask. To check any repeated bitmask,
2399 try using 16 bits from the other 32-bit half of val. */
2402 {
2413 }
2415 {
2417 {
2421 }
2423 }
2424 }
2426 /* Generate 2-4 instructions, skipping 16 bits of all zeroes or ones which
2427 are emitted by the initial mov. If one_match > zero_match, skip set bits,
2428 otherwise skip zero bits. */
2440 {
2446 num_insns ++;
2447 }
2450 }
2452 /* Return whether imm is a 128-bit immediate which is simple enough to
2453 expand inline. */
2454 bool
2456 {
2467 }
2470 /* Return the number of temporary registers that aarch64_add_offset_1
2471 would need to add OFFSET to a register. */
2473 static unsigned int
2475 {
2477 }
2479 /* A subroutine of aarch64_add_offset. Set DEST to SRC + OFFSET for
2480 a non-polynomial OFFSET. MODE is the mode of the addition.
2481 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
2482 be set and CFA adjustments added to the generated instructions.
2484 TEMP1, if nonnull, is a register of mode MODE that can be used as a
2485 temporary if register allocation is already complete. This temporary
2486 register may overlap DEST but must not overlap SRC. If TEMP1 is known
2487 to hold abs (OFFSET), EMIT_MOVE_IMM can be set to false to avoid emitting
2488 the immediate again.
2490 Since this function may be used to adjust the stack pointer, we must
2491 ensure that it cannot cause transient stack deallocation (for example
2492 by first incrementing SP and then decrementing when adjusting by a
2493 large immediate). */
2495 static void
2499 {
2507 {
2509 {
2512 }
2514 }
2516 /* Single instruction adjustment. */
2518 {
2522 }
2524 /* Emit 2 additions/subtractions if the adjustment is less than 24 bits
2525 and either:
2527 a) the offset cannot be loaded by a 16-bit move or
2528 b) there is no spare register into which we can move it. */
2532 {
2541 }
2543 /* Emit a move immediate if required and an addition/subtraction. */
2545 {
2548 }
2553 {
2557 }
2558 }
2560 /* Return the number of temporary registers that aarch64_add_offset
2561 would need to move OFFSET into a register or add OFFSET to a register;
2562 ADD_P is true if we want the latter rather than the former. */
2564 static unsigned int
2566 {
2567 /* This follows the same structure as aarch64_add_offset. */
2576 /* Need one register for the ADDVL/ADDPL result. */
2579 {
2582 /* Need one register for the CNT result and one for the multiplication
2583 factor. If necessary, the second temporary can be reused for the
2584 constant part of the offset. */
2586 /* Need one register for the CNT result (which might then
2587 be shifted). */
2589 }
2591 }
2593 /* If X can be represented as a poly_int64, return the number
2594 of temporaries that are required to add it to a register.
2595 Return -1 otherwise. */
2597 int
2599 {
2600 poly_int64 offset;
2604 }
2606 /* Set DEST to SRC + OFFSET. MODE is the mode of the addition.
2607 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
2608 be set and CFA adjustments added to the generated instructions.
2610 TEMP1, if nonnull, is a register of mode MODE that can be used as a
2611 temporary if register allocation is already complete. This temporary
2612 register may overlap DEST if !FRAME_RELATED_P but must not overlap SRC.
2613 If TEMP1 is known to hold abs (OFFSET), EMIT_MOVE_IMM can be set to
2614 false to avoid emitting the immediate again.
2616 TEMP2, if nonnull, is a second temporary register that doesn't
2617 overlap either DEST or REG.
2619 Since this function may be used to adjust the stack pointer, we must
2620 ensure that it cannot cause transient stack deallocation (for example
2621 by first incrementing SP and then decrementing when adjusting by a
2622 large immediate). */
2624 static void
2628 {
2632 || !frame_related_p
2636 /* Try using ADDVL or ADDPL to add the whole value. */
2638 {
2643 }
2645 /* Coefficient 1 is multiplied by the number of 128-bit blocks in an
2646 SVE vector register, over and above the minimum size of 128 bits.
2647 This is equivalent to half the value returned by CNTD with a
2648 vector shape of ALL. */
2652 /* Try using ADDVL or ADDPL to add the VG-based part. */
2656 {
2659 {
2663 }
2664 else
2665 {
2670 }
2671 }
2672 /* Otherwise use a CNT-based sequence. */
2674 {
2675 /* Use a subtraction if we have a negative factor. */
2678 {
2681 }
2683 /* Calculate CNTD * FACTOR / 2. First try to fold the division
2684 into the multiplication. */
2685 rtx val;
2688 /* Use a right shift by 1. */
2690 else
2694 {
2696 {
2697 /* "CNTB Xn, ALL, MUL #FACTOR" is out of range, so calculate
2698 the value with the minimum multiplier and shift it into
2699 position. */
2703 }
2705 }
2706 else
2707 {
2708 /* Use CNTD, then multiply it by FACTOR. */
2712 /* Go back to using a negative multiplication factor if we have
2713 no register from which to subtract. */
2715 {
2718 }
2722 }
2725 {
2726 /* Multiply by 1 << SHIFT. */
2729 }
2731 {
2732 /* Divide by 2. */
2735 }
2737 /* Calculate SRC +/- CNTD * FACTOR / 2. */
2739 {
2742 }
2744 {
2747 }
2750 {
2753 {
2757 poly_offset)));
2758 }
2762 }
2763 else
2764 {
2768 }
2771 }
2775 }
2777 /* Like aarch64_add_offset, but the offset is given as an rtx rather
2778 than a poly_int64. */
2780 void
2783 {
2786 }
2788 /* Add DELTA to the stack pointer, marking the instructions frame-related.
2789 TEMP1 is available as a temporary if nonnull. EMIT_MOVE_IMM is false
2790 if TEMP1 already contains abs (DELTA). */
2794 {
2797 }
2799 /* Subtract DELTA from the stack pointer, marking the instructions
2800 frame-related if FRAME_RELATED_P. TEMP1 is available as a temporary
2801 if nonnull. */
2805 {
2808 }
2810 /* Set DEST to (vec_series BASE STEP). */
2812 static void
2814 {
2818 /* Each operand can be a register or an immediate in the range [-16, 15]. */
2825 }
2827 /* Try to duplicate SRC into SVE register DEST, given that SRC is an
2828 integer of mode INT_MODE. Return true on success. */
2830 static bool
2832 rtx src)
2833 {
2834 /* If the constant is smaller than 128 bits, we can do the move
2835 using a vector of SRC_MODEs. */
2837 {
2844 }
2846 /* Use LD1RQ[BHWD] to load the 128 bits from memory. */
2851 /* Make sure that the address is legitimate. */
2853 {
2856 }
2865 }
2867 /* Expand a move of general CONST_VECTOR SRC into DEST, given that it
2868 isn't a simple duplicate or series. */
2870 static void
2872 {
2879 {
2880 /* The constant is a repeating seqeuence of at least two elements,
2881 where the repeating elements occupy no more than 128 bits.
2882 Get an integer representation of the replicated value. */
2883 scalar_int_mode int_mode;
2885 /* For now, always use LD1RQ to load the value on big-endian
2886 targets, since the handling of smaller integers includes a
2887 subreg that is semantically an element reverse. */
2889 else
2890 {
2894 }
2899 }
2901 /* Expand each pattern individually. */
2902 rtx_vector_builder builder;
2905 {
2910 }
2912 /* Use permutes to interleave the separate vectors. */
2914 {
2917 {
2922 }
2923 }
2925 }
2927 /* Set DEST to immediate IMM. For SVE vector modes, GEN_VEC_DUPLICATE
2928 is a pattern that can be used to set DEST to a replicated scalar
2929 element. */
2931 void
2934 {
2937 /* Check on what type of symbol it is. */
2938 scalar_int_mode int_mode;
2944 {
2945 rtx mem;
2946 poly_int64 offset;
2947 HOST_WIDE_INT const_offset;
2950 /* If we have (const (plus symbol offset)), separate out the offset
2951 before we start classifying the symbol. */
2954 /* We must always add an offset involving VL separately, rather than
2955 folding it into the relocation. */
2957 {
2960 else
2961 {
2962 /* Do arithmetic on 32-bit values if the result is smaller
2963 than that. */
2965 {
2966 /* It is invalid to do symbol calculations in modes
2967 narrower than SImode. */
2971 }
2973 {
2977 }
2978 else
2981 }
2983 }
2987 {
2991 {
2997 }
3002 /* If we aren't generating PC relative literals, then
3003 we need to expand the literal pool access carefully.
3004 This is something that needs to be done in a number
3005 of places, so could well live as a separate function. */
3007 {
3014 }
3031 {
3037 }
3038 /* FALLTHRU */
3051 }
3052 }
3055 {
3063 {
3064 /* If the constant is out of range of an SVE vector move,
3065 load it from memory if we can, otherwise move it into
3066 a register and use a DUP. */
3072 {
3075 }
3077 }
3081 else
3082 {
3086 }
3089 }
3093 }
3095 /* Emit an SVE predicated move from SRC to DEST. PRED is a predicate
3096 that is known to contain PTRUE. */
3098 void
3100 {
3103 UNSPEC_MERGE_PTRUE)));
3104 }
3106 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one
3107 operand is in memory. In this case we need to use the predicated LD1
3108 and ST1 instead of LDR and STR, both for correctness on big-endian
3109 targets and because LD1 and ST1 support a wider range of addressing modes.
3110 PRED_MODE is the mode of the predicate.
3112 See the comment at the head of aarch64-sve.md for details about the
3113 big-endian handling. */
3115 void
3117 {
3122 {
3126 else
3129 }
3131 }
3133 /* Called only on big-endian targets. See whether an SVE vector move
3134 from SRC to DEST is effectively a REV[BHW] instruction, because at
3135 least one operand is a subreg of an SVE vector that has wider or
3136 narrower elements. Return true and emit the instruction if so.
3138 For example:
3140 (set (reg:VNx8HI R1) (subreg:VNx8HI (reg:VNx16QI R2) 0))
3142 represents a VIEW_CONVERT between the following vectors, viewed
3143 in memory order:
3145 R2: { [0].high, [0].low, [1].high, [1].low, ... }
3146 R1: { [0], [1], [2], [3], ... }
3148 The high part of lane X in R2 should therefore correspond to lane X*2
3149 of R1, but the register representations are:
3151 msb lsb
3152 R2: ...... [1].high [1].low [0].high [0].low
3153 R1: ...... [3] [2] [1] [0]
3155 where the low part of lane X in R2 corresponds to lane X*2 in R1.
3156 We therefore need a reverse operation to swap the high and low values
3157 around.
3159 This is purely an optimization. Without it we would spill the
3160 subreg operand to the stack in one mode and reload it in the
3161 other mode, which has the same effect as the REV. */
3163 bool
3165 {
3172 /* The optimization handles two single SVE REGs with different element
3173 sizes. */
3182 /* Generate *aarch64_sve_mov<mode>_subreg_be. */
3185 UNSPEC_REV_SUBREG);
3188 }
3190 /* Return a copy of X with mode MODE, without changing its other
3191 attributes. Unlike gen_lowpart, this doesn't care whether the
3192 mode change is valid. */
3194 static rtx
3196 {
3203 }
3205 /* Split a *aarch64_sve_mov<mode>_subreg_be pattern with the given
3206 operands. */
3208 void
3210 {
3211 /* Decide which REV operation we need. The mode with narrower elements
3212 determines the mode of the operands and the mode with the wider
3213 elements determines the reverse width. */
3228 else
3232 /* Emit:
3234 (set DEST (unspec [PTRUE (unspec [SRC] UNSPEC_REV<nn>)]
3235 UNSPEC_MERGE_PTRUE))
3237 with the appropriate modes. */
3243 UNSPEC_MERGE_PTRUE);
3245 }
3247 static bool
3249 tree exp ATTRIBUTE_UNUSED)
3250 {
3251 /* Currently, always true. */
3253 }
3255 /* Implement TARGET_PASS_BY_REFERENCE. */
3257 static bool
3259 machine_mode mode,
3260 const_tree type,
3262 {
3263 HOST_WIDE_INT size;
3264 machine_mode dummymode;
3267 /* GET_MODE_SIZE (BLKmode) is useless since it is 0. */
3270 else
3271 /* No frontends can create types with variable-sized modes, so we
3272 shouldn't be asked to pass or return them. */
3275 /* Aggregates are passed by reference based on their size. */
3277 {
3279 }
3281 /* Variable sized arguments are always returned by reference. */
3285 /* Can this be a candidate to be passed in fp/simd register(s)? */
3288 NULL))
3291 /* Arguments which are variable sized or larger than 2 registers are
3292 passed by reference unless they are a homogenous floating point
3293 aggregate. */
3295 }
3297 /* Return TRUE if VALTYPE is padded to its least significant bits. */
3298 static bool
3300 {
3301 machine_mode dummy_mode;
3304 /* Never happens in little-endian mode. */
3308 /* Only composite types smaller than or equal to 16 bytes can
3309 be potentially returned in registers. */
3315 /* But not a composite that is an HFA (Homogeneous Floating-point Aggregate)
3316 or an HVA (Homogeneous Short-Vector Aggregate); such a special composite
3317 is always passed/returned in the least significant bits of fp/simd
3318 register(s). */
3324 }
3326 /* Implement TARGET_FUNCTION_VALUE.
3327 Define how to find the value returned by a function. */
3329 static rtx
3332 {
3333 machine_mode mode;
3336 machine_mode ag_mode;
3343 {
3347 {
3350 }
3351 }
3355 {
3357 {
3360 }
3361 else
3362 {
3364 rtx par;
3368 {
3373 }
3375 }
3376 }
3377 else
3379 }
3381 /* Implements TARGET_FUNCTION_VALUE_REGNO_P.
3382 Return true if REGNO is the number of a hard register in which the values
3383 of called function may come back. */
3385 static bool
3387 {
3388 /* Maximum of 16 bytes can be returned in the general registers. Examples
3389 of 16-byte return values are: 128-bit integers and 16-byte small
3390 structures (excluding homogeneous floating-point aggregates). */
3394 /* Up to four fp/simd registers can return a function value, e.g. a
3395 homogeneous floating-point aggregate having four members. */
3400 }
3402 /* Implement TARGET_RETURN_IN_MEMORY.
3404 If the type T of the result of a function is such that
3405 void func (T arg)
3406 would require that arg be passed as a value in a register (or set of
3407 registers) according to the parameter passing rules, then the result
3408 is returned in the same registers as would be used for such an
3409 argument. */
3411 static bool
3413 {
3414 HOST_WIDE_INT size;
3415 machine_mode ag_mode;
3421 /* Simple scalar types always returned in registers. */
3425 type,
3428 NULL))
3431 /* Types larger than 2 registers returned in memory. */
3434 }
3436 static bool
3439 {
3442 type,
3444 nregs,
3445 NULL);
3446 }
3448 /* Given MODE and TYPE of a function argument, return the alignment in
3449 bits. The idea is to suppress any stronger alignment requested by
3450 the user and opt for the natural alignment (specified in AAPCS64 \S 4.1).
3451 This is a helper function for local use only. */
3453 static unsigned int
3455 {
3476 }
3478 /* Layout a function argument according to the AAPCS64 rules. The rule
3479 numbers refer to the rule numbers in the AAPCS64. */
3481 static void
3483 const_tree type,
3485 {
3489 HOST_WIDE_INT size;
3491 /* We need to do this once per argument. */
3497 /* Size in bytes, rounded to the nearest multiple of 8 bytes. */
3500 else
3501 /* No frontends can create types with variable-sized modes, so we
3502 shouldn't be asked to pass or return them. */
3508 mode,
3509 type,
3512 /* allocate_ncrn may be false-positive, but allocate_nvrn is quite reliable.
3513 The following code thus handles passing by SIMD/FP registers first. */
3517 /* C1 - C5 for floating point, homogenous floating point aggregates (HFA)
3518 and homogenous short-vector aggregates (HVA). */
3520 {
3525 {
3528 {
3531 }
3532 else
3533 {
3534 rtx par;
3538 {
3541 rtx offset = gen_int_mode
3545 }
3547 }
3549 }
3550 else
3551 {
3552 /* C.3 NSRN is set to 8. */
3555 }
3556 }
3561 /* C6 - C9. though the sign and zero extension semantics are
3562 handled elsewhere. This is the case where the argument fits
3563 entirely general registers. */
3565 {
3569 /* C.8 if the argument has an alignment of 16 then the NGRN is
3570 rounded up to the next even number. */
3573 /* The == 16 * BITS_PER_UNIT instead of >= 16 * BITS_PER_UNIT
3574 comparison is there because for > 16 * BITS_PER_UNIT
3575 alignment nregs should be > 2 and therefore it should be
3576 passed by reference rather than value. */
3578 {
3581 }
3583 /* NREGS can be 0 when e.g. an empty structure is to be passed.
3584 A reg is still generated for it, but the caller should be smart
3585 enough not to use it. */
3588 else
3589 {
3590 rtx par;
3595 {
3600 }
3602 }
3606 }
3608 /* C.11 */
3611 /* The argument is passed on stack; record the needed number of words for
3612 this argument and align the total size if necessary. */
3613 on_stack:
3620 }
3622 /* Implement TARGET_FUNCTION_ARG. */
3624 static rtx
3627 {
3636 }
3638 void
3640 const_tree fntype ATTRIBUTE_UNUSED,
3641 rtx libname ATTRIBUTE_UNUSED,
3642 const_tree fndecl ATTRIBUTE_UNUSED,
3644 {
3658 {
3665 }
3667 }
3669 static void
3671 machine_mode mode,
3672 const_tree type,
3674 {
3677 {
3687 }
3688 }
3690 bool
3692 {
3695 }
3697 /* Implement FUNCTION_ARG_BOUNDARY. Every parameter gets at least
3698 PARM_BOUNDARY bits of alignment, but will be given anything up
3699 to STACK_BOUNDARY bits if the type requires it. This makes sure
3700 that both before and after the layout of each argument, the Next
3701 Stacked Argument Address (NSAA) will have a minimum alignment of
3702 8 bytes. */
3704 static unsigned int
3706 {
3709 }
3711 /* Implement TARGET_GET_RAW_RESULT_MODE and TARGET_GET_RAW_ARG_MODE. */
3713 static fixed_size_mode
3715 {
3717 /* Don't use the SVE part of the register for __builtin_apply and
3718 __builtin_return. The SVE registers aren't used by the normal PCS,
3719 so using them there would be a waste of time. The PCS extensions
3720 for SVE types are fundamentally incompatible with the
3721 __builtin_return/__builtin_apply interface. */
3724 }
3726 /* Implement TARGET_FUNCTION_ARG_PADDING.
3728 Small aggregate types are placed in the lowest memory address.
3730 The related parameter passing rules are B.4, C.3, C.5 and C.14. */
3732 static pad_direction
3734 {
3735 /* On little-endian targets, the least significant byte of every stack
3736 argument is passed at the lowest byte address of the stack slot. */
3740 /* Otherwise, integral, floating-point and pointer types are padded downward:
3741 the least significant byte of a stack argument is passed at the highest
3742 byte address of the stack slot. */
3749 /* Everything else padded upward, i.e. data in first byte of stack slot. */
3751 }
3753 /* Similarly, for use by BLOCK_REG_PADDING (MODE, TYPE, FIRST).
3755 It specifies padding for the last (may also be the only)
3756 element of a block move between registers and memory. If
3757 assuming the block is in the memory, padding upward means that
3758 the last element is padded after its highest significant byte,
3759 while in downward padding, the last element is padded at the
3760 its least significant byte side.
3762 Small aggregates and small complex types are always padded
3763 upwards.
3765 We don't need to worry about homogeneous floating-point or
3766 short-vector aggregates; their move is not affected by the
3767 padding direction determined here. Regardless of endianness,
3768 each element of such an aggregate is put in the least
3769 significant bits of a fp/simd register.
3771 Return !BYTES_BIG_ENDIAN if the least significant byte of the
3772 register has useful data, and return the opposite if the most
3773 significant byte does. */
3775 bool
3778 {
3780 /* Small composite types are always padded upward. */
3782 {
3783 HOST_WIDE_INT size;
3786 else
3787 /* No frontends can create types with variable-sized modes, so we
3788 shouldn't be asked to pass or return them. */
3792 }
3794 /* Otherwise, use the default padding. */
3796 }
3798 static scalar_int_mode
3800 {
3802 }
3804 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
3806 /* We use the 12-bit shifted immediate arithmetic instructions so values
3807 must be multiple of (1 << 12), i.e. 4096. */
3808 #define ARITH_FACTOR 4096
3810 #if (PROBE_INTERVAL % ARITH_FACTOR) != 0
3811 #error Cannot use simple address calculation for stack probing
3812 #endif
3814 /* The pair of scratch registers used for stack probing. */
3815 #define PROBE_STACK_FIRST_REG 9
3816 #define PROBE_STACK_SECOND_REG 10
3818 /* Emit code to probe a range of stack addresses from FIRST to FIRST+POLY_SIZE,
3819 inclusive. These are offsets from the current stack pointer. */
3821 static void
3823 {
3824 HOST_WIDE_INT size;
3826 {
3829 }
3833 /* See the same assertion on PROBE_INTERVAL above. */
3836 /* See if we have a constant small number of probes to generate. If so,
3837 that's the easy case. */
3839 {
3846 }
3848 /* The run-time loop is made up of 8 insns in the generic case while the
3849 compile-time loop is made up of 4+2*(n-2) insns for n # of intervals. */
3851 {
3856 stack_pointer_rtx,
3860 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
3861 it exceeds SIZE. If only two probes are needed, this will not
3862 generate any code. Then probe at FIRST + SIZE. */
3864 {
3868 }
3872 {
3877 }
3878 else
3880 }
3882 /* Otherwise, do the same as above, but in a loop. Note that we must be
3883 extra careful with variables wrapping around because we might be at
3884 the very top (or the very bottom) of the address space and we have
3885 to be able to handle this case properly; in particular, we use an
3886 equality test for the loop condition. */
3887 else
3888 {
3891 /* Step 1: round SIZE to the previous multiple of the interval. */
3896 /* Step 2: compute initial and final value of the loop counter. */
3898 /* TEST_ADDR = SP + FIRST. */
3902 /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE. */
3905 {
3909 }
3910 else
3914 /* Step 3: the loop
3916 do
3917 {
3918 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
3919 probe at TEST_ADDR
3920 }
3921 while (TEST_ADDR != LAST_ADDR)
3923 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
3924 until it is equal to ROUNDED_SIZE. */
3929 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
3930 that SIZE is equal to ROUNDED_SIZE. */
3933 {
3937 {
3942 }
3943 else
3945 }
3946 }
3948 /* Make sure nothing is scheduled before we are done. */
3950 }
3952 /* Probe a range of stack addresses from REG1 to REG2 inclusive. These are
3953 absolute addresses. */
3957 {
3964 /* Loop. */
3967 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
3972 /* Probe at TEST_ADDR. */
3975 /* Test if TEST_ADDR == LAST_ADDR. */
3979 /* Branch. */
3985 }
3987 /* Determine whether a frame chain needs to be generated. */
3988 static bool
3990 {
3991 /* Force a frame chain for EH returns so the return address is at FP+8. */
3995 /* A leaf function cannot have calls or write LR. */
3998 /* Don't use a frame chain in leaf functions if leaf frame pointers
3999 are disabled. */
4004 }
4006 /* Mark the registers that need to be saved by the callee and calculate
4007 the size of the callee-saved registers area and frame record (both FP
4008 and LR may be omitted). */
4009 static void
4011 {
4020 #define SLOT_NOT_REQUIRED (-2)
4021 #define SLOT_REQUIRED (-1)
4026 /* First mark all the registers that really need to be saved... */
4033 /* ... that includes the eh data registers (if needed)... */
4039 /* ... and any callee saved register that dataflow says is live. */
4049 {
4052 }
4055 {
4056 /* FP and LR are placed in the linkage record. */
4062 }
4064 /* Now assign stack slots for them. */
4067 {
4074 }
4082 {
4083 /* If there is an alignment gap between integer and fp callee-saves,
4084 allocate the last fp register to it if possible. */
4086 {
4089 }
4098 }
4104 HOST_WIDE_INT varargs_and_saved_regs_size
4112 /* Both these values are already aligned. */
4136 {
4137 /* Simple, small frame with no outgoing arguments:
4138 stp reg1, reg2, [sp, -frame_size]!
4139 stp reg3, reg4, [sp, 16] */
4141 }
4146 max_push_offset)))
4147 {
4148 /* Frame with small outgoing arguments:
4149 sub sp, sp, frame_size
4150 stp reg1, reg2, [sp, outgoing_args_size]
4151 stp reg3, reg4, [sp, outgoing_args_size + 16] */
4155 }
4158 {
4159 /* Frame with large outgoing arguments but a small local area:
4160 stp reg1, reg2, [sp, -hard_fp_offset]!
4161 stp reg3, reg4, [sp, 16]
4162 sub sp, sp, outgoing_args_size */
4166 }
4167 else
4168 {
4169 /* Frame with large local area and outgoing arguments using frame pointer:
4170 sub sp, sp, hard_fp_offset
4171 stp x29, x30, [sp, 0]
4172 add x29, sp, 0
4173 stp reg3, reg4, [sp, 16]
4174 sub sp, sp, outgoing_args_size */
4178 }
4181 }
4183 /* Return true if the register REGNO is saved on entry to
4184 the current function. */
4186 static bool
4188 {
4190 }
4192 /* Return the next register up from REGNO up to LIMIT for the callee
4193 to save. */
4195 static unsigned
4197 {
4199 regno ++;
4201 }
4203 /* Push the register number REGNO of mode MODE to the stack with write-back
4204 adjusting the stack by ADJUSTMENT. */
4206 static void
4208 HOST_WIDE_INT adjustment)
4209 {
4220 }
4222 /* Generate and return an instruction to store the pair of registers
4223 REG and REG2 of mode MODE to location BASE with write-back adjusting
4224 the stack location BASE by ADJUSTMENT. */
4226 static rtx
4228 HOST_WIDE_INT adjustment)
4229 {
4231 {
4242 }
4243 }
4245 /* Push registers numbered REGNO1 and REGNO2 to the stack, adjusting the
4246 stack pointer by ADJUSTMENT. */
4248 static void
4250 {
4265 }
4267 /* Load the pair of register REG, REG2 of mode MODE from stack location BASE,
4268 adjusting it by ADJUSTMENT afterwards. */
4270 static rtx
4272 HOST_WIDE_INT adjustment)
4273 {
4275 {
4284 }
4285 }
4287 /* Pop the two registers numbered REGNO1, REGNO2 from the stack, adjusting it
4288 afterwards by ADJUSTMENT and writing the appropriate REG_CFA_RESTORE notes
4289 into CFI_OPS. */
4291 static void
4294 {
4301 {
4305 }
4306 else
4307 {
4312 }
4313 }
4315 /* Generate and return a store pair instruction of mode MODE to store
4316 register REG1 to MEM1 and register REG2 to MEM2. */
4318 static rtx
4320 rtx reg2)
4321 {
4323 {
4332 }
4333 }
4335 /* Generate and regurn a load pair isntruction of mode MODE to load register
4336 REG1 from MEM1 and register REG2 from MEM2. */
4338 static rtx
4340 rtx mem2)
4341 {
4343 {
4352 }
4353 }
4355 /* Return TRUE if return address signing should be enabled for the current
4356 function, otherwise return FALSE. */
4358 bool
4360 {
4361 /* This function should only be called after frame laid out. */
4364 /* If signing scope is AARCH64_FUNCTION_NON_LEAF, we only sign a leaf function
4365 if it's LR is pushed onto stack. */
4369 }
4371 /* Emit code to save the callee-saved registers from register number START
4372 to LIMIT to the stack at the location starting at offset START_OFFSET,
4373 skipping any write-back candidates if SKIP_WB is true. */
4375 static void
4378 {
4386 {
4388 poly_int64 offset;
4401 offset));
4410 {
4412 rtx mem2;
4416 offset));
4418 reg2));
4420 /* The first part of a frame-related parallel insn is
4421 always assumed to be relevant to the frame
4422 calculations; subsequent parts, are only
4423 frame-related if explicitly marked. */
4426 }
4427 else
4431 }
4432 }
4434 /* Emit code to restore the callee registers of mode MODE from register
4435 number START up to and including LIMIT. Restore from the stack offset
4436 START_OFFSET, skipping any write-back candidates if SKIP_WB is true.
4437 Write the appropriate REG_CFA_RESTORE notes into CFI_OPS. */
4439 static void
4443 {
4447 poly_int64 offset;
4452 {
4473 {
4475 rtx mem2;
4483 }
4484 else
4487 }
4488 }
4490 /* Return true if OFFSET is a signed 4-bit value multiplied by the size
4491 of MODE. */
4495 {
4496 HOST_WIDE_INT multiple;
4499 }
4501 /* Return true if OFFSET is a unsigned 6-bit value multiplied by the size
4502 of MODE. */
4506 {
4507 HOST_WIDE_INT multiple;
4510 }
4512 /* Return true if OFFSET is a signed 7-bit value multiplied by the size
4513 of MODE. */
4515 bool
4517 {
4518 HOST_WIDE_INT multiple;
4521 }
4523 /* Return true if OFFSET is a signed 9-bit value. */
4527 poly_int64 offset)
4528 {
4529 HOST_WIDE_INT const_offset;
4532 }
4534 /* Return true if OFFSET is a signed 9-bit value multiplied by the size
4535 of MODE. */
4539 {
4540 HOST_WIDE_INT multiple;
4543 }
4545 /* Return true if OFFSET is an unsigned 12-bit value multiplied by the size
4546 of MODE. */
4550 {
4551 HOST_WIDE_INT multiple;
4554 }
4556 /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
4558 static sbitmap
4560 {
4566 /* The registers we need saved to the frame. */
4569 {
4574 /* Check that we can access the stack slot of the register with one
4575 direct load with no adjustments needed. */
4578 }
4580 /* Don't mess with the hard frame pointer. */
4586 /* If aarch64_layout_frame has chosen registers to store/restore with
4587 writeback don't interfere with them to avoid having to output explicit
4588 stack adjustment instructions. */
4598 }
4600 /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
4602 static sbitmap
4604 {
4612 /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
4618 {
4622 /* If there is a callee-save at an adjacent offset, add it too
4623 to increase the use of LDP/STP. */
4628 {
4632 }
4633 }
4636 }
4638 /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS.
4639 Nothing to do for aarch64. */
4641 static void
4643 {
4644 }
4646 /* Return the next set bit in BMP from START onwards. Return the total number
4647 of bits in BMP if no set bit is found at or after START. */
4649 static unsigned int
4651 {
4662 }
4664 /* Do the work for aarch64_emit_prologue_components and
4665 aarch64_emit_epilogue_components. COMPONENTS is the bitmap of registers
4666 to save/restore, PROLOGUE_P indicates whether to emit the prologue sequence
4667 for these components or the epilogue sequence. That is, it determines
4668 whether we should emit stores or loads and what kind of CFA notes to attach
4669 to the insns. Otherwise the logic for the two sequences is very
4670 similar. */
4672 static void
4674 {
4676 ? HARD_FRAME_POINTER_REGNUM
4684 {
4685 /* AAPCS64 section 5.1.2 requires only the bottom 64 bits to be saved
4686 so DFmode for the vector registers is enough. */
4698 /* No more registers to handle after REGNO.
4699 Emit a single save/restore and exit. */
4701 {
4706 else
4709 }
4712 /* The next register is not of the same class or its offset is not
4713 mergeable with the current one into a pair. */
4718 {
4723 else
4728 }
4730 /* REGNO2 can be saved/restored in a pair with REGNO. */
4742 else
4747 {
4750 }
4751 else
4752 {
4755 }
4758 }
4759 }
4761 /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
4763 static void
4765 {
4767 }
4769 /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
4771 static void
4773 {
4775 }
4777 /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
4779 static void
4781 {
4785 }
4787 /* Add a REG_CFA_EXPRESSION note to INSN to say that register REG
4788 is saved at BASE + OFFSET. */
4790 static void
4793 {
4797 }
4799 /* AArch64 stack frames generated by this compiler look like:
4801 +-------------------------------+
4802 | |
4803 | incoming stack arguments |
4804 | |
4805 +-------------------------------+
4806 | | <-- incoming stack pointer (aligned)
4807 | callee-allocated save area |
4808 | for register varargs |
4809 | |
4810 +-------------------------------+
4811 | local variables | <-- frame_pointer_rtx
4812 | |
4813 +-------------------------------+
4814 | padding0 | \
4815 +-------------------------------+ |
4816 | callee-saved registers | | frame.saved_regs_size
4817 +-------------------------------+ |
4818 | LR' | |
4819 +-------------------------------+ |
4820 | FP' | / <- hard_frame_pointer_rtx (aligned)
4821 +-------------------------------+
4822 | dynamic allocation |
4823 +-------------------------------+
4824 | padding |
4825 +-------------------------------+
4826 | outgoing stack arguments | <-- arg_pointer
4827 | |
4828 +-------------------------------+
4829 | | <-- stack_pointer_rtx (aligned)
4831 Dynamic stack allocations via alloca() decrease stack_pointer_rtx
4832 but leave frame_pointer_rtx and hard_frame_pointer_rtx
4833 unchanged. */
4835 /* Generate the prologue instructions for entry into a function.
4836 Establish the stack frame by decreasing the stack pointer with a
4837 properly calculated size and, if necessary, create a frame record
4838 filled with the values of LR and previous frame pointer. The
4839 current FP is also set up if it is in use. */
4841 void
4843 {
4856 /* Sign return address for functions. */
4858 {
4862 }
4868 {
4870 {
4874 (frame_size
4876 }
4879 }
4890 {
4893 {
4898 }
4903 {
4904 /* Variable-sized frames need to describe the save slot
4905 address using DW_CFA_expression rather than DW_CFA_offset.
4906 This means that, without taking further action, the
4907 locations of the registers that we've already saved would
4908 remain based on the stack pointer even after we redefine
4909 the CFA based on the frame pointer. We therefore need new
4910 DW_CFA_expressions to re-express the save slots with addresses
4911 based on the frame pointer. */
4915 /* Add an explicit CFA definition if this was previously
4916 implicit. */
4918 {
4920 callee_offset);
4923 }
4925 /* Change the save slot expressions for the registers that
4926 we've already saved. */
4931 reg_offset);
4932 }
4934 }
4941 }
4943 /* Return TRUE if we can use a simple_return insn.
4945 This function checks whether the callee saved stack is empty, which
4946 means no restore actions are need. The pro_and_epilogue will use
4947 this to check whether shrink-wrapping opt is feasible. */
4949 bool
4951 {
4961 }
4963 /* Generate the epilogue instructions for returning from a function.
4964 This is almost exactly the reverse of the prolog sequence, except
4965 that we need to insert barriers to avoid scheduling loads that read
4966 from a deallocated stack, and we optimize the unwind records by
4967 emitting them all together if possible. */
4968 void
4970 {
4981 /* A stack clash protection prologue may not have left IP0_REGNUM or
4982 IP1_REGNUM in a usable state. The same is true for allocations
4983 with an SVE component, since we then need both temporary registers
4984 for each allocation. */
4989 /* We need to add memory barrier to prevent read from deallocated stack. */
4990 bool need_barrier_p
4994 /* Emit a barrier to prevent loads from a deallocated stack. */
4998 {
5001 }
5003 /* Restore the stack pointer from the frame pointer if it may not
5004 be the same as the stack pointer. */
5009 /* If writeback is used when restoring callee-saves, the CFA
5010 is restored on the instruction doing the writeback. */
5014 else
5030 {
5031 /* Emit delayed restores and set the CFA to be SP + initial_adjust. */
5037 }
5043 {
5044 /* Emit delayed restores and reset the CFA to be SP. */
5049 }
5051 /* We prefer to emit the combined return/authenticate instruction RETAA,
5052 however there are three cases in which we must instead emit an explicit
5053 authentication instruction.
5055 1) Sibcalls don't return in a normal way, so if we're about to call one
5056 we must authenticate.
5058 2) The RETAA instruction is not available before ARMv8.3-A, so if we are
5059 generating code for !TARGET_ARMV8_3 we can't use it and must
5060 explicitly authenticate.
5062 3) On an eh_return path we make extra stack adjustments to update the
5063 canonical frame address to be the exception handler's CFA. We want
5064 to authenticate using the CFA of the function which calls eh_return.
5065 */
5068 {
5072 }
5074 /* Stack adjustment for exception handler. */
5076 {
5077 /* We need to unwind the stack by the offset computed by
5078 EH_RETURN_STACKADJ_RTX. We have already reset the CFA
5079 to be SP; letting the CFA move during this adjustment
5080 is just as correct as retaining the CFA from the body
5081 of the function. Therefore, do nothing special. */
5083 }
5088 }
5090 /* Implement EH_RETURN_HANDLER_RTX. EH returns need to either return
5091 normally or return to a previous frame after unwinding.
5093 An EH return uses a single shared return sequence. The epilogue is
5094 exactly like a normal epilogue except that it has an extra input
5095 register (EH_RETURN_STACKADJ_RTX) which contains the stack adjustment
5096 that must be applied after the frame has been destroyed. An extra label
5097 is inserted before the epilogue which initializes this register to zero,
5098 and this is the entry point for a normal return.
5100 An actual EH return updates the return address, initializes the stack
5101 adjustment and jumps directly into the epilogue (bypassing the zeroing
5102 of the adjustment). Since the return address is typically saved on the
5103 stack when a function makes a call, the saved LR must be updated outside
5104 the epilogue.
5106 This poses problems as the store is generated well before the epilogue,
5107 so the offset of LR is not known yet. Also optimizations will remove the
5108 store as it appears dead, even after the epilogue is generated (as the
5109 base or offset for loading LR is different in many cases).
5111 To avoid these problems this implementation forces the frame pointer
5112 in eh_return functions so that the location of LR is fixed and known early.
5113 It also marks the store volatile, so no optimization is permitted to
5114 remove the store. */
5115 rtx
5117 {
5121 /* Mark the store volatile, so no optimization is permitted to remove it. */
5124 }
5126 /* Output code to add DELTA to the first argument, and then jump
5127 to FUNCTION. Used for C++ multiple inheritance. */
5128 static void
5130 HOST_WIDE_INT delta,
5131 HOST_WIDE_INT vcall_offset,
5132 tree function)
5133 {
5134 /* The this pointer is always in x0. Note that this differs from
5135 Arm where the this pointer maybe bumped to r1 if r0 is required
5136 to return a pointer to an aggregate. On AArch64 a result value
5137 pointer will be in x8. */
5151 else
5152 {
5157 {
5161 else
5164 }
5168 else
5175 else
5176 {
5178 Pmode);
5180 }
5184 else
5190 }
5192 /* Generate a tail call to the target function. */
5194 {
5197 }
5209 /* Stop pretending to be a post-reload pass. */
5211 }
5213 static bool
5215 {
5220 {
5224 /* Don't recurse into UNSPEC_TLS looking for TLS symbols; these are
5225 TLS offsets, not real symbol references. */
5228 }
5230 }
5233 /* Return true if val can be encoded as a 12-bit unsigned immediate with
5234 a left shift of 0 or 12 bits. */
5235 bool
5237 {
5240 );
5241 }
5244 /* Return true if val is an immediate that can be loaded into a
5245 register by a MOVZ instruction. */
5246 static bool
5248 {
5250 {
5254 }
5255 else
5256 {
5257 /* Ignore sign extension. */
5259 }
5262 }
5264 /* VAL is a value with the inner mode of MODE. Replicate it to fill a
5265 64-bit (DImode) integer. */
5267 static unsigned HOST_WIDE_INT
5269 {
5272 {
5276 }
5278 }
5280 /* Multipliers for repeating bitmasks of width 32, 16, 8, 4, and 2. */
5283 {
5289 };
5292 /* Return true if val is a valid bitmask immediate. */
5294 bool
5296 {
5300 /* Check for a single sequence of one bits and return quickly if so.
5301 The special cases of all ones and all zeroes returns false. */
5308 /* Replicate 32-bit immediates so we can treat them as 64-bit. */
5312 /* Invert if the immediate doesn't start with a zero bit - this means we
5313 only need to search for sequences of one bits. */
5317 /* Find the first set bit and set tmp to val with the first sequence of one
5318 bits removed. Return success if there is a single sequence of ones. */
5325 /* Find the next set bit and compute the difference in bit position. */
5330 /* Check the bit position difference is a power of 2, and that the first
5331 sequence of one bits fits within 'bits' bits. */
5335 /* Check the sequence of one bits is repeated 64/bits times. */
5337 }
5339 /* Create mask of ones, covering the lowest to highest bits set in VAL_IN.
5340 Assumed precondition: VAL_IN Is not zero. */
5342 unsigned HOST_WIDE_INT
5344 {
5351 }
5353 /* Create constant where bits outside of lowest bit set to highest bit set
5354 are set to 1. */
5356 unsigned HOST_WIDE_INT
5358 {
5360 }
5362 /* Return true if VAL_IN is a valid 'and' bitmask immediate. */
5364 bool
5366 {
5367 scalar_int_mode int_mode;
5380 }
5382 /* Return true if val is an immediate that can be loaded into a
5383 register in a single instruction. */
5384 bool
5386 {
5387 scalar_int_mode int_mode;
5394 }
5396 static bool
5398 {
5404 /* There's no way to calculate VL-based values using relocations. */
5412 {
5416 else
5417 /* Avoid generating a 64-bit relocation in ILP32; leave
5418 to aarch64_expand_mov_immediate to handle it properly. */
5420 }
5423 }
5425 /* Implement TARGET_CASE_VALUES_THRESHOLD.
5426 The expansion for a table switch is quite expensive due to the number
5427 of instructions, the table lookup and hard to predict indirect jump.
5428 When optimizing for speed, and -O3 enabled, use the per-core tuning if
5429 set, otherwise use tables for > 16 cases as a tradeoff between size and
5430 performance. When optimizing for size, use the default setting. */
5432 static unsigned int
5434 {
5435 /* Use the specified limit for the number of cases before using jump
5436 tables at higher optimization levels. */
5440 else
5442 }
5444 /* Return true if register REGNO is a valid index register.
5445 STRICT_P is true if REG_OK_STRICT is in effect. */
5447 bool
5449 {
5451 {
5459 }
5461 }
5463 /* Return true if register REGNO is a valid base register for mode MODE.
5464 STRICT_P is true if REG_OK_STRICT is in effect. */
5466 bool
5468 {
5470 {
5478 }
5480 /* The fake registers will be eliminated to either the stack or
5481 hard frame pointer, both of which are usually valid base registers.
5482 Reload deals with the cases where the eliminated form isn't valid. */
5487 }
5489 /* Return true if X is a valid base register for mode MODE.
5490 STRICT_P is true if REG_OK_STRICT is in effect. */
5492 static bool
5494 {
5501 }
5503 /* Return true if address offset is a valid index. If it is, fill in INFO
5504 appropriately. STRICT_P is true if REG_OK_STRICT is in effect. */
5506 static bool
5509 {
5511 rtx index;
5514 /* (reg:P) */
5517 {
5521 }
5522 /* (sign_extend:DI (reg:SI)) */
5527 {
5532 }
5533 /* (mult:DI (sign_extend:DI (reg:SI)) (const_int scale)) */
5540 {
5545 }
5546 /* (ashift:DI (sign_extend:DI (reg:SI)) (const_int shift)) */
5553 {
5558 }
5559 /* (sign_extract:DI (mult:DI (reg:DI) (const_int scale)) 32+shift 0) */
5566 {
5574 }
5575 /* (and:DI (mult:DI (reg:DI) (const_int scale))
5576 (const_int 0xffffffff<<shift)) */
5583 {
5589 }
5590 /* (sign_extract:DI (ashift:DI (reg:DI) (const_int shift)) 32+shift 0) */
5597 {
5605 }
5606 /* (and:DI (ashift:DI (reg:DI) (const_int shift))
5607 (const_int 0xffffffff<<shift)) */
5614 {
5620 }
5621 /* (mult:P (reg:P) (const_int scale)) */
5626 {
5630 }
5631 /* (ashift:P (reg:P) (const_int shift)) */
5636 {
5640 }
5641 else
5650 {
5654 }
5655 else
5656 {
5661 }
5665 {
5670 }
5673 }
5675 /* Return true if MODE is one of the modes for which we
5676 support LDP/STP operations. */
5678 static bool
5680 {
5685 }
5687 /* Return true if REGNO is a virtual pointer register, or an eliminable
5688 "soft" frame register. Like REGNO_PTR_FRAME_P except that we don't
5689 include stack_pointer or hard_frame_pointer. */
5690 static bool
5692 {
5697 }
5699 /* Return true if X is a valid address of type TYPE for machine mode MODE.
5700 If it is, fill in INFO appropriately. STRICT_P is true if
5701 REG_OK_STRICT is in effect. */
5703 static bool
5707 {
5710 poly_int64 offset;
5712 HOST_WIDE_INT const_size;
5714 /* On BE, we use load/store pair for all large int mode load/stores.
5715 TI/TFmode may also use a load/store pair. */
5728 /* For SVE, only accept [Rn], [Rn, Rm, LSL #shift] and
5729 [Rn, #offset, MUL VL]. */
5734 /* On LE, for AdvSIMD, don't support anything other than POST_INC or
5735 REG addressing. */
5737 && !BYTES_BIG_ENDIAN
5745 {
5762 {
5769 }
5774 {
5780 /* TImode and TFmode values are allowed in both pairs of X
5781 registers and individual Q registers. The available
5782 address modes are:
5783 X,X: 7-bit signed scaled offset
5784 Q: 9-bit signed offset
5785 We conservatively require an offset representable in either mode.
5786 When performing the check for pairs of X registers i.e. LDP/STP
5787 pass down DImode since that is the natural size of the LDP/STP
5788 instruction memory accesses. */
5794 /* A 7bit offset check because OImode will emit a ldp/stp
5795 instruction (only big endian will get here).
5796 For ldp/stp instructions, the offset is scaled for the size of a
5797 single element of the pair. */
5801 /* Three 9/12 bit offsets checks because CImode will emit three
5802 ldr/str instructions (only big endian will get here). */
5809 /* Two 7bit offsets checks because XImode will emit two ldp/stp
5810 instructions (only big endian will get here). */
5816 /* Make "m" use the LD1 offset range for SVE data modes, so
5817 that pre-RTL optimizers like ivopts will work to that
5818 instead of the wider LDR/STR range. */
5825 {
5826 poly_int64 end_offset = (offset
5833 end_offset)));
5834 }
5843 else
5846 }
5849 {
5850 /* Look for base + (scaled/extended) index register. */
5853 {
5856 }
5859 {
5862 }
5863 }
5884 {
5888 /* TImode and TFmode values are allowed in both pairs of X
5889 registers and individual Q registers. The available
5890 address modes are:
5891 X,X: 7-bit signed scaled offset
5892 Q: 9-bit signed offset
5893 We conservatively require an offset representable in either mode.
5894 */
5903 else
5905 }
5911 /* load literal: pc-relative constant pool entry. Only supported
5912 for SI mode or larger. */
5918 {
5926 }
5935 {
5941 {
5942 /* The symbol and offset must be aligned to the access size. */
5948 {
5952 }
5958 else
5967 }
5968 }
5973 }
5974 }
5976 /* Return true if the address X is valid for a PRFM instruction.
5977 STRICT_P is true if we should do strict checking with
5978 aarch64_classify_address. */
5980 bool
5982 {
5985 /* PRFM accepts the same addresses as DImode... */
5990 /* ... except writeback forms. */
5992 }
5994 bool
5996 {
5997 rtx offset;
6001 }
6003 /* Classify the base of symbolic expression X. */
6005 enum aarch64_symbol_type
6007 {
6008 rtx offset;
6012 }
6015 /* Return TRUE if X is a legitimate address for accessing memory in
6016 mode MODE. */
6017 static bool
6019 {
6023 }
6025 /* Return TRUE if X is a legitimate address of type TYPE for accessing
6026 memory in mode MODE. STRICT_P is true if REG_OK_STRICT is in effect. */
6027 bool
6029 aarch64_addr_query_type type)
6030 {
6034 }
6036 /* Implement TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT. */
6038 static bool
6040 poly_int64 orig_offset,
6041 machine_mode mode)
6042 {
6043 HOST_WIDE_INT size;
6045 {
6048 /* A general SVE offset is A * VQ + B. Remove the A component from
6049 coefficient 0 in order to get the constant B. */
6052 /* Split an out-of-range address displacement into a base and
6053 offset. Use 4KB range for 1- and 2-byte accesses and a 16KB
6054 range otherwise to increase opportunities for sharing the base
6055 address of different sizes. Unaligned accesses use the signed
6056 9-bit range, TImode/TFmode use the intersection of signed
6057 scaled 7-bit and signed 9-bit offset. */
6062 else
6068 /* Split the offset into second_offset and the rest. */
6072 }
6073 else
6074 {
6075 /* Get the mode we should use as the basis of the range. For structure
6076 modes this is the mode of one vector. */
6078 machine_mode step_mode
6081 /* Get the "mul vl" multiplier we'd like to use. */
6085 /* LDR supports a 9-bit range, but the move patterns for
6086 structure modes require all vectors to be in range of the
6087 same base. The simplest way of accomodating that while still
6088 promoting reuse of anchor points between different modes is
6089 to use an 8-bit range unconditionally. */
6091 else
6092 /* Predicates are only handled singly, so we might as well use
6093 the full range. */
6098 /* Convert the "mul vl" multiplier into a byte offset. */
6103 /* Split the offset into second_offset and the rest. */
6107 }
6108 }
6110 /* Return the binary representation of floating point constant VALUE in INTVAL.
6111 If the value cannot be converted, return false without setting INTVAL.
6112 The conversion is done in the given MODE. */
6113 bool
6115 {
6117 /* We make a general exception for 0. */
6119 {
6122 }
6124 scalar_float_mode mode;
6128 /* Only support up to DF mode. */
6140 {
6144 }
6145 else
6150 }
6152 /* Return TRUE if rtx X is an immediate constant that can be moved using a
6153 single MOV(+MOVK) followed by an FMOV. */
6154 bool
6156 {
6161 /* Determine whether it's cheaper to write float constants as
6162 mov/movk pairs over ldr/adrp pairs. */
6168 {
6170 ? SImode
6175 }
6178 }
6180 /* Return TRUE if rtx X is immediate constant 0.0 */
6181 bool
6183 {
6190 }
6192 /* Return TRUE if rtx X is immediate constant that fits in a single
6193 MOVI immediate operation. */
6194 bool
6196 {
6200 machine_mode vmode;
6201 scalar_int_mode imode;
6206 {
6210 /* We make a general exception for 0. */
6215 }
6219 else
6222 /* use a 64 bit mode for everything except for DI/DF mode, where we use
6223 a 128 bit vector mode. */
6230 }
6233 /* Return the fixed registers used for condition codes. */
6235 static bool
6237 {
6241 }
6243 /* This function is used by the call expanders of the machine description.
6244 RESULT is the register in which the result is returned. It's NULL for
6245 "call" and "sibcall".
6246 MEM is the location of the function call.
6247 SIBCALL indicates whether this function call is normal call or sibling call.
6248 It will generate different pattern accordingly. */
6250 void
6252 {
6254 rtvec vec;
6255 machine_mode mode;
6262 /* Decide if we should generate indirect calls by loading the
6263 address of the callee into a register before performing
6264 the branch-and-link. */
6278 else
6285 }
6287 /* Emit call insn with PAT and do aarch64-specific handling. */
6289 void
6291 {
6297 }
6299 machine_mode
6301 {
6302 /* All floating point compares return CCFP if it is an equality
6303 comparison, and CCFPE otherwise. */
6305 {
6307 {
6328 }
6329 }
6331 /* Equality comparisons of short modes against zero can be performed
6332 using the TST instruction with the appropriate bitmask. */
6338 /* Similarly, comparisons of zero_extends from shorter modes can
6339 be performed using an ANDS with an immediate mask. */
6355 /* A compare with a shifted operand. Because of canonicalization,
6356 the comparison will have to be swapped when we emit the assembly
6357 code. */
6365 /* Similarly for a negated operand, but we can only do this for
6366 equalities. */
6373 /* A test for unsigned overflow. */
6380 /* For everything else, return CCmode. */
6382 }
6384 static int
6387 int
6389 {
6396 }
6398 static int
6400 {
6402 {
6406 {
6420 }
6425 {
6437 }
6442 {
6454 }
6459 {
6465 }
6470 {
6474 }
6479 {
6483 }
6488 }
6491 }
6493 bool
6495 HOST_WIDE_INT minval,
6496 HOST_WIDE_INT maxval)
6497 {
6498 rtx elt;
6502 }
6504 bool
6506 {
6508 }
6510 /* Return true if VEC is a constant in which every element is in the range
6511 [MINVAL, MAXVAL]. The elements do not need to have the same value. */
6513 static bool
6515 HOST_WIDE_INT minval,
6516 HOST_WIDE_INT maxval)
6517 {
6529 {
6534 }
6536 }
6538 /* N Z C V. */
6539 #define AARCH64_CC_V 1
6540 #define AARCH64_CC_C (1 << 1)
6541 #define AARCH64_CC_Z (1 << 2)
6542 #define AARCH64_CC_N (1 << 3)
6544 /* N Z C V flags for ccmp. Indexed by AARCH64_COND_CODE. */
6546 {
6563 };
6565 /* Print floating-point vector immediate operand X to F, negating it
6566 first if NEGATE is true. Return true on success, false if it isn't
6567 a constant we can handle. */
6569 static bool
6571 {
6572 rtx elt;
6581 /* We only handle the SVE single-bit immediates here. */
6588 else
6592 }
6594 /* Return the equivalent letter for size. */
6595 static char
6597 {
6599 {
6605 }
6606 }
6608 /* Print operand X to file F in a target specific manner according to CODE.
6609 The acceptable formatting commands given by CODE are:
6610 'c': An integer or symbol address without a preceding #
6611 sign.
6612 'C': Take the duplicated element in a vector constant
6613 and print it in hex.
6614 'D': Take the duplicated element in a vector constant
6615 and print it as an unsigned integer, in decimal.
6616 'e': Print the sign/zero-extend size as a character 8->b,
6617 16->h, 32->w.
6618 'p': Prints N such that 2^N == X (X must be power of 2 and
6619 const int).
6620 'P': Print the number of non-zero bits in X (a const_int).
6621 'H': Print the higher numbered register of a pair (TImode)
6622 of regs.
6623 'm': Print a condition (eq, ne, etc).
6624 'M': Same as 'm', but invert condition.
6625 'N': Take the duplicated element in a vector constant
6626 and print the negative of it in decimal.
6627 'b/h/s/d/q': Print a scalar FP/SIMD register name.
6628 'S/T/U/V': Print a FP/SIMD register name for a register list.
6629 The register printed is the FP/SIMD register name
6630 of X + 0/1/2/3 for S/T/U/V.
6631 'R': Print a scalar FP/SIMD register name + 1.
6632 'X': Print bottom 16 bits of integer constant in hex.
6633 'w/x': Print a general register name or the zero register
6634 (32-bit or 64-bit).
6635 '0': Print a normal operand, if it's a general register,
6636 then we assume DImode.
6637 'k': Print NZCV for conditional compare instructions.
6638 'A': Output address constant representing the first
6639 argument of X, specifying a relocation offset
6640 if appropriate.
6641 'L': Output constant address specified by X
6642 with a relocation offset if appropriate.
6643 'G': Prints address of X, specifying a PC relative
6644 relocation mode if appropriate.
6645 'y': Output address of LDP or STP - this is used for
6646 some LDP/STPs which don't use a PARALLEL in their
6647 pattern (so the mode needs to be adjusted).
6648 'z': Output address of a typical LDP or STP. */
6650 static void
6652 {
6653 rtx elt;
6655 {
6658 {
6670 {
6673 }
6674 /* Fall through. */
6678 }
6682 {
6687 {
6690 }
6693 {
6706 }
6707 }
6711 {
6715 {
6718 }
6721 }
6726 {
6729 }
6736 {
6739 }
6746 {
6748 /* CONST_TRUE_RTX means al/nv (al is the default, don't print it). */
6750 {
6754 }
6757 {
6760 }
6767 }
6772 {
6775 }
6781 ;
6782 else
6783 {
6786 }
6795 {
6796 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6798 }
6807 {
6808 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6810 }
6818 {
6819 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
6821 }
6827 {
6830 }
6835 {
6836 /* Print a replicated constant in hex. */
6838 {
6841 }
6844 }
6848 {
6849 /* Print a replicated constant in decimal, treating it as
6850 unsigned. */
6852 {
6855 }
6858 }
6865 {
6868 }
6871 {
6874 }
6877 {
6880 }
6882 /* Fall through */
6886 {
6889 }
6892 {
6895 {
6898 else
6899 {
6900 char suffix
6905 }
6906 }
6907 else
6926 {
6929 }
6930 /* fall through */
6934 {
6937 }
6943 ;
6944 else
6945 {
6948 }
6952 /* Since we define TARGET_SUPPORTS_WIDE_INT we shouldn't ever
6953 be getting CONST_DOUBLEs holding integers. */
6956 {
6959 }
6961 {
6962 #define buf_size 20
6970 #undef buf_size
6971 }
6977 }
6985 {
7012 }
7018 {
7053 }
7059 {
7065 }
7070 {
7071 HOST_WIDE_INT cond_code;
7074 {
7077 }
7082 }
7087 {
7092 {
7095 }
7098 /* LDP/STP which uses a single double-width memory operand.
7099 Adjust the mode to appear like a typical LDP/STP.
7100 Currently this is supported for 16-byte accesses only. */
7105 }
7111 }
7112 }
7114 /* Print address 'x' of a memory access with mode 'mode'.
7115 'op' is the context required by aarch64_classify_address. It can either be
7116 MEM for a normal memory access or PARALLEL for LDP/STP. */
7117 static bool
7119 aarch64_addr_query_type type)
7120 {
7124 /* Check all addresses are Pmode - including ILP32. */
7130 {
7135 {
7136 HOST_WIDE_INT vnum
7141 }
7143 {
7144 HOST_WIDE_INT vnum
7149 }
7150 else
7159 else
7168 else
7177 else
7183 /* Writeback is only supported for fixed-width modes. */
7186 {
7209 }
7221 }
7224 }
7226 /* Print address 'x' of a LDP/STP with mode 'mode'. */
7227 static bool
7229 {
7231 }
7233 /* Print address 'x' of a memory access with mode 'mode'. */
7234 static void
7236 {
7239 }
7241 bool
7243 {
7250 /* UNSPEC_TLS entries for a symbol include a LABEL_REF for the
7251 referencing instruction, but they are constant offsets, not
7252 symbols. */
7258 {
7260 {
7266 }
7269 }
7272 }
7274 /* Implement REGNO_REG_CLASS. */
7276 enum reg_class
7278 {
7296 }
7298 /* OFFSET is an address offset for mode MODE, which has SIZE bytes.
7299 If OFFSET is out of range, return an offset of an anchor point
7300 that is in range. Return 0 otherwise. */
7302 static HOST_WIDE_INT
7304 machine_mode mode)
7305 {
7306 /* Does it look like we'll need a 16-byte load/store-pair operation? */
7310 /* For offsets that aren't a multiple of the access size, the limit is
7311 -256...255. */
7313 {
7314 /* BLKmode typically uses LDP of X-registers. */
7318 }
7320 /* Small negative offsets are supported. */
7327 /* Use 12-bit offset by access size. */
7329 }
7331 static rtx
7333 {
7334 /* Try to split X+CONST into Y=X+(CONST & ~mask), Y+(CONST&mask),
7335 where mask is selected by alignment and size of the offset.
7336 We try to pick as large a range for the offset as possible to
7337 maximize the chance of a CSE. However, for aligned addresses
7338 we limit the range to 4k so that structures with different sized
7339 elements are likely to use the same base. We need to be careful
7340 not to split a CONST for some forms of address expression, otherwise
7341 it will generate sub-optimal code. */
7344 {
7350 {
7354 /* Force any scaling into a temp for CSE. */
7358 /* Let the pointer register be in op0. */
7362 /* If the pointer is virtual or frame related, then we know that
7363 virtual register instantiation or register elimination is going
7364 to apply a second constant. We want the two constants folded
7365 together easily. Therefore, emit as (OP0 + CONST) + OP1. */
7367 {
7371 }
7373 /* Otherwise, in order to encourage CSE (and thence loop strength
7374 reduce) scaled addresses, emit as (OP0 + OP1) + CONST. */
7378 }
7380 HOST_WIDE_INT size;
7382 {
7384 mode);
7386 {
7390 }
7391 }
7392 }
7395 }
7397 /* Return the reload icode required for a constant pool in mode. */
7398 static enum insn_code
7400 {
7402 {
7438 }
7441 }
7442 static reg_class_t
7444 reg_class_t rclass,
7445 machine_mode mode,
7447 {
7448 /* Use aarch64_sve_reload_be for SVE reloads that cannot be handled
7449 directly by the *aarch64_sve_mov<mode>_be move pattern. See the
7450 comment at the head of aarch64-sve.md for more details about the
7451 big-endian handling. */
7457 {
7460 }
7462 /* If we have to disable direct literal pool loads and stores because the
7463 function is too big, then we need a scratch register. */
7468 {
7471 }
7473 /* Without the TARGET_SIMD instructions we cannot move a Q register
7474 to a Q register directly. We need a scratch. */
7478 {
7484 }
7486 /* A TFmode or TImode memory access should be handled via an FP_REGS
7487 because AArch64 has richer addressing modes for LDR/STR instructions
7488 than LDP/STP instructions. */
7497 }
7499 static bool
7501 {
7504 /* If we need a frame pointer, ARG_POINTER_REGNUM and FRAME_POINTER_REGNUM
7505 can only eliminate to HARD_FRAME_POINTER_REGNUM. */
7509 }
7511 poly_int64
7513 {
7517 {
7524 }
7527 {
7531 }
7534 }
7536 /* Implement RETURN_ADDR_RTX. We do not support moving back to a
7537 previous frame. */
7539 rtx
7541 {
7545 }
7548 static void
7550 {
7552 {
7555 }
7556 else
7557 {
7560 }
7565 }
7567 static void
7569 {
7573 /* Don't need to copy the trailing D-words, we fill those in below. */
7585 /* XXX We should really define a "clear_cache" pattern and use
7586 gen_clear_cache(). */
7591 ptr_mode);
7592 }
7594 static unsigned char
7596 {
7597 /* ??? Logically we should only need to provide a value when
7598 HARD_REGNO_MODE_OK says that at least one register in REGCLASS
7599 can hold MODE, but at the moment we need to handle all modes.
7600 Just ignore any runtime parts for registers that can't store them. */
7604 {
7630 }
7632 }
7634 static reg_class_t
7636 {
7641 {
7647 }
7649 /* Register eliminiation can result in a request for
7650 SP+constant->FP_REGS. We cannot support such operations which
7651 use SP as source and an FP_REG as destination, so reject out
7652 right now. */
7654 {
7657 /* Look through a possible SUBREG introduced by ILP32. */
7663 POINTER_REGS));
7665 }
7668 }
7670 void
7672 {
7674 }
7676 static void
7678 {
7681 else
7682 {
7684 /* While priority is known to be in range [0, 65535], so 18 bytes
7685 would be enough, the compiler might not know that. To avoid
7686 -Wformat-truncation false positive, use a larger size. */
7693 }
7694 }
7696 static void
7698 {
7701 else
7702 {
7704 /* While priority is known to be in range [0, 65535], so 18 bytes
7705 would be enough, the compiler might not know that. To avoid
7706 -Wformat-truncation false positive, use a larger size. */
7713 }
7714 }
7718 {
7724 {
7725 {
7728 },
7729 {
7732 },
7733 {
7736 },
7737 /* We assume that DImode is only generated when not optimizing and
7738 that we don't really need 64-bit address offsets. That would
7739 imply an object file with 8GB of code in a single function! */
7740 {
7743 }
7744 };
7753 /* Need to implement table size reduction, by chaning the code below. */
7763 }
7766 /* Return size in bits of an arithmetic operand which is shifted/scaled and
7767 masked such that it is suitable for a UXTB, UXTH, or UXTW extend
7768 operator. */
7770 int
7772 {
7774 {
7777 {
7781 }
7782 }
7784 }
7786 /* Constant pools are per function only when PC relative
7787 literal loads are true or we are in the large memory
7788 model. */
7792 {
7795 }
7797 static bool
7799 {
7800 /* We can't use blocks for constants when we're using a per-function
7801 constant pool. */
7803 }
7805 /* Select appropriate section for constants depending
7806 on where we place literal pools. */
7810 rtx x,
7812 {
7817 }
7819 /* Implement ASM_OUTPUT_POOL_EPILOGUE. */
7820 void
7822 HOST_WIDE_INT offset)
7823 {
7824 /* When using per-function literal pools, we must ensure that any code
7825 section is aligned to the minimal instruction length, lest we get
7826 errors from the assembler re "unaligned instructions". */
7829 }
7831 /* Costs. */
7833 /* Helper function for rtx cost calculation. Strip a shift expression
7834 from X. Returns the inner operand if successful, or the original
7835 expression on failure. */
7836 static rtx
7838 {
7841 /* We accept both ROTATERT and ROTATE: since the RHS must be a constant
7842 we can convert both to ROR during final output. */
7857 }
7859 /* Helper function for rtx cost calculation. Strip an extend
7860 expression from X. Returns the inner operand if successful, or the
7861 original expression on failure. We deal with a number of possible
7862 canonicalization variations here. If STRIP_SHIFT is true, then
7863 we can strip off a shift also. */
7864 static rtx
7866 {
7867 scalar_int_mode mode;
7873 /* Zero and sign extraction of a widened value. */
7881 /* It can also be represented (for zero-extend) as an AND with an
7882 immediate. */
7891 /* Now handle extended register, as this may also have an optional
7892 left shift by 1..4. */
7907 }
7909 /* Return true iff CODE is a shift supported in combination
7910 with arithmetic instructions. */
7912 static bool
7914 {
7916 }
7919 /* Return true iff X is a cheap shift without a sign extend. */
7921 static bool
7923 {
7949 }
7951 /* Helper function for rtx cost calculation. Calculate the cost of
7952 a MULT or ASHIFT, which may be part of a compound PLUS/MINUS rtx.
7953 Return the calculated cost of the expression, recursing manually in to
7954 operands where needed. */
7956 static int
7958 {
7974 /* Integer multiply/fma. */
7976 {
7977 /* The multiply will be canonicalized as a shift, cost it as such. */
7981 {
7985 {
7987 {
7988 /* If the shift is considered cheap,
7989 then don't add any cost. */
7991 ;
7993 /* ARITH + shift-by-register. */
7996 /* ARITH + extended register. We don't have a cost field
7997 for ARITH+EXTEND+SHIFT, so use extend_arith here. */
7999 else
8000 /* ARITH + shift-by-immediate. */
8002 }
8003 else
8004 /* LSL (immediate). */
8007 }
8008 /* Strip extends as we will have costed them in the case above. */
8015 }
8017 /* MNEG or [US]MNEGL. Extract the NEG operand and indicate that it's a
8018 compound and let the below cases handle it. After all, MNEG is a
8019 special-case alias of MSUB. */
8021 {
8024 }
8026 /* Integer multiplies or FMAs have zero/sign extending variants. */
8031 {
8036 {
8038 /* SMADDL/UMADDL/UMSUBL/SMSUBL. */
8040 else
8041 /* MUL/SMULL/UMULL. */
8043 }
8046 }
8048 /* This is either an integer multiply or a MADD. In both cases
8049 we want to recurse and cost the operands. */
8054 {
8056 /* MADD/MSUB. */
8058 else
8059 /* MUL. */
8061 }
8064 }
8065 else
8066 {
8068 {
8069 /* Floating-point FMA/FMUL can also support negations of the
8070 operands, unless the rounding mode is upward or downward in
8071 which case FNMUL is different than FMUL with operand negation. */
8075 {
8080 }
8083 /* FMADD/FNMADD/FNMSUB/FMSUB. */
8085 else
8086 /* FMUL/FNMUL. */
8088 }
8093 }
8094 }
8096 static int
8098 machine_mode mode,
8099 addr_space_t as ATTRIBUTE_UNUSED,
8101 {
8109 {
8111 {
8112 /* This is a CONST or SYMBOL ref which will be split
8113 in a different way depending on the code model in use.
8114 Cost it through the generic infrastructure. */
8116 /* Divide through by the cost of one instruction to
8117 bring it to the same units as the address costs. */
8119 /* The cost is then the cost of preparing the address,
8120 followed by an immediate (possibly 0) offset. */
8122 }
8123 else
8124 {
8125 /* This is most likely a jump table from a case
8126 statement. */
8128 }
8129 }
8132 {
8144 else
8163 }
8167 {
8168 /* For the sake of calculating the cost of the shifted register
8169 component, we can treat same sized modes in the same way. */
8176 else
8177 /* We can't tell, or this is a 128-bit vector. */
8179 }
8182 }
8184 /* Return the cost of a branch. If SPEED_P is true then the compiler is
8185 optimizing for speed. If PREDICTABLE_P is true then the branch is predicted
8186 to be taken. */
8188 int
8190 {
8191 /* When optimizing for speed, use the cost of unpredictable branches. */
8197 else
8199 }
8201 /* Return true if the RTX X in mode MODE is a zero or sign extract
8202 usable in an ADD or SUB (extended register) instruction. */
8203 static bool
8205 {
8206 /* Catch add with a sign extract.
8207 This is add_<optab><mode>_multp2. */
8210 {
8221 op1))
8222 {
8224 }
8225 }
8226 /* The simple case <ARITH>, XD, XN, XM, [us]xt.
8227 No shift. */
8233 }
8235 static bool
8237 {
8239 {
8251 }
8252 }
8254 /* Return true iff X is an rtx that will match an extr instruction
8255 i.e. as described in the *extr<mode>5_insn family of patterns.
8256 OP0 and OP1 will be set to the operands of the shifts involved
8257 on success and will be NULL_RTX otherwise. */
8259 static bool
8261 {
8263 scalar_int_mode mode;
8278 {
8279 /* Canonicalise locally to ashift in op0, lshiftrt in op1. */
8291 {
8295 }
8296 }
8299 }
8301 /* Calculate the cost of calculating (if_then_else (OP0) (OP1) (OP2)),
8302 storing it in *COST. Result is true if the total cost of the operation
8303 has now been calculated. */
8304 static bool
8306 {
8307 rtx inner;
8308 rtx comparator;
8312 {
8316 }
8317 else
8318 {
8322 }
8325 {
8326 /* Conditional branch. */
8329 else
8330 {
8332 {
8334 {
8335 /* TBZ/TBNZ/CBZ/CBNZ. */
8337 /* TBZ/TBNZ. */
8340 else
8341 /* CBZ/CBNZ. */
8345 }
8346 }
8348 {
8349 /* TBZ/TBNZ. */
8352 }
8353 }
8354 }
8356 {
8357 /* CCMP. */
8359 {
8360 /* Increase cost of CCMP reg, 0, imm, CC to prefer CMP reg, 0. */
8364 {
8371 else
8373 }
8375 }
8377 /* It's a conditional operation based on the status flags,
8378 so it must be some flavor of CSEL. */
8380 /* CSNEG, CSINV, and CSINC are handled for free as part of CSEL. */
8386 {
8387 /* CSEL with zero-extension (*cmovdi_insn_uxtw). */
8390 }
8395 }
8397 /* We don't know what this is, cost all operands. */
8399 }
8401 /* Check whether X is a bitfield operation of the form shift + extend that
8402 maps down to a UBFIZ/SBFIZ/UBFX/SBFX instruction. If so, return the
8403 operand to which the bitfield operation is applied. Otherwise return
8404 NULL_RTX. */
8406 static rtx
8408 {
8422 {
8440 }
8443 }
8445 /* Return true if the mask and a shift amount from an RTX of the form
8446 (x << SHFT_AMNT) & MASK are valid to combine into a UBFIZ instruction of
8447 mode MODE. See the *andim_ashift<mode>_bfiz pattern. */
8449 bool
8451 rtx shft_amnt)
8452 {
8457 }
8459 /* Calculate the cost of calculating X, storing it in *COST. Result
8460 is true if the total cost of the operation has now been calculated. */
8461 static bool
8464 {
8469 scalar_int_mode int_mode;
8471 /* By default, assume that everything has equivalent cost to the
8472 cheapest instruction. Any additional costs are applied as a delta
8473 above this default. */
8477 {
8479 /* The cost depends entirely on the operands to SET. */
8485 {
8488 {
8502 }
8511 /* Fall through. */
8513 /* The cost is one per vector-register copied. */
8515 {
8518 }
8519 /* const0_rtx is in general free, but we will use an
8520 instruction to set a register to 0. */
8522 {
8523 /* The cost is 1 per register copied. */
8526 }
8527 else
8528 /* Cost is just the cost of the RHS of the set. */
8534 /* Bit-field insertion. Strip any redundant widening of
8535 the RHS to meet the width of the target. */
8546 {
8547 /* MOV immediate is assumed to always be cheap. */
8549 }
8550 else
8551 {
8552 /* BFM. */
8556 }
8561 /* We can't make sense of this, assume default cost. */
8564 }
8568 /* If an instruction can incorporate a constant within the
8569 instruction, the instruction's expression avoids calling
8570 rtx_cost() on the constant. If rtx_cost() is called on a
8571 constant, then it is usually because the constant must be
8572 moved into a register by one or more instructions.
8574 The exception is constant 0, which can be expressed
8575 as XZR/WZR and is therefore free. The exception to this is
8576 if we have (set (reg) (const0_rtx)) in which case we must cost
8577 the move. However, we can catch that when we cost the SET, so
8578 we don't need to consider that here. */
8581 else
8582 {
8583 /* To an approximation, building any other constant is
8584 proportionally expensive to the number of instructions
8585 required to build that constant. This is true whether we
8586 are compiling for SPEED or otherwise. */
8591 }
8596 /* First determine number of instructions to do the move
8597 as an integer constant. */
8601 {
8607 ? SImode
8613 }
8616 {
8617 /* mov[df,sf]_aarch64. */
8619 /* FMOV (scalar immediate). */
8622 {
8623 /* This will be a load from memory. */
8626 else
8628 }
8629 else
8630 /* Otherwise this is +0.0. We get this using MOVI d0, #0
8631 or MOV v0.s[0], wzr - neither of which are modeled by the
8632 cost tables. Just use the default cost. */
8633 {
8634 }
8635 }
8641 {
8642 /* For loads we want the base cost of a load, plus an
8643 approximation for the additional cost of the addressing
8644 mode. */
8658 }
8666 {
8668 {
8669 /* FNEG. */
8671 }
8673 }
8676 {
8679 {
8680 /* CSETM. */
8683 }
8685 /* Cost this as SUB wzr, X. */
8689 }
8692 {
8693 /* Support (neg(fma...)) as a single instruction only if
8694 sign of zeros is unimportant. This matches the decision
8695 making in aarch64.md. */
8697 {
8698 /* FNMADD. */
8701 }
8703 {
8704 /* FNMUL. */
8707 }
8709 /* FNEG. */
8712 }
8719 {
8722 else
8724 }
8734 {
8738 }
8741 {
8742 /* TODO: A write to the CC flags possibly costs extra, this
8743 needs encoding in the cost tables. */
8746 /* ANDS. */
8748 {
8751 }
8754 {
8755 /* ADDS (and CMN alias). */
8758 }
8761 {
8762 /* SUBS. */
8765 }
8770 {
8771 /* COMPARE of ZERO_EXTRACT form of TST-immediate.
8772 Handle it here directly rather than going to cost_logic
8773 since we know the immediate generated for the TST is valid
8774 so we can avoid creating an intermediate rtx for it only
8775 for costing purposes. */
8782 }
8785 {
8786 /* CMN. */
8793 }
8795 /* CMP.
8797 Compare can freely swap the order of operands, and
8798 canonicalization puts the more complex operation first.
8799 But the integer MINUS logic expects the shift/extend
8800 operation in op1. */
8803 {
8806 }
8808 }
8811 {
8812 /* FCMP. */
8817 {
8819 /* FCMP supports constant 0.0 for no extra cost. */
8821 }
8823 }
8826 {
8827 /* Vector compare. */
8832 {
8833 /* Vector cm (eq|ge|gt|lt|le) supports constant 0.0 for no extra
8834 cost. */
8836 }
8838 }
8842 {
8846 cost_minus:
8849 /* Detect valid immediates. */
8855 {
8857 /* SUB(S) (immediate). */
8860 }
8862 /* Look for SUB (extended register). */
8865 {
8873 }
8877 /* Cost this as an FMA-alike operation. */
8881 {
8884 speed);
8886 }
8891 {
8893 {
8894 /* Vector SUB. */
8896 }
8898 {
8899 /* SUB(S). */
8901 }
8903 {
8904 /* FSUB. */
8906 }
8907 }
8909 }
8912 {
8913 rtx new_op0;
8918 cost_plus:
8921 {
8922 /* CSINC. */
8926 }
8931 {
8935 /* ADD (immediate). */
8938 }
8942 /* Look for ADD (extended register). */
8945 {
8953 }
8955 /* Strip any extend, leave shifts behind as we will
8956 cost them through mult_cost. */
8961 {
8963 speed);
8965 }
8970 {
8972 {
8973 /* Vector ADD. */
8975 }
8977 {
8978 /* ADD. */
8980 }
8982 {
8983 /* FADD. */
8985 }
8986 }
8988 }
8994 {
8997 else
8999 }
9004 {
9008 {
9011 else
9013 }
9015 }
9018 {
9025 }
9026 /* Fall through. */
9029 cost_logic:
9034 {
9038 }
9046 {
9047 /* This is a UBFM/SBFM. */
9052 }
9055 {
9057 {
9058 /* We have a mask + shift version of a UBFIZ
9059 i.e. the *andim_ashift<mode>_bfiz pattern. */
9063 {
9070 }
9072 {
9073 /* We possibly get the immediate for free, this is not
9074 modelled. */
9081 }
9082 }
9083 else
9084 {
9087 /* Handle ORN, EON, or BIC. */
9093 /* If we had a shift on op0 then this is a logical-shift-
9094 by-register/immediate operation. Otherwise, this is just
9095 a logical operation. */
9097 {
9099 {
9100 /* Shift by immediate. */
9103 else
9105 }
9106 else
9108 }
9110 /* In both cases we want to cost both operands. */
9117 }
9118 }
9126 {
9127 /* Vector NOT. */
9130 }
9132 /* MVN-shifted-reg. */
9134 {
9141 }
9142 /* EON can have two forms: (xor (not a) b) but also (not (xor a b)).
9143 Handle the second form here taking care that 'a' in the above can
9144 be a shift. */
9146 {
9155 {
9158 else
9160 }
9163 }
9164 /* MVN. */
9173 /* If a value is written in SI mode, then zero extended to DI
9174 mode, the operation will in general be free as a write to
9175 a 'w' register implicitly zeroes the upper bits of an 'x'
9176 register. However, if this is
9178 (set (reg) (zero_extend (reg)))
9180 we must cost the explicit register move. */
9184 {
9187 /* If OP_COST is non-zero, then the cost of the zero extend
9188 is effectively the cost of the inner operation. Otherwise
9189 we have a MOV instruction and we take the cost from the MOV
9190 itself. This is true independently of whether we are
9191 optimizing for space or time. */
9196 }
9198 {
9199 /* All loads can zero extend to any size for free. */
9202 }
9206 {
9211 }
9214 {
9216 {
9217 /* UMOV. */
9219 }
9220 else
9221 {
9222 /* We generate an AND instead of UXTB/UXTH. */
9224 }
9225 }
9230 {
9231 /* LDRSH. */
9233 {
9240 }
9242 }
9246 {
9251 }
9254 {
9257 else
9259 }
9267 {
9269 {
9271 {
9272 /* Vector shift (immediate). */
9274 }
9275 else
9276 {
9277 /* LSL (immediate), UBMF, UBFIZ and friends. These are all
9278 aliases. */
9280 }
9281 }
9283 /* We can incorporate zero/sign extend for free. */
9290 }
9291 else
9292 {
9294 {
9296 /* Vector shift (register). */
9298 }
9299 else
9300 {
9302 /* LSLV. */
9309 {
9311 /* We already demanded XEXP (op1, 0) to be REG_P, so
9312 don't recurse into it. */
9314 }
9315 }
9317 }
9327 {
9328 /* ASR (immediate) and friends. */
9330 {
9333 else
9335 }
9339 }
9340 else
9341 {
9343 {
9345 /* Vector shift (register). */
9347 }
9348 else
9349 {
9351 /* ASR (register) and friends. */
9358 {
9360 /* We already demanded XEXP (op1, 0) to be REG_P, so
9361 don't recurse into it. */
9363 }
9364 }
9366 }
9372 {
9373 /* LDR. */
9376 }
9379 {
9380 /* ADRP, followed by ADD. */
9384 }
9387 {
9388 /* ADR. */
9391 }
9394 {
9395 /* One extra load instruction, after accessing the GOT. */
9399 }
9404 /* ADRP/ADD (immediate). */
9411 /* UBFX/SBFX. */
9413 {
9416 else
9418 }
9420 /* We can trust that the immediates used will be correct (there
9421 are no by-register forms), so we need only cost op0. */
9427 /* aarch64_rtx_mult_cost always handles recursion to its
9428 operands. */
9432 /* We can expand signed mod by power of 2 using a NEGS, two parallel
9433 ANDs and a CSNEG. Assume here that CSNEG is the same as the cost of
9434 an unconditional negate. This case should only ever be reached through
9435 the set_smod_pow2_cheap check in expmed.c. */
9439 {
9440 /* We expand to 4 instructions. Reset the baseline. */
9448 }
9450 /* Fall-through. */
9453 {
9454 /* Slighly prefer UMOD over SMOD. */
9461 }
9468 {
9472 /* There is no integer SQRT, so only DIV and UDIV can get
9473 here. */
9475 /* Slighly prefer UDIV over SDIV. */
9477 else
9479 }
9505 {
9508 else
9510 }
9512 /* FMSUB, FNMADD, and FNMSUB are free. */
9519 /* aarch64_fnma4_elt_to_64v2df has the NEG as operand 1,
9520 and the by-element operand as operand 0. */
9524 /* Catch vector-by-element operations. The by-element operand can
9525 either be (vec_duplicate (vec_select (x))) or just
9526 (vec_select (x)), depending on whether we are multiplying by
9527 a vector or a scalar.
9529 Canonicalization is not very good in these cases, FMA4 will put the
9530 by-element operand as operand 0, FNMA4 will have it as operand 1. */
9541 /* If the remaining parameters are not registers,
9542 get the cost to put them into registers. */
9556 {
9558 {
9559 /*Vector truncate. */
9561 }
9562 else
9564 }
9569 {
9571 {
9572 /*Vector conversion. */
9574 }
9575 else
9577 }
9583 /* Strip the rounding part. They will all be implemented
9584 by the fcvt* family of instructions anyway. */
9586 {
9595 }
9598 {
9601 else
9603 }
9605 /* We can combine fmul by a power of 2 followed by a fcvt into a single
9606 fixed-point fcvt. */
9611 {
9615 }
9622 {
9623 /* ABS (vector). */
9626 }
9628 {
9631 /* FABD, which is analogous to FADD. */
9633 {
9640 }
9641 /* Simple FABS is analogous to FNEG. */
9644 }
9645 else
9646 {
9647 /* Integer ABS will either be split to
9648 two arithmetic instructions, or will be an ABS
9649 (scalar), which we don't model. */
9653 }
9659 {
9662 else
9663 {
9664 /* FMAXNM/FMINNM/FMAX/FMIN.
9665 TODO: This may not be accurate for all implementations, but
9666 we do not model this in the cost tables. */
9668 }
9669 }
9673 /* The floating point round to integer frint* instructions. */
9675 {
9680 }
9683 {
9688 }
9693 /* Decompose <su>muldi3_highpart. */
9695 mode == DImode
9696 /* (lshiftrt:TI */
9699 /* (mult:TI */
9701 /* (ANY_EXTEND:TI (reg:DI))
9702 (ANY_EXTEND:TI (reg:DI))) */
9709 /* (const_int 64) */
9712 {
9713 /* UMULH/SMULH. */
9721 }
9723 /* Fall through. */
9726 }
9734 }
9736 /* Wrapper around aarch64_rtx_costs, dumps the partial, or total cost
9737 calculated for X. This cost is stored in *COST. Returns true
9738 if the total cost of X was calculated. */
9739 static bool
9742 {
9747 {
9752 }
9755 }
9757 static int
9760 {
9766 /* Caller save and pointer regs are equivalent to GENERAL_REGS. */
9773 /* Moving between GPR and stack cost is the same as GP2GP. */
9778 /* To/From the stack register, we move via the gprs. */
9784 {
9785 /* 128-bit operations on general registers require 2 instructions. */
9793 /* When AdvSIMD instructions are disabled it is not possible to move
9794 a 128-bit value directly between Q registers. This is handled in
9795 secondary reload. A general register is used as a scratch to move
9796 the upper DI value and the lower DI value is moved directly,
9797 hence the cost is the sum of three moves. */
9802 }
9812 }
9814 static int
9816 reg_class_t rclass ATTRIBUTE_UNUSED,
9818 {
9820 }
9822 /* Return true if it is safe and beneficial to use the approximate rsqrt optabs
9823 to optimize 1.0/sqrt. */
9825 static bool
9827 {
9829 && flag_unsafe_math_optimizations
9833 }
9835 /* Function to decide when to use the approximate reciprocal square root
9836 builtin. */
9838 static tree
9840 {
9846 }
9850 /* Select reciprocal square root initial estimate insn depending on machine
9851 mode. */
9853 static rsqrte_type
9855 {
9857 {
9864 }
9865 }
9869 /* Select reciprocal square root series step insn depending on machine mode. */
9871 static rsqrts_type
9873 {
9875 {
9882 }
9883 }
9885 /* Emit instruction sequence to compute either the approximate square root
9886 or its approximate reciprocal, depending on the flag RECP, and return
9887 whether the sequence was emitted or not. */
9889 bool
9891 {
9895 {
9898 }
9901 {
9908 || flag_trapping_math
9909 || !flag_unsafe_math_optimizations
9912 }
9913 else
9914 /* Caller assumes we cannot fail. */
9920 /* When calculating the approximate square root, compare the
9921 argument with 0.0 and create a mask. */
9927 /* Estimate the approximate reciprocal square root. */
9931 /* Iterate over the series twice for SF and thrice for DF. */
9934 /* Optionally iterate over the series once less for faster performance
9935 while sacrificing the accuracy. */
9938 iterations--;
9940 /* Iterate over the series to calculate the approximate reciprocal square
9941 root. */
9944 {
9952 }
9955 {
9956 /* Qualify the approximate reciprocal square root when the argument is
9957 0.0 by squashing the intermediary result to 0.0. */
9963 /* Calculate the approximate square root. */
9965 }
9967 /* Finalize the approximation. */
9971 }
9975 /* Select reciprocal initial estimate insn depending on machine mode. */
9977 static recpe_type
9979 {
9981 {
9988 }
9989 }
9993 /* Select reciprocal series step insn depending on machine mode. */
9995 static recps_type
9997 {
9999 {
10006 }
10007 }
10009 /* Emit the instruction sequence to compute the approximation for the division
10010 of NUM by DEN in QUO and return whether the sequence was emitted or not. */
10012 bool
10014 {
10025 || flag_trapping_math
10026 || !flag_unsafe_math_optimizations
10034 /* Estimate the approximate reciprocal. */
10038 /* Iterate over the series twice for SF and thrice for DF. */
10041 /* Optionally iterate over the series once less for faster performance,
10042 while sacrificing the accuracy. */
10044 iterations--;
10046 /* Iterate over the series to calculate the approximate reciprocal. */
10049 {
10054 }
10057 {
10058 /* As the approximate reciprocal of DEN is already calculated, only
10059 calculate the approximate division when NUM is not 1.0. */
10062 }
10064 /* Finalize the approximation. */
10067 }
10069 /* Return the number of instructions that can be issued per cycle. */
10070 static int
10072 {
10074 }
10076 static int
10078 {
10082 }
10085 /* Implement TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD as
10086 autopref_multipass_dfa_lookahead_guard from haifa-sched.c. It only
10087 has an effect if PARAM_SCHED_AUTOPREF_QUEUE_DEPTH > 0. */
10089 static int
10092 {
10094 }
10097 /* Vectorizer cost model target hooks. */
10099 /* Implement targetm.vectorize.builtin_vectorization_cost. */
10100 static int
10102 tree vectype,
10104 {
10113 {
10164 }
10165 }
10167 /* Implement targetm.vectorize.add_stmt_cost. */
10168 static unsigned
10172 {
10177 {
10182 /* Statements in an inner loop relative to the loop being
10183 vectorized are weighted more heavily. The value here is
10184 arbitrary and could potentially be improved with analysis. */
10190 }
10193 }
10197 /* Parse the TO_PARSE string and put the architecture struct that it
10198 selects into RES and the architectural features into ISA_FLAGS.
10199 Return an aarch64_parse_opt_result describing the parse result.
10200 If there is an error parsing, RES and ISA_FLAGS are left unchanged. */
10202 static enum aarch64_parse_opt_result
10205 {
10217 else
10224 /* Loop through the list of supported ARCHes to find a match. */
10226 {
10228 {
10232 {
10233 /* TO_PARSE string contains at least one extension. */
10234 enum aarch64_parse_opt_result ext_res
10239 }
10240 /* Extension parsing was successful. Confirm the result
10241 arch and ISA flags. */
10245 }
10246 }
10248 /* ARCH name not found in list. */
10250 }
10252 /* Parse the TO_PARSE string and put the result tuning in RES and the
10253 architecture flags in ISA_FLAGS. Return an aarch64_parse_opt_result
10254 describing the parse result. If there is an error parsing, RES and
10255 ISA_FLAGS are left unchanged. */
10257 static enum aarch64_parse_opt_result
10260 {
10272 else
10279 /* Loop through the list of supported CPUs to find a match. */
10281 {
10283 {
10288 {
10289 /* TO_PARSE string contains at least one extension. */
10290 enum aarch64_parse_opt_result ext_res
10295 }
10296 /* Extension parsing was successfull. Confirm the result
10297 cpu and ISA flags. */
10301 }
10302 }
10304 /* CPU name not found in list. */
10306 }
10308 /* Parse the TO_PARSE string and put the cpu it selects into RES.
10309 Return an aarch64_parse_opt_result describing the parse result.
10310 If the parsing fails the RES does not change. */
10312 static enum aarch64_parse_opt_result
10314 {
10320 /* Loop through the list of supported CPUs to find a match. */
10322 {
10324 {
10327 }
10328 }
10330 /* CPU name not found in list. */
10332 }
10334 /* Parse TOKEN, which has length LENGTH to see if it is an option
10335 described in FLAG. If it is, return the index bit for that fusion type.
10336 If not, error (printing OPTION_NAME) and return zero. */
10338 static unsigned int
10343 {
10345 {
10349 }
10353 }
10355 /* Parse OPTION which is a comma-separated list of flags to enable.
10356 FLAGS gives the list of flags we understand, INITIAL_STATE gives any
10357 default state we inherit from the CPU tuning structures. OPTION_NAME
10358 gives the top-level option we are parsing in the -moverride string,
10359 for use in error messages. */
10361 static unsigned int
10366 {
10373 {
10376 token_length,
10377 flags,
10378 option_name);
10379 /* If we find "none" (or, for simplicity's sake, an error) anywhere
10380 in the token stream, reset the supported operations. So:
10382 adrp+add.cmp+branch.none.adrp+add
10384 would have the result of turning on only adrp+add fusion. */
10390 }
10392 /* We ended with a comma, print something. */
10394 {
10397 }
10399 /* We still have one more token to parse. */
10402 token_length,
10403 flags,
10404 option_name);
10410 }
10412 /* Support for overriding instruction fusion. */
10414 static void
10417 {
10419 aarch64_fusible_pairs,
10422 }
10424 /* Support for overriding other tuning flags. */
10426 static void
10429 {
10430 tune->extra_tuning_flags
10432 aarch64_tuning_flags,
10435 }
10437 /* Parse TOKEN, which has length LENGTH to see if it is a tuning option
10438 we understand. If it is, extract the option string and handoff to
10439 the appropriate function. */
10441 void
10445 {
10451 {
10454 }
10456 /* Get the length of the option name. */
10458 /* Skip the '=' to get to the option string. */
10459 option_part++;
10462 {
10464 {
10467 }
10468 }
10472 }
10474 /* A checking mechanism for the implementation of the tls size. */
10476 static void
10478 {
10483 {
10485 /* Both the default and maximum TLS size allowed under tiny is 1M which
10486 needs two instructions to address, so we clamp the size to 24. */
10491 /* The maximum TLS size allowed under small is 4G. */
10496 /* The maximum TLS size allowed under large is 16E.
10497 FIXME: 16E should be 64bit, we only support 48bit offset now. */
10503 }
10506 }
10508 /* Parse STRING looking for options in the format:
10509 string :: option:string
10510 option :: name=substring
10511 name :: {a-z}
10512 substring :: defined by option. */
10514 static void
10517 {
10528 {
10530 /* Make this substring look like a string. */
10534 }
10536 /* One last option to parse. */
10539 }
10542 static void
10544 {
10545 /* PR 70044: We have to be careful about being called multiple times for the
10546 same function. This means all changes should be repeatable. */
10548 /* Set aarch64_use_frame_pointer based on -fno-omit-frame-pointer.
10549 Disable the frame pointer flag so the mid-end will not use a frame
10550 pointer in leaf functions in order to support -fomit-leaf-frame-pointer.
10551 Set x_flag_omit_frame_pointer to the special value 2 to differentiate
10552 between -fomit-frame-pointer (1) and -fno-omit-frame-pointer (2). */
10557 /* If not optimizing for size, set the default
10558 alignment to what the target wants. */
10560 {
10567 }
10569 /* We default to no pc-relative literal loads. */
10573 /* If -mpc-relative-literal-loads is set on the command line, this
10574 implies that the user asked for PC relative literal loads. */
10578 /* In the tiny memory model it makes no sense to disallow PC relative
10579 literal pool loads. */
10584 /* When enabling the lower precision Newton series for the square root, also
10585 enable it for the reciprocal square root, since the latter is an
10586 intermediary step for the former. */
10589 }
10591 /* 'Unpack' up the internal tuning structs and update the options
10592 in OPTS. The caller must have set up selected_tune and selected_arch
10593 as all the other target-specific codegen decisions are
10594 derived from them. */
10596 void
10598 {
10601 /* Make a copy of the tuning parameters attached to the core, which
10602 we may later overwrite. */
10610 /* This target defaults to strict volatile bitfields. */
10619 {
10631 }
10633 /* We don't mind passing in global_options_set here as we don't use
10634 the *options_set structs anyway. */
10636 queue_depth,
10640 /* Set up parameters to be used in prefetching algorithm. Do not
10641 override the defaults unless we are tuning for a core we have
10642 researched values for. */
10674 /* Use the alternative scheduling-pressure algorithm by default. */
10679 /* Enable sw prefetching at specified optimization level for
10680 CPUS that have prefetch. Lower optimization level threshold by 1
10681 when profiling is enabled. */
10689 }
10691 /* Print a hint with a suggestion for a core or architecture name that
10692 most closely resembles what the user passed in STR. ARCH is true if
10693 the user is asking for an architecture name. ARCH is false if the user
10694 is asking for a core name. */
10696 static void
10698 {
10704 #ifdef HAVE_LOCAL_CPU_DETECT
10705 /* Add also "native" as possible value. */
10708 #endif
10715 else
10719 }
10721 /* Print a hint with a suggestion for a core name that most closely resembles
10722 what the user passed in STR. */
10726 {
10728 }
10730 /* Print a hint with a suggestion for an architecture name that most closely
10731 resembles what the user passed in STR. */
10735 {
10737 }
10739 /* Validate a command-line -mcpu option. Parse the cpu and extensions (if any)
10740 specified in STR and throw errors if appropriate. Put the results if
10741 they are valid in RES and ISA_FLAGS. Return whether the option is
10742 valid. */
10744 static bool
10747 {
10748 enum aarch64_parse_opt_result parse_res
10755 {
10768 }
10771 }
10773 /* Validate a command-line -march option. Parse the arch and extensions
10774 (if any) specified in STR and throw errors if appropriate. Put the
10775 results, if they are valid, in RES and ISA_FLAGS. Return whether the
10776 option is valid. */
10778 static bool
10781 {
10782 enum aarch64_parse_opt_result parse_res
10789 {
10802 }
10805 }
10807 /* Validate a command-line -mtune option. Parse the cpu
10808 specified in STR and throw errors if appropriate. Put the
10809 result, if it is valid, in RES. Return whether the option is
10810 valid. */
10812 static bool
10814 {
10815 enum aarch64_parse_opt_result parse_res
10822 {
10832 }
10834 }
10836 /* Return the CPU corresponding to the enum CPU.
10837 If it doesn't specify a cpu, return the default. */
10841 {
10845 /* The & 0x3f is to extract the bottom 6 bits that encode the
10846 default cpu as selected by the --with-cpu GCC configure option
10847 in config.gcc.
10848 ???: The whole TARGET_CPU_DEFAULT and AARCH64_CPU_DEFAULT_FLAGS
10849 flags mechanism should be reworked to make it more sane. */
10851 }
10853 /* Return the architecture corresponding to the enum ARCH.
10854 If it doesn't specify a valid architecture, return the default. */
10858 {
10865 }
10867 /* Return the VG value associated with -msve-vector-bits= value VALUE. */
10869 static poly_uint16
10871 {
10872 /* For now generate vector-length agnostic code for -msve-vector-bits=128.
10873 This ensures we can clearly distinguish SVE and Advanced SIMD modes when
10874 deciding which .md file patterns to use and when deciding whether
10875 something is a legitimate address or constant. */
10878 else
10880 }
10882 /* Implement TARGET_OPTION_OVERRIDE. This is called once in the beginning
10883 and is used to parse the -m{cpu,tune,arch} strings and setup the initial
10884 tuning structs. In particular it must set selected_tune and
10885 aarch64_isa_flags that define the available ISA features and tuning
10886 decisions. It must also set selected_arch as this will be used to
10887 output the .arch asm tags for each function. */
10889 static void
10891 {
10904 /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
10905 If either of -march or -mtune is given, they override their
10906 respective component of -mcpu. */
10918 /* If the user did not specify a processor, choose the default
10919 one for them. This will be the CPU set during configuration using
10920 --with-cpu, otherwise it is "generic". */
10922 {
10924 {
10928 }
10929 else
10930 {
10931 /* Get default configure-time CPU. */
10934 }
10938 }
10939 /* If both -mcpu and -march are specified check that they are architecturally
10940 compatible, warn if they're not and prefer the -march ISA flags. */
10942 {
10944 {
10948 }
10953 }
10954 else
10955 {
10956 /* -mcpu but no -march. */
10963 }
10965 /* Set the arch as well as we will need it when outputing
10966 the .arch directive in assembly. */
10968 {
10971 }
10976 #ifndef HAVE_AS_MABI_OPTION
10977 /* The compiler may have been configured with 2.23.* binutils, which does
10978 not have support for ILP32. */
10981 #endif
10983 /* Convert -msve-vector-bits to a VG count. */
10989 /* Make sure we properly set up the explicit options. */
11000 /* Save these options as the default ones in case we push and pop them later
11001 while processing functions with potential target attributes. */
11002 target_option_default_node = target_option_current_node
11004 }
11006 /* Implement targetm.override_options_after_change. */
11008 static void
11010 {
11012 }
11016 {
11020 }
11022 void
11024 {
11026 }
11028 /* A checking mechanism for the implementation of the various code models. */
11029 static void
11031 {
11033 {
11035 {
11040 #ifdef HAVE_AS_SMALL_PIC_RELOCS
11042 ? AARCH64_CMODEL_SMALL_PIC
11044 #else
11046 #endif
11054 }
11055 }
11056 else
11058 }
11060 /* Implement TARGET_OPTION_SAVE. */
11062 static void
11064 {
11066 }
11068 /* Implements TARGET_OPTION_RESTORE. Restore the backend codegen decisions
11069 using the information saved in PTR. */
11071 static void
11073 {
11081 }
11083 /* Implement TARGET_OPTION_PRINT. */
11085 static void
11087 {
11098 }
11102 void
11104 {
11106 }
11108 /* Restore or save the TREE_TARGET_GLOBALS from or to NEW_TREE.
11109 Used by aarch64_set_current_function and aarch64_pragma_target_parse to
11110 make sure optab availability predicates are recomputed when necessary. */
11112 void
11114 {
11119 else
11121 }
11123 /* Implement TARGET_SET_CURRENT_FUNCTION. Unpack the codegen decisions
11124 like tuning and ISA features from the DECL_FUNCTION_SPECIFIC_TARGET
11125 of the function, if such exists. This function may be called multiple
11126 times on a single function so use aarch64_previous_fndecl to avoid
11127 setting up identical state. */
11129 static void
11131 {
11135 tree old_tree = (aarch64_previous_fndecl
11141 /* If current function has no attributes but the previous one did,
11142 use the default node. */
11146 /* If nothing to do, return. #pragma GCC reset or #pragma GCC pop to
11147 the default have been handled by aarch64_save_restore_target_globals from
11148 aarch64_pragma_target_parse. */
11154 /* First set the target options. */
11158 }
11160 /* Enum describing the various ways we can handle attributes.
11161 In many cases we can reuse the generic option handling machinery. */
11163 enum aarch64_attr_opt_type
11164 {
11168 aarch64_attr_custom /* Attribute requires a custom handling function. */
11169 };
11171 /* All the information needed to handle a target attribute.
11172 NAME is the name of the attribute.
11173 ATTR_TYPE specifies the type of behavior of the attribute as described
11174 in the definition of enum aarch64_attr_opt_type.
11175 ALLOW_NEG is true if the attribute supports a "no-" form.
11176 HANDLER is the function that takes the attribute string as an argument
11177 It is needed only when the ATTR_TYPE is aarch64_attr_custom.
11178 OPT_NUM is the enum specifying the option that the attribute modifies.
11179 This is needed for attributes that mirror the behavior of a command-line
11180 option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
11181 aarch64_attr_enum. */
11183 struct aarch64_attribute_info
11184 {
11190 };
11192 /* Handle the ARCH_STR argument to the arch= target attribute. */
11194 static bool
11196 {
11198 enum aarch64_parse_opt_result parse_res
11202 {
11207 }
11210 {
11223 }
11226 }
11228 /* Handle the argument CPU_STR to the cpu= target attribute. */
11230 static bool
11232 {
11234 enum aarch64_parse_opt_result parse_res
11238 {
11246 }
11249 {
11262 }
11265 }
11267 /* Handle the argument STR to the tune= target attribute. */
11269 static bool
11271 {
11273 enum aarch64_parse_opt_result parse_res
11277 {
11282 }
11285 {
11292 }
11295 }
11297 /* Parse an architecture extensions target attribute string specified in STR.
11298 For example "+fp+nosimd". Show any errors if needed. Return TRUE
11299 if successful. Update aarch64_isa_flags to reflect the ISA features
11300 modified. */
11302 static bool
11304 {
11308 /* We allow "+nothing" in the beginning to clear out all architectural
11309 features if the user wants to handpick specific features. */
11311 {
11314 }
11319 {
11322 }
11325 {
11336 }
11339 }
11341 /* The target attributes that we support. On top of these we also support just
11342 ISA extensions, like __attribute__ ((target ("+crc"))), but that case is
11343 handled explicitly in aarch64_process_one_target_attr. */
11346 {
11348 OPT_mgeneral_regs_only },
11350 OPT_mfix_cortex_a53_835769 },
11352 OPT_mfix_cortex_a53_843419 },
11356 OPT_momit_leaf_frame_pointer },
11359 OPT_march_ },
11362 OPT_mtune_ },
11364 OPT_msign_return_address_ },
11366 };
11368 /* Parse ARG_STR which contains the definition of one target attribute.
11369 Show appropriate errors if any or return true if the attribute is valid. */
11371 static bool
11373 {
11379 {
11382 }
11387 /* Skip leading whitespace. */
11389 str_to_check++;
11391 /* We have something like __attribute__ ((target ("+fp+nosimd"))).
11392 It is easier to detect and handle it explicitly here rather than going
11393 through the machinery for the rest of the target attributes in this
11394 function. */
11399 {
11402 }
11405 /* If we found opt=foo then terminate STR_TO_CHECK at the '='
11406 and point ARG to "foo". */
11408 {
11410 arg++;
11411 }
11415 {
11416 /* If the names don't match up, or the user has given an argument
11417 to an attribute that doesn't accept one, or didn't give an argument
11418 to an attribute that expects one, fail to match. */
11427 {
11430 }
11432 /* If the name matches but the attribute does not allow "no-" versions
11433 then we can't match. */
11435 {
11438 }
11441 {
11442 /* Has a custom handler registered.
11443 For example, cpu=, arch=, tune=. */
11450 /* Either set or unset a boolean option. */
11452 {
11460 }
11461 /* Set or unset a bit in the target_flags. aarch64_handle_option
11462 should know what mask to apply given the option number. */
11464 {
11466 /* We only need to specify the option number.
11467 aarch64_handle_option will know which mask to apply. */
11473 }
11474 /* Use the option setting machinery to set an option to an enum. */
11476 {
11483 {
11486 global_dc);
11487 }
11488 else
11489 {
11491 }
11493 }
11496 }
11497 }
11499 /* If we reached here we either have found an attribute and validated
11500 it or didn't match any. If we matched an attribute but its arguments
11501 were malformed we will have returned false already. */
11503 }
11505 /* Count how many times the character C appears in
11506 NULL-terminated string STR. */
11508 static unsigned int
11510 {
11513 {
11515 res++;
11517 str++;
11518 }
11521 }
11523 /* Parse the tree in ARGS that contains the target attribute information
11524 and update the global target options space. */
11526 bool
11528 {
11530 {
11531 do
11532 {
11535 {
11538 }
11543 }
11546 {
11549 }
11556 {
11559 }
11561 /* Used to catch empty spaces between commas i.e.
11562 attribute ((target ("attr1,,attr2"))). */
11565 /* Handle multiple target attributes separated by ','. */
11570 {
11571 num_attrs++;
11573 {
11576 }
11579 }
11582 {
11585 }
11588 }
11590 /* Implement TARGET_OPTION_VALID_ATTRIBUTE_P. This is used to
11591 process attribute ((target ("..."))). */
11593 static bool
11595 {
11598 tree old_optimize;
11602 /* If what we're processing is the current pragma string then the
11603 target option node is already stored in target_option_current_node
11604 by aarch64_pragma_target_parse in aarch64-c.c. Use that to avoid
11605 having to re-parse the string. This is especially useful to keep
11606 arm_neon.h compile times down since that header contains a lot
11607 of intrinsics enclosed in pragmas. */
11609 {
11612 }
11618 /* If the function changed the optimization levels as well as setting
11619 target options, start with the optimizations specified. */
11624 /* Save the current target options to restore at the end. */
11627 /* If fndecl already has some target attributes applied to it, unpack
11628 them so that we add this attribute on top of them, rather than
11629 overwriting them. */
11631 {
11637 }
11638 else
11644 /* Set up any additional state. */
11646 {
11648 /* Initialize SIMD builtins if we haven't already.
11649 Set current_target_pragma to NULL for the duration so that
11650 the builtin initialization code doesn't try to tag the functions
11651 being built with the attributes specified by any current pragma, thus
11652 going into an infinite recursion. */
11654 {
11659 }
11661 }
11662 else
11668 {
11673 }
11681 }
11683 /* Helper for aarch64_can_inline_p. In the case where CALLER and CALLEE are
11684 tri-bool options (yes, no, don't care) and the default value is
11685 DEF, determine whether to reject inlining. */
11687 static bool
11690 {
11691 /* If the callee doesn't care, always allow inlining. */
11695 /* If the caller doesn't care, always allow inlining. */
11699 /* Otherwise, allow inlining if either the callee and caller values
11700 agree, or if the callee is using the default value. */
11702 }
11704 /* Implement TARGET_CAN_INLINE_P. Decide whether it is valid
11705 to inline CALLEE into CALLER based on target-specific info.
11706 Make sure that the caller and callee have compatible architectural
11707 features. Then go through the other possible target attributes
11708 and see if they can block inlining. Try not to reject always_inline
11709 callees unless they are incompatible architecturally. */
11711 static bool
11713 {
11725 /* Callee's ISA flags should be a subset of the caller's. */
11730 /* Allow non-strict aligned functions inlining into strict
11731 aligned ones. */
11741 /* If the architectural features match up and the callee is always_inline
11742 then the other attributes don't matter. */
11754 /* Honour explicit requests to workaround errata. */
11767 /* If the user explicitly specified -momit-leaf-frame-pointer for the
11768 caller and calle and they don't match up, reject inlining. */
11775 /* If the callee has specific tuning overrides, respect them. */
11780 /* If the user specified tuning override strings for the
11781 caller and callee and they don't match up, reject inlining.
11782 We just do a string compare here, we don't analyze the meaning
11783 of the string, as it would be too costly for little gain. */
11791 }
11793 /* Return true if SYMBOL_REF X binds locally. */
11795 static bool
11797 {
11801 }
11803 /* Return true if SYMBOL_REF X is thread local */
11804 static bool
11806 {
11814 }
11816 /* Classify a TLS symbol into one of the TLS kinds. */
11817 enum aarch64_symbol_type
11819 {
11823 {
11830 {
11836 }
11847 else
11856 }
11857 }
11859 /* Return the correct method for accessing X + OFFSET, where X is either
11860 a SYMBOL_REF or LABEL_REF. */
11862 enum aarch64_symbol_type
11864 {
11866 {
11868 {
11883 }
11884 }
11887 {
11892 {
11894 /* When we retrieve symbol + offset address, we have to make sure
11895 the offset does not cause overflow of the final address. But
11896 we have no way of knowing the address of symbol at compile time
11897 so we can't accurately say if the distance between the PC and
11898 symbol + offset is outside the addressible range of +/-1M in the
11899 TINY code model. So we rely on images not being greater than
11900 1M and cap the offset at 1M and anything beyond 1M will have to
11901 be loaded using an alternative mechanism. Furthermore if the
11902 symbol is a weak reference to something that isn't known to
11903 resolve to a symbol in this module, then force to memory. */
11911 /* Same reasoning as the tiny code model, but the offset cap here is
11912 4G. */
11933 /* This is alright even in PIC code as the constant
11934 pool reference is always PC relative and within
11935 the same translation unit. */
11938 else
11943 }
11944 }
11946 /* By default push everything into the constant pool. */
11948 }
11950 bool
11952 {
11954 }
11956 bool
11958 {
11966 }
11968 /* Implement TARGET_LEGITIMATE_CONSTANT_P hook. Return true for constants
11969 that should be rematerialized rather than spilled. */
11971 static bool
11973 {
11974 /* Support CSE and rematerialization of common constants. */
11980 /* Do not allow vector struct mode constants for Advanced SIMD.
11981 We could support 0 and -1 easily, but they need support in
11982 aarch64-simd.md. */
11987 /* Only accept variable-length vector constants if they can be
11988 handled directly.
11990 ??? It would be possible to handle rematerialization of other
11991 constants via secondary reloads. */
11998 /* Accept polynomial constants that can be calculated by using the
11999 destination of a move as the sole temporary. Constants that
12000 require a second temporary cannot be rematerialized (they can't be
12001 forced to memory and also aren't legitimate constants). */
12002 poly_int64 offset;
12006 /* If an offset is being added to something else, we need to allow the
12007 base to be moved into the destination register, meaning that there
12008 are no free temporaries for the offset. */
12013 /* Do not allow const (plus (anchor_symbol, const_int)). */
12017 /* Treat symbols as constants. Avoid TLS symbols as they are complex,
12018 so spilling them is better than rematerialization. */
12022 /* Label references are always constant. */
12027 }
12029 rtx
12031 {
12037 /* Can return in any reg. */
12040 }
12042 /* On AAPCS systems, this is the "struct __va_list". */
12045 /* Implement TARGET_BUILD_BUILTIN_VA_LIST.
12046 Return the type to use as __builtin_va_list.
12048 AAPCS64 \S 7.1.4 requires that va_list be a typedef for a type defined as:
12050 struct __va_list
12051 {
12052 void *__stack;
12053 void *__gr_top;
12054 void *__vr_top;
12055 int __gr_offs;
12056 int __vr_offs;
12057 }; */
12059 static tree
12061 {
12062 tree va_list_name;
12065 /* Create the type. */
12067 /* Give it the required name. */
12069 TYPE_DECL,
12071 va_list_type);
12076 /* Create the fields. */
12079 ptr_type_node);
12082 ptr_type_node);
12085 ptr_type_node);
12088 integer_type_node);
12091 integer_type_node);
12093 /* Tell tree-stdarg pass about our internal offset fields.
12094 NOTE: va_list_gpr/fpr_counter_field are only used for tree comparision
12095 purpose to identify whether the code is updating va_list internal
12096 offset fields through irregular way. */
12118 /* Compute its layout. */
12122 }
12124 /* Implement TARGET_EXPAND_BUILTIN_VA_START. */
12125 static void
12127 {
12131 tree t;
12145 {
12148 }
12157 NULL_TREE);
12159 NULL_TREE);
12161 NULL_TREE);
12163 NULL_TREE);
12165 NULL_TREE);
12167 /* Emit code to initialize STACK, which points to the next varargs stack
12168 argument. CUM->AAPCS_STACK_SIZE gives the number of stack words used
12169 by named arguments. STACK is 8-byte aligned. */
12176 /* Emit code to initialize GRTOP, the top of the GR save area.
12177 virtual_incoming_args_rtx should have been 16 byte aligned. */
12182 /* Emit code to initialize VRTOP, the top of the VR save area.
12183 This address is gr_save_area_bytes below GRTOP, rounded
12184 down to the next 16-byte boundary. */
12194 /* Emit code to initialize GROFF, the offset from GRTOP of the
12195 next GPR argument. */
12200 /* Likewise emit code to initialize VROFF, the offset from FTOP
12201 of the next VR argument. */
12205 }
12207 /* Implement TARGET_GIMPLIFY_VA_ARG_EXPR. */
12209 static tree
12212 {
12213 tree addr;
12219 machine_mode mode;
12246 type,
12250 {
12251 /* No frontends can create types with variable-sized modes, so we
12252 shouldn't be asked to pass or return them. */
12255 /* TYPE passed in fp/simd registers. */
12267 {
12270 }
12273 {
12275 }
12276 }
12277 else
12278 {
12279 /* TYPE passed in general registers. */
12292 {
12294 }
12295 }
12297 /* Get a local temporary for the field value. */
12300 /* Emit code to branch if off >= 0. */
12306 {
12307 /* Emit: offs = (offs + 15) & -16. */
12313 }
12314 else
12317 /* Update ap.__[g|v]r_offs */
12322 /* String up. */
12326 /* [cond2] if (ap.__[g|v]r_offs > 0) */
12331 /* String up: make sure the assignment happens before the use. */
12335 /* Prepare the trees handling the argument that is passed on the stack;
12336 the top level node will store in ON_STACK. */
12339 {
12340 /* if (alignof(type) > 8) (arg = arg + 15) & -16; */
12345 }
12346 else
12348 /* Advance ap.__stack */
12353 /* String up roundup and advance. */
12356 /* String up with arg */
12358 /* Big-endianness related address adjustment. */
12361 {
12365 }
12370 /* Adjustment to OFFSET in the case of BIG_ENDIAN. */
12380 {
12381 /* type ha; // treat as "struct {ftype field[n];}"
12382 ... [computing offs]
12383 for (i = 0; i <nregs; ++i, offs += 16)
12384 ha.field[i] = *((ftype *)(ap.__vr_top + offs));
12385 return ha; */
12389 /* Declare a local variable. */
12393 /* Establish the base type. */
12395 {
12414 {
12418 }
12422 }
12424 /* *(field_ptr_t)&ha = *((field_ptr_t)vr_saved_area */
12432 /* ha.field[i] = *((field_ptr_t)vr_saved_area + i) */
12434 {
12444 }
12448 }
12458 }
12460 /* Implement TARGET_SETUP_INCOMING_VARARGS. */
12462 static void
12466 {
12468 CUMULATIVE_ARGS local_cum;
12472 /* The caller has advanced CUM up to, but not beyond, the last named
12473 argument. Advance a local copy of CUM past the last "real" named
12474 argument, to find out how many registers are left over. */
12478 /* Found out how many registers we need to save.
12479 Honor tree-stdvar analysis results. */
12488 {
12491 }
12494 {
12496 {
12499 /* virtual_incoming_args_rtx should have been 16-byte aligned. */
12507 }
12509 {
12510 /* We can't use move_block_from_reg, because it will use
12511 the wrong mode, storing D regs only. */
12515 /* Set OFF to the offset from virtual_incoming_args_rtx of
12516 the first vector register. The VR save area lies below
12517 the GR one, and is aligned to 16 bytes. */
12524 {
12532 }
12533 }
12534 }
12536 /* We don't save the size into *PRETEND_SIZE because we want to avoid
12537 any complication of having crtl->args.pretend_args_size changed. */
12542 }
12544 static void
12546 {
12549 {
12551 {
12554 }
12555 }
12558 {
12561 }
12562 }
12564 /* Walk down the type tree of TYPE counting consecutive base elements.
12565 If *MODEP is VOIDmode, then set it to the first valid floating point
12566 type. If a non-floating point type is found, or if a floating point
12567 type that doesn't match a non-VOIDmode *MODEP is found, then return -1,
12568 otherwise return the count in the sub-tree. */
12569 static int
12571 {
12572 machine_mode mode;
12573 HOST_WIDE_INT size;
12576 {
12606 /* Use V2SImode and V4SImode as representatives of all 64-bit
12607 and 128-bit vector types. */
12610 {
12619 }
12624 /* Vector modes are considered to be opaque: two vectors are
12625 equivalent for the purposes of being homogeneous aggregates
12626 if they are the same size. */
12633 {
12637 /* Can't handle incomplete types nor sizes that are not
12638 fixed. */
12645 || !index
12656 /* There must be no padding. */
12662 }
12665 {
12668 tree field;
12670 /* Can't handle incomplete types nor sizes that are not
12671 fixed. */
12677 {
12685 }
12687 /* There must be no padding. */
12693 }
12697 {
12698 /* These aren't very interesting except in a degenerate case. */
12701 tree field;
12703 /* Can't handle incomplete types nor sizes that are not
12704 fixed. */
12710 {
12718 }
12720 /* There must be no padding. */
12726 }
12730 }
12733 }
12735 /* Return TRUE if the type, as described by TYPE and MODE, is a short vector
12736 type as described in AAPCS64 \S 4.1.2.
12738 See the comment above aarch64_composite_type_p for the notes on MODE. */
12740 static bool
12742 machine_mode mode)
12743 {
12753 }
12755 /* Return TRUE if the type, as described by TYPE and MODE, is a composite
12756 type as described in AAPCS64 \S 4.3. This includes aggregate, union and
12757 array types. The C99 floating-point complex types are also considered
12758 as composite types, according to AAPCS64 \S 7.1.1. The complex integer
12759 types, which are GCC extensions and out of the scope of AAPCS64, are
12760 treated as composite types here as well.
12762 Note that MODE itself is not sufficient in determining whether a type
12763 is such a composite type or not. This is because
12764 stor-layout.c:compute_record_mode may have already changed the MODE
12765 (BLKmode) of a RECORD_TYPE TYPE to some other mode. For example, a
12766 structure with only one field may have its MODE set to the mode of the
12767 field. Also an integer mode whose size matches the size of the
12768 RECORD_TYPE type may be used to substitute the original mode
12769 (i.e. BLKmode) in certain circumstances. In other words, MODE cannot be
12770 solely relied on. */
12772 static bool
12774 machine_mode mode)
12775 {
12788 }
12790 /* Return TRUE if an argument, whose type is described by TYPE and MODE,
12791 shall be passed or returned in simd/fp register(s) (providing these
12792 parameter passing registers are available).
12794 Upon successful return, *COUNT returns the number of needed registers,
12795 *BASE_MODE returns the mode of the individual register and when IS_HAF
12796 is not NULL, *IS_HA indicates whether or not the argument is a homogeneous
12797 floating-point aggregate or a homogeneous short-vector aggregate. */
12799 static bool
12801 const_tree type,
12805 {
12813 {
12816 }
12818 {
12822 }
12824 {
12828 {
12831 }
12832 else
12834 }
12835 else
12840 }
12842 /* Implement TARGET_STRUCT_VALUE_RTX. */
12844 static rtx
12847 {
12849 }
12851 /* Implements target hook vector_mode_supported_p. */
12852 static bool
12854 {
12857 }
12859 /* Return appropriate SIMD container
12860 for MODE within a vector of WIDTH bits. */
12861 static machine_mode
12863 {
12866 {
12883 }
12887 {
12890 {
12907 }
12908 else
12910 {
12923 }
12924 }
12926 }
12928 /* Return 128-bit container as the preferred SIMD mode for MODE. */
12929 static machine_mode
12931 {
12934 }
12936 /* Return a list of possible vector sizes for the vectorizer
12937 to iterate over. */
12938 static void
12940 {
12945 }
12947 /* Implement TARGET_MANGLE_TYPE. */
12951 {
12952 /* The AArch64 ABI documents say that "__va_list" has to be
12953 managled as if it is in the "std" namespace. */
12957 /* Half-precision float. */
12961 /* Mangle AArch64-specific internal types. TYPE_NAME is non-NULL_TREE for
12962 builtin types. */
12966 /* Use the default mangling. */
12968 }
12970 /* Find the first rtx_insn before insn that will generate an assembly
12971 instruction. */
12975 {
12979 do
12980 {
12982 }
12986 }
12988 static bool
12990 {
12992 /* A number of these may be AArch32 only. */
12997 };
13000 {
13003 }
13006 }
13008 /* Check if there is a register dependency between a load and the insn
13009 for which we hold recog_data. */
13011 static bool
13013 {
13014 rtx load_reg;
13024 {
13030 }
13032 }
13035 /* When working around the Cortex-A53 erratum 835769,
13036 given rtx_insn INSN, return true if it is a 64-bit multiply-accumulate
13037 instruction and has a preceding memory instruction such that a NOP
13038 should be inserted between them. */
13040 bool
13042 {
13045 rtx body;
13058 /* aarch64_prev_real_insn can call recog_memoized on insns other than INSN.
13059 Restore recog state to INSN to avoid state corruption. */
13067 /* If the previous insn is a memory op and there is no dependency between
13068 it and the DImode madd, emit a NOP between them. If body is NULL then we
13069 have a complex memory operation, probably a load/store pair.
13070 Be conservative for now and emit a NOP. */
13077 }
13080 /* Implement FINAL_PRESCAN_INSN. */
13082 void
13084 {
13087 }
13090 /* Return true if BASE_OR_STEP is a valid immediate operand for an SVE INDEX
13091 instruction. */
13093 bool
13095 {
13098 }
13100 /* Return true if X is a valid immediate for the SVE ADD and SUB
13101 instructions. Negate X first if NEGATE_P is true. */
13103 bool
13105 {
13106 rtx elt;
13120 }
13122 /* Return true if X is a valid immediate operand for an SVE logical
13123 instruction such as AND. */
13125 bool
13127 {
13128 rtx elt;
13134 }
13136 /* Return true if X is a valid immediate for the SVE DUP and CPY
13137 instructions. */
13139 bool
13141 {
13142 rtx elt;
13152 }
13154 /* Return true if X is a valid immediate operand for an SVE CMP instruction.
13155 SIGNED_P says whether the operand is signed rather than unsigned. */
13157 bool
13159 {
13160 rtx elt;
13164 && (signed_p
13167 }
13169 /* Return true if X is a valid immediate operand for an SVE FADD or FSUB
13170 instruction. Negate X first if NEGATE_P is true. */
13172 bool
13174 {
13175 rtx elt;
13176 REAL_VALUE_TYPE r;
13192 }
13194 /* Return true if X is a valid immediate operand for an SVE FMUL
13195 instruction. */
13197 bool
13199 {
13200 rtx elt;
13202 /* GCC will never generate a multiply with an immediate of 2, so there is no
13203 point testing for it (even though it is a valid constant). */
13207 }
13209 /* Return true if replicating VAL32 is a valid 2-byte or 4-byte immediate
13210 for the Advanced SIMD operation described by WHICH and INSN. If INFO
13211 is nonnull, use it to describe valid immediates. */
13212 static bool
13217 {
13218 /* Try a 4-byte immediate with LSL. */
13221 {
13226 }
13228 /* Try a 2-byte immediate with LSL. */
13233 {
13238 }
13240 /* Try a 4-byte immediate with MSL, except for cases that MVN
13241 can handle. */
13244 {
13247 {
13252 }
13253 }
13256 }
13258 /* Return true if replicating VAL64 is a valid immediate for the
13259 Advanced SIMD operation described by WHICH. If INFO is nonnull,
13260 use it to describe valid immediates. */
13261 static bool
13265 {
13271 {
13282 /* Try using a replicated byte. */
13286 {
13290 }
13291 }
13293 /* Try using a bit-to-bytemask. */
13295 {
13298 {
13302 }
13304 {
13308 }
13309 }
13311 }
13313 /* Return true if replicating VAL64 gives a valid immediate for an SVE MOV
13314 instruction. If INFO is nonnull, use it to describe valid immediates. */
13316 static bool
13319 {
13323 {
13327 {
13332 }
13333 }
13336 {
13337 /* DUP with no shift. */
13341 }
13343 {
13344 /* DUP with LSL #8. */
13348 }
13350 {
13351 /* DUPM. */
13355 }
13357 }
13359 /* Return true if OP is a valid SIMD immediate for the operation
13360 described by WHICH. If INFO is nonnull, use it to describe valid
13361 immediates. */
13362 bool
13365 {
13379 {
13388 }
13392 else
13395 /* Handle PFALSE and PTRUE. */
13400 scalar_float_mode elt_float_mode;
13403 {
13407 {
13411 }
13412 }
13420 /* Expand the vector constant out into a byte vector, with the least
13421 significant byte of the register first. */
13425 {
13426 /* The vector is provided in gcc endian-neutral fashion.
13427 For aarch64_be Advanced SIMD, it must be laid out in the vector
13428 register in reverse order. */
13440 {
13443 }
13444 }
13446 /* The immediate must repeat every eight bytes. */
13452 /* Get the repeating 8-byte value as an integer. No endian correction
13453 is needed here because bytes is already in lsb-first order. */
13461 else
13463 }
13465 /* Check whether X is a VEC_SERIES-like constant that starts at 0 and
13466 has a step in the range of INDEX. Return the index expression if so,
13467 otherwise return null. */
13468 rtx
13470 {
13477 }
13479 /* Check of immediate shift constants are within range. */
13480 bool
13482 {
13486 else
13488 }
13490 /* Return the bitmask CONST_INT to select the bits required by a zero extract
13491 operation of width WIDTH at bit position POS. */
13493 rtx
13495 {
13499 unsigned HOST_WIDE_INT mask
13502 }
13504 bool
13506 {
13525 }
13527 /* Return a const_int vector of VAL. */
13528 rtx
13530 {
13533 }
13535 /* Check OP is a legal scalar immediate for the MOVI instruction. */
13537 bool
13539 {
13540 machine_mode vmode;
13545 }
13547 /* Construct and return a PARALLEL RTX vector with elements numbering the
13548 lanes of either the high (HIGH == TRUE) or low (HIGH == FALSE) half of
13549 the vector - from the perspective of the architecture. This does not
13550 line up with GCC's perspective on lane numbers, so we end up with
13551 different masks depending on our target endian-ness. The diagram
13552 below may help. We must draw the distinction when building masks
13553 which select one half of the vector. An instruction selecting
13554 architectural low-lanes for a big-endian target, must be described using
13555 a mask selecting GCC high-lanes.
13557 Big-Endian Little-Endian
13559 GCC 0 1 2 3 3 2 1 0
13560 | x | x | x | x | | x | x | x | x |
13561 Architecture 3 2 1 0 3 2 1 0
13563 Low Mask: { 2, 3 } { 0, 1 }
13564 High Mask: { 0, 1 } { 2, 3 }
13566 MODE Is the mode of the vector and NUNITS is the number of units in it. */
13568 rtx
13570 {
13575 rtx t1;
13580 else
13588 }
13590 /* Check OP for validity as a PARALLEL RTX vector with elements
13591 numbering the lanes of either the high (HIGH == TRUE) or low lanes,
13592 from the perspective of the architecture. See the diagram above
13593 aarch64_simd_vect_par_cnst_half for more details. */
13595 bool
13598 {
13612 {
13619 }
13621 }
13623 /* Bounds-check lanes. Ensure OPERAND lies between LOW (inclusive) and
13624 HIGH (exclusive). */
13625 void
13627 const_tree exp)
13628 {
13629 HOST_WIDE_INT lane;
13634 {
13637 else
13639 }
13640 }
13642 /* Peform endian correction on lane number N, which indexes a vector
13643 of mode MODE, and return the result as an SImode rtx. */
13645 rtx
13647 {
13649 }
13651 /* Return TRUE if OP is a valid vector addressing mode. */
13653 bool
13655 {
13658 }
13660 /* Return true if OP is a valid MEM operand for an SVE LD1R instruction. */
13662 bool
13664 {
13666 scalar_mode mode;
13673 }
13675 /* Return true if OP is a valid MEM operand for an SVE LDR instruction.
13676 The conditions for STR are the same. */
13677 bool
13679 {
13686 }
13688 /* Return true if OP is a valid MEM operand for an SVE_STRUCT mode.
13689 We need to be able to access the individual pieces, so the range
13690 is different from LD[234] and ST[234]. */
13691 bool
13693 {
13700 ADDR_QUERY_ANY)
13708 }
13710 /* Emit a register copy from operand to operand, taking care not to
13711 early-clobber source registers in the process.
13713 COUNT is the number of components into which the copy needs to be
13714 decomposed. */
13715 void
13718 {
13728 else
13732 }
13734 /* Compute and return the length of aarch64_simd_reglist<mode>, where <mode> is
13735 one of VSTRUCT modes: OI, CI, or XI. */
13736 int
13738 {
13739 /* This is only used (and only meaningful) for Advanced SIMD, not SVE. */
13741 }
13743 /* Implement target hook TARGET_VECTOR_ALIGNMENT. The AAPCS64 sets the maximum
13744 alignment of a vector to 128 bits. SVE predicates have an alignment of
13745 16 bits. */
13746 static HOST_WIDE_INT
13748 {
13750 /* ??? Checking the mode isn't ideal, but VECTOR_BOOLEAN_TYPE_P can
13751 be set for non-predicate vectors of booleans. Modes are the most
13752 direct way we have of identifying real SVE predicate types. */
13756 }
13758 /* Implement target hook TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT. */
13759 static HOST_WIDE_INT
13761 {
13763 {
13764 /* If the length of the vector is fixed, try to align to that length,
13765 otherwise don't try to align at all. */
13766 HOST_WIDE_INT result;
13770 }
13772 }
13774 /* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE. */
13775 static bool
13777 {
13781 /* For fixed-length vectors, check that the vectorizer will aim for
13782 full-vector alignment. This isn't true for generic GCC vectors
13783 that are wider than the ABI maximum of 128 bits. */
13789 /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned. */
13791 }
13793 /* Return true if the vector misalignment factor is supported by the
13794 target. */
13795 static bool
13799 {
13801 {
13802 /* Return if movmisalign pattern is not supported for this mode. */
13806 /* Misalignment factor is unknown at compile time. */
13809 }
13811 is_packed);
13812 }
13814 /* If VALS is a vector constant that can be loaded into a register
13815 using DUP, generate instructions to do so and return an RTX to
13816 assign to the register. Otherwise return NULL_RTX. */
13817 static rtx
13819 {
13822 rtx x;
13827 /* We can load this constant by using DUP and a constant in a
13828 single ARM register. This will be cheaper than a vector
13829 load. */
13832 }
13835 /* Generate code to load VALS, which is a PARALLEL containing only
13836 constants (for vec_init) or CONST_VECTOR, efficiently into a
13837 register. Returns an RTX to copy into the register, or NULL_RTX
13838 for a PARALLEL that can not be converted into a CONST_VECTOR. */
13839 static rtx
13841 {
13843 rtx const_dup;
13851 {
13852 /* A CONST_VECTOR must contain only CONST_INTs and
13853 CONST_DOUBLEs, but CONSTANT_P allows more (e.g. SYMBOL_REF).
13854 Only store valid constants in a CONST_VECTOR. */
13857 {
13860 n_const++;
13861 }
13864 }
13865 else
13870 /* Load using MOVI/MVNI. */
13873 /* Loaded using DUP. */
13876 /* Load from constant pool. We can not take advantage of single-cycle
13877 LD1 because we need a PC-relative addressing mode. */
13879 else
13880 /* A PARALLEL containing something not valid inside CONST_VECTOR.
13881 We can not construct an initializer. */
13883 }
13885 /* Expand a vector initialisation sequence, such that TARGET is
13886 initialised to contain VALS. */
13888 void
13890 {
13893 /* The number of vector elements. */
13895 /* The number of vector elements which are not constant. */
13898 /* The first element of vals. */
13902 /* Count the number of variable elements to initialise. */
13904 {
13908 else
13912 }
13914 /* No variable elements, hand off to aarch64_simd_make_constant which knows
13915 how best to handle this. */
13917 {
13920 {
13923 }
13924 }
13926 /* Splat a single non-constant element if we can. */
13928 {
13932 }
13937 /* If there are only variable elements, try to optimize
13938 the insertion using dup for the most common element
13939 followed by insertions. */
13941 /* The algorithm will fill matches[*][0] with the earliest matching element,
13942 and matches[X][1] with the count of duplicate elements (if X is the
13943 earliest element which has duplicates). */
13946 {
13949 {
13951 {
13953 {
13957 }
13958 }
13959 }
13964 {
13967 }
13969 /* Create a duplicate of the most common element, unless all elements
13970 are equally useless to us, in which case just immediately set the
13971 vector register using the first element. */
13974 {
13975 /* For vectors of two 64-bit elements, we can do even better. */
13980 {
13983 /* Combine can pick up this case, but handling it directly
13984 here leaves clearer RTL.
13986 This is load_pair_lanes<mode>, and also gives us a clean-up
13987 for store_pair_lanes<mode>. */
13990 && !STRICT_ALIGNMENT
13995 {
13996 rtx t;
13999 else
14003 }
14004 }
14005 /* The subreg-move sequence below will move into lane zero of the
14006 vector register. For big-endian we want that position to hold
14007 the last element of VALS. */
14011 }
14012 else
14013 {
14016 }
14018 /* Insert the rest. */
14020 {
14026 }
14028 }
14030 /* Initialise a vector which is part-variable. We want to first try
14031 to build those lanes which are constant in the most efficient way we
14032 can. */
14034 {
14037 /* Load constant part of vector. We really don't care what goes into the
14038 parts we will overwrite, but we're more likely to be able to load the
14039 constant efficiently if it has fewer, larger, repeating parts
14040 (see aarch64_simd_valid_immediate). */
14042 {
14048 {
14049 /* Look in the copied vector, as more elements are const. */
14052 {
14055 }
14056 }
14058 }
14060 }
14062 /* Insert the variable lanes directly. */
14064 {
14070 }
14071 }
14073 static unsigned HOST_WIDE_INT
14075 {
14079 }
14081 /* Select a format to encode pointers in exception handling data. */
14082 int
14084 {
14087 {
14093 /* text+got+data < 4Gb. 4-byte signed relocs are sufficient
14094 for everything. */
14098 /* No assumptions here. 8-byte relocs required. */
14101 }
14103 }
14105 /* The last .arch and .tune assembly strings that we printed. */
14109 /* Implement ASM_DECLARE_FUNCTION_NAME. Output the ISA features used
14110 by the function fndecl. */
14112 void
14114 tree fndecl)
14115 {
14121 else
14132 /* Only update the assembler .arch string if it is distinct from the last
14133 such string we printed. */
14136 {
14139 }
14141 /* Print the cpu name we're tuning for in the comments, might be
14142 useful to readers of the generated asm. Do it only when it changes
14143 from function to function and verbose assembly is requested. */
14148 {
14152 }
14154 /* Don't forget the type directive for ELF. */
14157 }
14159 /* Implements TARGET_ASM_FILE_START. Output the assembly header. */
14161 static void
14163 {
14180 }
14182 /* Emit load exclusive. */
14184 static void
14187 {
14191 {
14198 }
14201 }
14203 /* Emit store exclusive. */
14205 static void
14208 {
14212 {
14219 }
14222 }
14224 /* Mark the previous jump instruction as unlikely. */
14226 static void
14228 {
14231 }
14233 /* Expand a compare and swap pattern. */
14235 void
14237 {
14242 gen_cas_fn gen;
14244 {
14245 gen_aarch64_compare_and_swapqi,
14246 gen_aarch64_compare_and_swaphi,
14247 gen_aarch64_compare_and_swapsi,
14248 gen_aarch64_compare_and_swapdi
14249 };
14251 {
14252 gen_aarch64_compare_and_swapqi_lse,
14253 gen_aarch64_compare_and_swaphi_lse,
14254 gen_aarch64_compare_and_swapsi_lse,
14255 gen_aarch64_compare_and_swapdi_lse
14256 };
14269 /* Normally the succ memory model must be stronger than fail, but in the
14270 unlikely event of fail being ACQUIRE and succ being RELEASE we need to
14271 promote succ to ACQ_REL so that we don't lose the acquire semantics. */
14278 {
14281 /* For short modes, we're going to perform the comparison in SImode,
14282 so do the zero-extension now. */
14286 /* Fall through. */
14290 /* Force the value into a register if needed. */
14297 }
14300 {
14307 }
14310 else
14321 }
14323 /* Test whether the target supports using a atomic load-operate instruction.
14324 CODE is the operation and AFTER is TRUE if the data in memory after the
14325 operation should be returned and FALSE if the data before the operation
14326 should be returned. Returns FALSE if the operation isn't supported by the
14327 architecture. */
14329 bool
14331 {
14336 {
14346 }
14347 }
14349 /* Emit a barrier, that is appropriate for memory model MODEL, at the end of a
14350 sequence implementing an atomic operation. */
14352 static void
14354 {
14361 {
14363 }
14364 }
14366 /* Emit an atomic compare-and-swap operation. RVAL is the destination register
14367 for the data in memory. EXPECTED is the value expected to be in memory.
14368 DESIRED is the value to store to memory. MEM is the memory location. MODEL
14369 is the memory ordering to use. */
14371 void
14374 rtx model)
14375 {
14377 machine_mode mode;
14382 {
14389 }
14391 /* Move the expected value into the CAS destination register. */
14394 /* Emit the CAS. */
14397 /* Compare the expected value with the value loaded by the CAS, to establish
14398 whether the swap was made. */
14400 }
14402 /* Split a compare and swap pattern. */
14404 void
14406 {
14408 machine_mode mode;
14413 rtx model_rtx;
14425 /* When OLDVAL is zero and we want the strong version we can emit a tighter
14426 loop:
14427 .label1:
14428 LD[A]XR rval, [mem]
14429 CBNZ rval, .label2
14430 ST[L]XR scratch, newval, [mem]
14431 CBNZ scratch, .label1
14432 .label2:
14433 CMP rval, 0. */
14438 {
14441 }
14444 /* The initial load can be relaxed for a __sync operation since a final
14445 barrier will be emitted to stop code hoisting. */
14449 else
14453 {
14458 }
14459 else
14460 {
14466 }
14471 {
14476 }
14477 else
14478 {
14482 }
14485 /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL
14486 to set the condition flags. If this is not used it will be removed by
14487 later passes. */
14489 {
14493 }
14494 /* Emit any final barrier needed for a __sync operation. */
14497 }
14499 /* Emit a BIC instruction. */
14501 static void
14503 {
14508 {
14513 }
14516 }
14518 /* Emit an atomic swap. */
14520 static void
14523 {
14527 {
14534 }
14537 }
14539 /* Operations supported by aarch64_emit_atomic_load_op. */
14541 enum aarch64_atomic_load_op_code
14542 {
14546 AARCH64_LDOP_BIC /* A & ~B */
14547 };
14549 /* Emit an atomic load-operate. */
14551 static void
14555 {
14558 {
14559 gen_aarch64_atomic_loadaddqi,
14560 gen_aarch64_atomic_loadaddhi,
14561 gen_aarch64_atomic_loadaddsi,
14562 gen_aarch64_atomic_loadadddi
14563 };
14565 {
14566 gen_aarch64_atomic_loadeorqi,
14567 gen_aarch64_atomic_loadeorhi,
14568 gen_aarch64_atomic_loadeorsi,
14569 gen_aarch64_atomic_loadeordi
14570 };
14572 {
14573 gen_aarch64_atomic_loadsetqi,
14574 gen_aarch64_atomic_loadsethi,
14575 gen_aarch64_atomic_loadsetsi,
14576 gen_aarch64_atomic_loadsetdi
14577 };
14579 {
14580 gen_aarch64_atomic_loadclrqi,
14581 gen_aarch64_atomic_loadclrhi,
14582 gen_aarch64_atomic_loadclrsi,
14583 gen_aarch64_atomic_loadclrdi
14584 };
14585 aarch64_atomic_load_op_fn gen;
14589 {
14596 }
14599 {
14606 }
14609 }
14611 /* Emit an atomic load+operate. CODE is the operation. OUT_DATA is the
14612 location to store the data read from memory. OUT_RESULT is the location to
14613 store the result of the operation. MEM is the memory location to read and
14614 modify. MODEL_RTX is the memory ordering to use. VALUE is the second
14615 operand for the operation. Either OUT_DATA or OUT_RESULT, but not both, can
14616 be NULL. */
14618 void
14621 {
14625 aarch64_atomic_load_op_code ldop_code;
14626 rtx src;
14627 rtx x;
14635 /* Make sure the value is in a register, putting it into a destination
14636 register if it needs to be manipulated. */
14639 {
14642 }
14643 else
14647 /* Preprocess the data for the operation as necessary. If the operation is
14648 a SET then emit a swap instruction and finish. */
14650 {
14656 /* Negate the value and treat it as a PLUS. */
14657 {
14658 rtx neg_src;
14660 /* Resize the value if necessary. */
14669 }
14670 /* Fall-through. */
14684 {
14685 rtx not_src;
14687 /* Resize the value if necessary. */
14696 }
14701 /* The operation can't be done with atomic instructions. */
14703 }
14707 /* If necessary, calculate the data in memory after the update by redoing the
14708 operation from values in registers. */
14713 {
14717 }
14722 {
14738 }
14743 }
14745 /* Split an atomic operation. */
14747 void
14750 {
14756 rtx x;
14758 /* Split the atomic operation into a sequence. */
14766 else
14770 /* The initial load can be relaxed for a __sync operation since a final
14771 barrier will be emitted to stop code hoisting. */
14775 else
14779 {
14793 {
14796 }
14797 /* Fall through. */
14803 }
14813 /* Emit any final barrier needed for a __sync operation. */
14816 }
14818 static void
14820 {
14821 /* Half-precision float operations. The compiler handles all operations
14822 with NULL libfuncs by converting to SFmode. */
14824 /* Conversions. */
14828 /* Arithmetic. */
14835 /* Comparisons. */
14843 }
14845 /* Target hook for c_mode_for_suffix. */
14846 static machine_mode
14848 {
14853 }
14855 /* We can only represent floating point constants which will fit in
14856 "quarter-precision" values. These values are characterised by
14857 a sign bit, a 4-bit mantissa and a 3-bit exponent. And are given
14858 by:
14860 (-1)^s * (n/16) * 2^r
14862 Where:
14863 's' is the sign bit.
14864 'n' is an integer in the range 16 <= n <= 31.
14865 'r' is an integer in the range -3 <= r <= 4. */
14867 /* Return true iff X can be represented by a quarter-precision
14868 floating point immediate operand X. Note, we cannot represent 0.0. */
14869 bool
14871 {
14872 /* This represents our current view of how many bits
14873 make up the mantissa. */
14883 /* We don't support HFmode constants yet. */
14889 /* We cannot represent infinities, NaNs or +/-zero. We won't
14890 know if we have +zero until we analyse the mantissa, but we
14891 can reject the other invalid values. */
14896 /* Extract exponent. */
14900 /* For the mantissa, we expand into two HOST_WIDE_INTS, apart from the
14901 highest (sign) bit, with a fixed binary point at bit point_pos.
14902 m1 holds the low part of the mantissa, m2 the high part.
14903 WARNING: If we ever have a representation using more than 2 * H_W_I - 1
14904 bits for the mantissa, this can fail (low bits will be lost). */
14908 /* If the low part of the mantissa has bits set we cannot represent
14909 the value. */
14912 /* We have rejected the lower HOST_WIDE_INT, so update our
14913 understanding of how many bits lie in the mantissa and
14914 look only at the high HOST_WIDE_INT. */
14918 /* We can only represent values with a mantissa of the form 1.xxxx. */
14923 /* Having filtered unrepresentable values, we may now remove all
14924 but the highest 5 bits. */
14927 /* We cannot represent the value 0.0, so reject it. This is handled
14928 elsewhere. */
14932 /* Then, as bit 4 is always set, we can mask it off, leaving
14933 the mantissa in the range [0, 15]. */
14937 /* GCC internally does not use IEEE754-like encoding (where normalized
14938 significands are in the range [1, 2). GCC uses [0.5, 1) (see real.c).
14939 Our mantissa values are shifted 4 places to the left relative to
14940 normalized IEEE754 so we must modify the exponent returned by REAL_EXP
14941 by 5 places to correct for GCC's representation. */
14945 }
14947 /* Returns the string with the instruction for AdvSIMD MOVI, MVNI, ORR or BIC
14948 immediate with a CONST_VECTOR of MODE and WIDTH. WHICH selects whether to
14949 output MOVI/MVNI, ORR or BIC immediate. */
14953 {
14963 /* This will return true to show const_vector is legal for use as either
14964 a AdvSIMD MOVI instruction (or, implicitly, MVNI), ORR or BIC immediate.
14965 It will also update INFO to show how the immediate should be generated.
14966 WHICH selects whether to check for MOVI/MVNI, ORR or BIC. */
14974 {
14976 /* For FP zero change it to a CONST_INT 0 and use the integer SIMD
14977 move immediate path. */
14980 else
14981 {
14990 else
14994 }
14995 }
15000 {
15010 else
15014 }
15015 else
15016 {
15017 /* For AARCH64_CHECK_BIC and AARCH64_CHECK_ORR. */
15023 else
15027 }
15029 }
15033 {
15035 /* If a floating point number was passed and we desire to use it in an
15036 integer mode do the conversion to integer. */
15038 {
15043 }
15045 machine_mode vmode;
15046 /* use a 64 bit mode for everything except for DI/DF mode, where we use
15047 a 128 bit vector mode. */
15053 }
15055 /* Return the output string to use for moving immediate CONST_VECTOR
15056 into an SVE register. */
15060 {
15071 {
15076 }
15079 {
15082 else
15083 {
15093 }
15094 }
15099 }
15101 /* Return the asm format for a PTRUE instruction whose destination has
15102 mode MODE. SUFFIX is the element size suffix. */
15106 {
15111 else
15114 }
15116 /* Split operands into moves from op[1] + op[2] into op[0]. */
15118 void
15120 {
15131 {
15132 /* No-op move. Can't split to nothing; emit something. */
15135 }
15137 /* Preserve register attributes for variable tracking. */
15142 /* Special case of reversed high/low parts. */
15145 {
15149 }
15151 {
15152 /* Try to avoid unnecessary moves if part of the result
15153 is in the right place already. */
15158 }
15159 else
15160 {
15165 }
15166 }
15168 /* vec_perm support. */
15170 struct expand_vec_perm_d
15171 {
15173 vec_perm_indices perm;
15174 machine_mode vmode;
15178 };
15180 /* Generate a variable permutation. */
15182 static void
15184 {
15195 {
15197 {
15198 /* Expand the argument to a V16QI mode by duplicating it. */
15202 }
15203 else
15204 {
15206 }
15207 }
15208 else
15209 {
15210 rtx pair;
15213 {
15217 }
15218 else
15219 {
15223 }
15224 }
15225 }
15227 /* Expand a vec_perm with the operands given by TARGET, OP0, OP1 and SEL.
15228 NELT is the number of elements in the vector. */
15230 void
15233 {
15236 rtx mask;
15238 /* The TBL instruction does not use a modulo index, so we must take care
15239 of that ourselves. */
15244 /* For big-endian, we also need to reverse the index within the vector
15245 (but not which vector). */
15247 {
15248 /* If one_vector_p, mask is a vector of (nelt - 1)'s already. */
15253 }
15255 }
15257 /* Generate (set TARGET (unspec [OP0 OP1] CODE)). */
15259 static void
15261 {
15265 }
15267 /* Expand an SVE vec_perm with the given operands. */
15269 void
15271 {
15274 /* Enforced by the pattern condition. */
15277 /* Note: vec_perm indices are supposed to wrap when they go beyond the
15278 size of the two value vectors, i.e. the upper bits of the indices
15279 are effectively ignored. SVE TBL instead produces 0 for any
15280 out-of-range indices, so we need to modulo all the vec_perm indices
15281 to ensure they are all in range. */
15284 /* Check if the sel only references the first values vector. */
15287 {
15290 }
15292 /* Check if the two values vectors are the same. */
15294 {
15300 }
15302 /* Run TBL on for each value vector and combine the results. */
15309 {
15314 }
15321 else
15323 }
15325 /* Recognize patterns suitable for the TRN instructions. */
15326 static bool
15328 {
15329 HOST_WIDE_INT odd;
15337 /* Note that these are little-endian tests.
15338 We correct for big-endian later. */
15345 /* Success! */
15351 /* We don't need a big-endian lane correction for SVE; see the comment
15352 at the head of aarch64-sve.md for details. */
15354 {
15357 }
15363 }
15365 /* Recognize patterns suitable for the UZP instructions. */
15366 static bool
15368 {
15369 HOST_WIDE_INT odd;
15376 /* Note that these are little-endian tests.
15377 We correct for big-endian later. */
15383 /* Success! */
15389 /* We don't need a big-endian lane correction for SVE; see the comment
15390 at the head of aarch64-sve.md for details. */
15392 {
15395 }
15401 }
15403 /* Recognize patterns suitable for the ZIP instructions. */
15404 static bool
15406 {
15415 /* Note that these are little-endian tests.
15416 We correct for big-endian later. */
15424 /* Success! */
15430 /* We don't need a big-endian lane correction for SVE; see the comment
15431 at the head of aarch64-sve.md for details. */
15433 {
15436 }
15442 }
15444 /* Recognize patterns for the EXT insn. */
15446 static bool
15448 {
15449 HOST_WIDE_INT location;
15450 rtx offset;
15452 /* The first element always refers to the first vector.
15453 Check if the extracted indices are increasing by one. */
15459 /* Success! */
15463 /* The case where (location == 0) is a no-op for both big- and little-endian,
15464 and is removed by the mid-end at optimization levels -O1 and higher.
15466 We don't need a big-endian lane correction for SVE; see the comment
15467 at the head of aarch64-sve.md for details. */
15469 {
15470 /* After setup, we want the high elements of the first vector (stored
15471 at the LSB end of the register), and the low elements of the second
15472 vector (stored at the MSB end of the register). So swap. */
15474 /* location != 0 (above), so safe to assume (nelt - location) < nelt.
15475 to_constant () is safe since this is restricted to Advanced SIMD
15476 vectors. */
15478 }
15484 UNSPEC_EXT));
15486 }
15488 /* Recognize patterns for the REV{64,32,16} insns, which reverse elements
15489 within each 64-bit, 32-bit or 16-bit granule. */
15491 static bool
15493 {
15494 HOST_WIDE_INT diff;
15496 machine_mode pred_mode;
15505 {
15508 }
15510 {
15513 }
15515 {
15518 }
15519 else
15527 /* Success! */
15533 {
15536 UNSPEC_MERGE_PTRUE);
15537 }
15540 }
15542 /* Recognize patterns for the REV insn, which reverses elements within
15543 a full vector. */
15545 static bool
15547 {
15556 /* Success! */
15563 }
15565 static bool
15567 {
15569 rtx in0;
15570 HOST_WIDE_INT elt;
15572 rtx lane;
15582 /* Success! */
15586 /* The generic preparation in aarch64_expand_vec_perm_const_1
15587 swaps the operand order and the permute indices if it finds
15588 d->perm[0] to be in the second operand. Thus, we can always
15589 use d->op0 and need not do any extra arithmetic to get the
15590 correct lane number. */
15598 }
15600 static bool
15602 {
15606 /* Make sure that the indices are constant. */
15615 /* Generic code will try constant permutation twice. Once with the
15616 original mode and again with the elements lowered to QImode.
15617 So wait and don't do the selector expansion ourselves. */
15621 /* to_constant is safe since this routine is specific to Advanced SIMD
15622 vectors. */
15625 /* If big-endian and two vectors we end up with a weird mixed-endian
15626 mode on NEON. Reverse the index within each word but not the word
15627 itself. to_constant is safe because we checked is_constant above. */
15637 }
15639 /* Try to implement D using an SVE TBL instruction. */
15641 static bool
15643 {
15646 /* Permuting two variable-length vectors could overflow the
15647 index range. */
15658 }
15660 static bool
15662 {
15663 /* The pattern matching functions above are written to look for a small
15664 number to begin the sequence (0, 1, N/2). If we begin with an index
15665 from the second operand, we can swap the operands. */
15668 {
15671 }
15677 {
15696 }
15698 }
15700 /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
15702 static bool
15705 {
15708 /* Check whether the mask can be applied to a single vector. */
15712 {
15715 }
15717 {
15720 }
15721 else
15741 }
15743 /* Generate a byte permute mask for a register of mode MODE,
15744 which has NUNITS units. */
15746 rtx
15748 {
15749 /* We have to reverse each vector because we dont have
15750 a permuted load that can reverse-load according to ABI rules. */
15751 rtx mask;
15764 }
15766 /* Return true if X is a valid second operand for the SVE instruction
15767 that implements integer comparison OP_CODE. */
15769 static bool
15771 {
15776 {
15791 }
15792 }
15794 /* Use predicated SVE instructions to implement the equivalent of:
15796 (set TARGET OP)
15798 given that PTRUE is an all-true predicate of the appropriate mode. */
15800 static void
15802 {
15805 UNSPEC_MERGE_PTRUE);
15808 }
15810 /* Likewise, but also clobber the condition codes. */
15812 static void
15814 {
15817 UNSPEC_MERGE_PTRUE);
15820 }
15822 /* Return the UNSPEC_COND_* code for comparison CODE. */
15824 static unsigned int
15826 {
15828 {
15843 }
15844 }
15846 /* Emit:
15848 (set TARGET (unspec [PRED OP0 OP1] UNSPEC_COND_<X>))
15850 where <X> is the operation associated with comparison CODE. This form
15851 of instruction is used when (and (CODE OP0 OP1) PRED) would have different
15852 semantics, such as when PRED might not be all-true and when comparing
15853 inactive lanes could have side effects. */
15855 static void
15858 {
15863 }
15865 /* Expand an SVE integer comparison using the SVE equivalent of:
15867 (set TARGET (CODE OP0 OP1)). */
15869 void
15871 {
15881 }
15883 /* Emit the SVE equivalent of:
15885 (set TMP1 (CODE1 OP0 OP1))
15886 (set TMP2 (CODE2 OP0 OP1))
15887 (set TARGET (ior:PRED_MODE TMP1 TMP2))
15889 PTRUE is an all-true predicate with the same mode as TARGET. */
15891 static void
15894 {
15903 }
15905 /* Emit the SVE equivalent of:
15907 (set TMP (CODE OP0 OP1))
15908 (set TARGET (not TMP))
15910 PTRUE is an all-true predicate with the same mode as TARGET. */
15912 static void
15915 {
15921 }
15923 /* Expand an SVE floating-point comparison using the SVE equivalent of:
15925 (set TARGET (CODE OP0 OP1))
15927 If CAN_INVERT_P is true, the caller can also handle inverted results;
15928 return true if the result is in fact inverted. */
15930 bool
15933 {
15939 {
15941 /* UNORDERED has no immediate form. */
15943 /* fall through */
15950 {
15951 /* There is native support for the comparison. */
15955 }
15958 /* This is a trapping operation (LT or GT). */
15964 {
15965 /* This would trap for signaling NaNs. */
15969 }
15970 /* fall through */
15976 {
15977 /* Work out which elements are ordered. */
15982 /* Test the opposite condition for the ordered elements,
15983 then invert the result. */
15986 else
15989 {
15993 }
15998 }
16002 /* ORDERED has no immediate form. */
16008 }
16010 /* There is native support for the inverse comparison. */
16013 {
16017 }
16020 }
16022 /* Expand an SVE vcond pattern with operands OPS. DATA_MODE is the mode
16023 of the data being selected and CMP_MODE is the mode of the values being
16024 compared. */
16026 void
16029 {
16030 machine_mode pred_mode
16035 {
16039 }
16040 else
16045 }
16047 /* Prepare a cond_<optab><mode> operation that has the operands
16048 given by OPERANDS, where:
16050 - operand 0 is the destination
16051 - operand 1 is a predicate
16052 - operands 2 to NOPS - 2 are the operands to an operation that is
16053 performed for active lanes
16054 - operand NOPS - 1 specifies the values to use for inactive lanes.
16056 COMMUTATIVE_P is true if operands 2 and 3 are commutative. In that case,
16057 no pattern is provided for a tie between operands 3 and NOPS - 1. */
16059 void
16062 {
16063 /* We can do the operation directly if the "else" value matches one
16064 of the other inputs. */
16067 {
16071 }
16073 /* If the "else" value is different from the other operands, we have
16074 the choice of doing a SEL on the output or a SEL on an input.
16075 Neither choice is better in all cases, but one advantage of
16076 selecting the input is that it can avoid a move when the output
16077 needs to be distinct from the inputs. E.g. if operand N maps to
16078 register N, selecting the output would give:
16080 MOVPRFX Z0.S, Z2.S
16081 ADD Z0.S, P1/M, Z0.S, Z3.S
16082 SEL Z0.S, P1, Z0.S, Z4.S
16084 whereas selecting the input avoids the MOVPRFX:
16086 SEL Z0.S, P1, Z2.S, Z4.S
16087 ADD Z0.S, P1/M, Z0.S, Z3.S. */
16093 }
16095 /* Implement TARGET_MODES_TIEABLE_P. In principle we should always return
16096 true. However due to issues with register allocation it is preferable
16097 to avoid tieing integer scalar and FP scalar modes. Executing integer
16098 operations in general registers is better than treating them as scalar
16099 vector operations. This reduces latency and avoids redundant int<->FP
16100 moves. So tie modes if they are either the same class, or vector modes
16101 with other vector modes, vector structs or any scalar mode. */
16103 static bool
16105 {
16109 /* We specifically want to allow elements of "structure" modes to
16110 be tieable to the structure. This more general condition allows
16111 other rarer situations too. The reason we don't extend this to
16112 predicate modes is that there are no predicate structure modes
16113 nor any specific instructions for extracting part of a predicate
16114 register. */
16119 /* Also allow any scalar modes with vectors. */
16125 }
16127 /* Return a new RTX holding the result of moving POINTER forward by
16128 AMOUNT bytes. */
16130 static rtx
16132 {
16137 }
16139 /* Return a new RTX holding the result of moving POINTER forward by the
16140 size of the mode it points to. */
16142 static rtx
16144 {
16146 }
16148 /* Copy one MODE sized block from SRC to DST, then progress SRC and DST by
16149 MODE bytes. */
16151 static void
16153 machine_mode mode)
16154 {
16157 /* "Cast" the pointers to the correct mode. */
16160 /* Emit the memcpy. */
16163 /* Move the pointers forward. */
16166 }
16168 /* Expand movmem, as if from a __builtin_memcpy. Return true if
16169 we succeed, otherwise return false. */
16171 bool
16173 {
16177 rtx base;
16180 /* When optimizing for size, give a better estimate of the length of a
16181 memcpy call, but use the default otherwise. */
16184 /* We can't do anything smart if the amount to copy is not constant. */
16190 /* Try to keep the number of instructions low. For cases below 16 bytes we
16191 need to make at most two moves. For cases above 16 bytes it will be one
16192 move for each 16 byte chunk, then at most two additional moves. */
16202 /* Simple cases. Copy 0-3 bytes, as (if applicable) a 2-byte, then a
16203 1-byte chunk. */
16205 {
16207 {
16210 }
16216 }
16218 /* Copy 4-8 bytes. First a 4-byte chunk, then (if applicable) a second
16219 4-byte chunk, partially overlapping with the previously copied chunk. */
16221 {
16225 {
16231 }
16233 }
16235 /* Copy more than 8 bytes. Copy chunks of 16 bytes until we run out of
16236 them, then (if applicable) an 8-byte chunk. */
16238 {
16240 {
16243 }
16244 else
16245 {
16248 }
16249 }
16251 /* Finish the final bytes of the copy. We can always do this in one
16252 instruction. We either copy the exact amount we need, or partially
16253 overlap with the previous chunk we copied and copy 8-bytes. */
16262 else
16263 {
16265 {
16269 }
16270 else
16271 {
16277 }
16278 }
16281 }
16283 /* Split a DImode store of a CONST_INT SRC to MEM DST as two
16284 SImode stores. Handle the case when the constant has identical
16285 bottom and top halves. This is beneficial when the two stores can be
16286 merged into an STP and we avoid synthesising potentially expensive
16287 immediates twice. Return true if such a split is possible. */
16289 bool
16291 {
16300 unsigned int orig_cost
16302 unsigned int lo_cost
16305 /* We want to transform:
16306 MOV x1, 49370
16307 MOVK x1, 0x140, lsl 16
16308 MOVK x1, 0xc0da, lsl 32
16309 MOVK x1, 0x140, lsl 48
16310 STR x1, [x0]
16311 into:
16312 MOV w1, 49370
16313 MOVK w1, 0x140, lsl 16
16314 STP w1, w1, [x0]
16315 So we want to perform this only when we save two instructions
16316 or more. When optimizing for size, however, accept any code size
16317 savings we can. */
16332 /* Don't emit an explicit store pair as this may not be always profitable.
16333 Let the sched-fusion logic decide whether to merge them. */
16338 }
16340 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
16342 static unsigned HOST_WIDE_INT
16344 {
16346 }
16348 static rtx
16351 {
16355 insn_code icode;
16366 {
16394 }
16399 {
16402 }
16411 {
16414 }
16420 }
16422 static rtx
16425 {
16429 insn_code icode;
16441 {
16469 }
16474 {
16477 }
16485 {
16490 }
16501 {
16504 }
16510 }
16512 #undef TARGET_GEN_CCMP_FIRST
16513 #define TARGET_GEN_CCMP_FIRST aarch64_gen_ccmp_first
16515 #undef TARGET_GEN_CCMP_NEXT
16516 #define TARGET_GEN_CCMP_NEXT aarch64_gen_ccmp_next
16518 /* Implement TARGET_SCHED_MACRO_FUSION_P. Return true if target supports
16519 instruction fusion of some sort. */
16521 static bool
16523 {
16525 }
16528 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P. Return true if PREV and CURR
16529 should be kept together during scheduling. */
16531 static bool
16533 {
16534 rtx set_dest;
16537 /* prev and curr are simple SET insns i.e. no flag setting or branching. */
16544 {
16545 /* We are trying to match:
16546 prev (mov) == (set (reg r0) (const_int imm16))
16547 curr (movk) == (set (zero_extract (reg r0)
16548 (const_int 16)
16549 (const_int 16))
16550 (const_int imm16_1)) */
16562 {
16564 }
16565 }
16568 {
16570 /* We're trying to match:
16571 prev (adrp) == (set (reg r1)
16572 (high (symbol_ref ("SYM"))))
16573 curr (add) == (set (reg r0)
16574 (lo_sum (reg r1)
16575 (symbol_ref ("SYM"))))
16576 Note that r0 need not necessarily be the same as r1, especially
16577 during pre-regalloc scheduling. */
16581 {
16589 }
16590 }
16593 {
16595 /* We're trying to match:
16596 prev (movk) == (set (zero_extract (reg r0)
16597 (const_int 16)
16598 (const_int 32))
16599 (const_int imm16_1))
16600 curr (movk) == (set (zero_extract (reg r0)
16601 (const_int 16)
16602 (const_int 48))
16603 (const_int imm16_2)) */
16619 }
16621 {
16622 /* We're trying to match:
16623 prev (adrp) == (set (reg r0)
16624 (high (symbol_ref ("SYM"))))
16625 curr (ldr) == (set (reg r1)
16626 (mem (lo_sum (reg r0)
16627 (symbol_ref ("SYM")))))
16628 or
16629 curr (ldr) == (set (reg r1)
16630 (zero_extend (mem
16631 (lo_sum (reg r0)
16632 (symbol_ref ("SYM")))))) */
16635 {
16648 }
16649 }
16657 {
16661 rtx cc_reg_1;
16666 && prev
16668 {
16669 /* FIXME: this misses some which is considered simple arthematic
16670 instructions for ThunderX. Simple shifts are missed here. */
16676 }
16677 }
16680 && curr_set
16683 {
16684 /* We're trying to match:
16685 prev (alu_insn) == (set (r0) plus ((r0) (r1/imm)))
16686 curr (cbz) == (set (pc) (if_then_else (eq/ne) (r0)
16687 (const_int 0))
16688 (label_ref ("SYM"))
16689 (pc)) */
16696 {
16697 /* Fuse ALU operations followed by conditional branch instruction. */
16699 {
16720 }
16721 }
16722 }
16725 }
16727 /* Return true iff the instruction fusion described by OP is enabled. */
16729 bool
16731 {
16733 }
16735 /* If MEM is in the form of [base+offset], extract the two parts
16736 of address and set to BASE and OFFSET, otherwise return false
16737 after clearing BASE and OFFSET. */
16739 bool
16741 {
16742 rtx addr;
16749 {
16753 }
16757 {
16761 }
16767 }
16769 /* Types for scheduling fusion. */
16770 enum sched_fusion_type
16771 {
16773 SCHED_FUSION_LD_SIGN_EXTEND,
16774 SCHED_FUSION_LD_ZERO_EXTEND,
16775 SCHED_FUSION_LD,
16776 SCHED_FUSION_ST,
16777 SCHED_FUSION_NUM
16778 };
16780 /* If INSN is a load or store of address in the form of [base+offset],
16781 extract the two parts and set to BASE and OFFSET. Return scheduling
16782 fusion type this INSN is. */
16784 static enum sched_fusion_type
16786 {
16804 {
16809 }
16811 {
16816 }
16821 {
16824 }
16825 else
16832 }
16834 /* Implement the TARGET_SCHED_FUSION_PRIORITY hook.
16836 Currently we only support to fuse ldr or str instructions, so FUSION_PRI
16837 and PRI are only calculated for these instructions. For other instruction,
16838 FUSION_PRI and PRI are simply set to MAX_PRI - 1. In the future, other
16839 type instruction fusion can be added by returning different priorities.
16841 It's important that irrelevant instructions get the largest FUSION_PRI. */
16843 static void
16846 {
16856 {
16860 }
16862 /* Set FUSION_PRI according to fusion type and base register. */
16865 /* Calculate PRI. */
16868 /* INSN with smaller offset goes first. */
16872 else
16877 }
16879 /* Implement the TARGET_SCHED_ADJUST_PRIORITY hook.
16880 Adjust priority of sha1h instructions so they are scheduled before
16881 other SHA1 instructions. */
16883 static int
16885 {
16889 {
16894 }
16897 }
16899 /* Given OPERANDS of consecutive load/store, check if we can merge
16900 them into ldp/stp. LOAD is true if they are load instructions.
16901 MODE is the mode of memory operands. */
16903 bool
16905 machine_mode mode)
16906 {
16912 {
16920 }
16921 else
16922 {
16927 }
16929 /* The mems cannot be volatile. */
16933 /* If we have SImode and slow unaligned ldp,
16934 check the alignment to be at least 8 byte. */
16938 && !optimize_size
16942 /* Check if the addresses are in the form of [base+offset]. */
16950 /* Check if the bases are same. */
16954 /* The operands must be of the same size. */
16960 /* We should only be trying this for fixed-sized modes. There is no
16961 SVE LDP/STP instruction. */
16963 /* Check if the offsets are consecutive. */
16967 /* Check if the addresses are clobbered by load. */
16969 {
16973 /* In increasing order, the last load can clobber the address. */
16976 }
16978 /* One of the memory accesses must be a mempair operand.
16979 If it is not the first one, they need to be swapped by the
16980 peephole. */
16987 else
16992 else
16995 /* Check if the registers are of same class. */
17000 }
17002 /* Given OPERANDS of consecutive load/store that can be merged,
17003 swap them if they are not in ascending order. */
17004 void
17006 {
17011 {
17014 }
17015 else
17016 {
17019 }
17028 {
17029 /* Irrespective of whether this is a load or a store,
17030 we do the same swap. */
17033 }
17034 }
17036 /* Taking X and Y to be HOST_WIDE_INT pointers, return the result of a
17037 comparison between the two. */
17038 int
17040 {
17043 }
17045 /* Taking X and Y to be pairs of RTX, one pointing to a MEM rtx and the
17046 other pointing to a REG rtx containing an offset, compare the offsets
17047 of the two pairs.
17049 Return:
17051 1 iff offset (X) > offset (Y)
17052 0 iff offset (X) == offset (Y)
17053 -1 iff offset (X) < offset (Y) */
17054 int
17056 {
17063 else
17068 else
17071 /* Extract the offsets. */
17078 }
17080 /* Given OPERANDS of consecutive load/store, check if we can merge
17081 them into ldp/stp by adjusting the offset. LOAD is true if they
17082 are load instructions. MODE is the mode of memory operands.
17084 Given below consecutive stores:
17086 str w1, [xb, 0x100]
17087 str w1, [xb, 0x104]
17088 str w1, [xb, 0x108]
17089 str w1, [xb, 0x10c]
17091 Though the offsets are out of the range supported by stp, we can
17092 still pair them after adjusting the offset, like:
17094 add scratch, xb, 0x100
17095 stp w1, w1, [scratch]
17096 stp w1, w1, [scratch, 0x8]
17098 The peephole patterns detecting this opportunity should guarantee
17099 the scratch register is avaliable. */
17101 bool
17103 scalar_mode mode)
17104 {
17111 {
17123 /* Do not attempt to merge the loads if the loads clobber each other. */
17128 }
17129 else
17130 {
17139 }
17140 /* Skip if memory operand is by itslef valid for ldp/stp. */
17144 /* The mems cannot be volatile. */
17149 /* Check if the addresses are in the form of [base+offset]. */
17163 /* Check if the bases are same. */
17175 /* Check if the offsets can be put in the right order to do a ldp/stp. */
17182 /* Check that offsets are within range of each other. The ldp/stp
17183 instructions have 7 bit immediate offsets, so use 0x80. */
17187 /* The offsets must be aligned with respect to each other. */
17191 /* Check if the addresses are clobbered by load. */
17198 /* If we have SImode and slow unaligned ldp,
17199 check the alignment to be at least 8 byte. */
17203 && !optimize_size
17209 else
17214 else
17219 else
17224 else
17227 /* Check if the registers are of same class. */
17232 }
17234 /* Given OPERANDS of consecutive load/store, this function pairs them
17235 into LDP/STP after adjusting the offset. It depends on the fact
17236 that the operands can be sorted so the offsets are correct for STP.
17237 MODE is the mode of memory operands. CODE is the rtl operator
17238 which should be applied to all memory operands, it's SIGN_EXTEND,
17239 ZERO_EXTEND or UNKNOWN. */
17241 bool
17244 {
17251 /* We make changes on a copy as we may still bail out. */
17255 /* Sort the operands. */
17259 {
17264 }
17265 else
17266 {
17272 }
17279 /* Adjust offset so it can fit in LDP/STP instruction. */
17287 /* The base offset is optimally half way between the two STP/LDP offsets. */
17290 else
17291 /* However, due to issues with negative LDP/STP offset generation for
17292 larger modes, for DF, DI and vector modes. we must not use negative
17293 addresses smaller than 9 signed unadjusted bits can store. This
17294 provides the most range in this case. */
17297 /* Adjust the base so that it is aligned with the addresses but still
17298 optimal. */
17300 /* Fix the offset, bearing in mind we want to make it bigger not
17301 smaller. */
17304 /* The negative range of LDP/STP is one larger than the positive range. */
17307 /* Check if base offset is too big or too small. We can attempt to resolve
17308 this issue by setting it to the maximum value and seeing if the offsets
17309 still fit. */
17311 {
17313 /* We must still make sure that the base offset is aligned with respect
17314 to the address. But it may may not be made any bigger. */
17316 }
17318 /* Likewise for the case where the base is too small. */
17320 {
17323 }
17325 /* Offset of the first STP/LDP. */
17328 /* Offset of the second STP/LDP. */
17331 /* The offsets must be within the range of the LDP/STP instructions. */
17350 {
17355 }
17357 {
17362 }
17365 {
17374 }
17375 else
17376 {
17385 }
17387 /* Emit adjusting instruction. */
17389 /* Emit ldp/stp instructions. */
17397 }
17399 /* Implement TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE. Assume for now that
17400 it isn't worth branching around empty masked ops (including masked
17401 stores). */
17403 static bool
17405 {
17407 }
17409 /* Return 1 if pseudo register should be created and used to hold
17410 GOT address for PIC code. */
17412 bool
17414 {
17416 }
17418 /* Implement TARGET_UNSPEC_MAY_TRAP_P. */
17420 static int
17422 {
17424 {
17431 }
17434 }
17437 /* If X is a positive CONST_DOUBLE with a value that is a power of 2
17438 return the log2 of that value. Otherwise return -1. */
17440 int
17442 {
17457 }
17459 /* If X is a vector of equal CONST_DOUBLE values and that value is
17460 Y, return the aarch64_fpconst_pow_of_2 of Y. Otherwise return -1. */
17462 int
17464 {
17482 }
17484 /* Implement TARGET_PROMOTED_TYPE to promote 16-bit floating point types
17485 to float.
17487 __fp16 always promotes through this hook.
17488 _Float16 may promote if TARGET_FLT_EVAL_METHOD is 16, but we do that
17489 through the generic excess precision logic rather than here. */
17491 static tree
17493 {
17499 }
17501 /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
17503 static bool
17505 optimization_type opt_type)
17506 {
17508 {
17514 }
17515 }
17517 /* Implement the TARGET_DWARF_POLY_INDETERMINATE_VALUE hook. */
17519 static unsigned int
17522 {
17523 /* Polynomial invariant 1 == (VG / 2) - 1. */
17528 }
17530 /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
17531 if MODE is HFmode, and punt to the generic implementation otherwise. */
17533 static bool
17535 {
17539 }
17541 /* Implement TARGET_SCALAR_MODE_SUPPORTED_P - return TRUE
17542 if MODE is HFmode, and punt to the generic implementation otherwise. */
17544 static bool
17546 {
17550 }
17552 /* Set the value of FLT_EVAL_METHOD.
17553 ISO/IEC TS 18661-3 defines two values that we'd like to make use of:
17555 0: evaluate all operations and constants, whose semantic type has at
17556 most the range and precision of type float, to the range and
17557 precision of float; evaluate all other operations and constants to
17558 the range and precision of the semantic type;
17560 N, where _FloatN is a supported interchange floating type
17561 evaluate all operations and constants, whose semantic type has at
17562 most the range and precision of _FloatN type, to the range and
17563 precision of the _FloatN type; evaluate all other operations and
17564 constants to the range and precision of the semantic type;
17566 If we have the ARMv8.2-A extensions then we support _Float16 in native
17567 precision, so we should set this to 16. Otherwise, we support the type,
17568 but want to evaluate expressions in float precision, so set this to
17569 0. */
17571 static enum flt_eval_method
17573 {
17575 {
17578 /* We can calculate either in 16-bit range and precision or
17579 32-bit range and precision. Make that decision based on whether
17580 we have native support for the ARMv8.2-A 16-bit floating-point
17581 instructions or not. */
17583 ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
17589 }
17591 }
17593 /* Implement TARGET_SCHED_CAN_SPECULATE_INSN. Return true if INSN can be
17594 scheduled for speculative execution. Reject the long-running division
17595 and square-root instructions. */
17597 static bool
17599 {
17601 {
17619 }
17620 }
17622 /* Implement TARGET_COMPUTE_PRESSURE_CLASSES. */
17624 static int
17626 {
17630 /* PR_REGS isn't a useful pressure class because many predicate pseudo
17631 registers need to go in PR_LO_REGS at some point during their
17632 lifetime. Splitting it into two halves has the effect of making
17633 all predicates count against PR_LO_REGS, so that we try whenever
17634 possible to restrict the number of live predicates to 8. This
17635 greatly reduces the amount of spilling in certain loops. */
17639 }
17641 /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
17643 static bool
17646 {
17648 {
17652 /* Don't allow changes between SVE data modes and non-SVE modes.
17653 See the comment at the head of aarch64-sve.md for details. */
17657 /* Don't allow changes in element size: lane 0 of the new vector
17658 would not then be lane 0 of the old vector. See the comment
17659 above aarch64_maybe_expand_sve_subreg_move for a more detailed
17660 description.
17662 In the worst case, this forces a register to be spilled in
17663 one mode and reloaded in the other, which handles the
17664 endianness correctly. */
17667 }
17669 }
17671 /* Implement TARGET_EARLY_REMAT_MODES. */
17673 static void
17675 {
17676 /* SVE values are not normally live across a call, so it should be
17677 worth doing early rematerialization even in VL-specific mode. */
17679 {
17684 }
17685 }
17687 /* Target-specific selftests. */
17689 #if CHECKING_P
17693 /* Selftest for the RTL loader.
17694 Verify that the RTL loader copes with a dump from
17695 print_rtx_function. This is essentially just a test that class
17696 function_reader can handle a real dump, but it also verifies
17697 that lookup_reg_by_dump_name correctly handles hard regs.
17698 The presence of hard reg names in the dump means that the test is
17699 target-specific, hence it is in this file. */
17701 static void
17703 {
17715 /* Verify crtl->return_rtx. */
17719 }
17721 /* Run all target-specific selftests. */
17723 static void
17725 {
17727 }
17733 #undef TARGET_ADDRESS_COST
17734 #define TARGET_ADDRESS_COST aarch64_address_cost
17736 /* This hook will determines whether unnamed bitfields affect the alignment
17737 of the containing structure. The hook returns true if the structure
17738 should inherit the alignment requirements of an unnamed bitfield's
17739 type. */
17740 #undef TARGET_ALIGN_ANON_BITFIELD
17741 #define TARGET_ALIGN_ANON_BITFIELD hook_bool_void_true
17743 #undef TARGET_ASM_ALIGNED_DI_OP
17746 #undef TARGET_ASM_ALIGNED_HI_OP
17749 #undef TARGET_ASM_ALIGNED_SI_OP
17752 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
17753 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK \
17754 hook_bool_const_tree_hwi_hwi_const_tree_true
17756 #undef TARGET_ASM_FILE_START
17757 #define TARGET_ASM_FILE_START aarch64_start_file
17759 #undef TARGET_ASM_OUTPUT_MI_THUNK
17760 #define TARGET_ASM_OUTPUT_MI_THUNK aarch64_output_mi_thunk
17762 #undef TARGET_ASM_SELECT_RTX_SECTION
17763 #define TARGET_ASM_SELECT_RTX_SECTION aarch64_select_rtx_section
17765 #undef TARGET_ASM_TRAMPOLINE_TEMPLATE
17766 #define TARGET_ASM_TRAMPOLINE_TEMPLATE aarch64_asm_trampoline_template
17768 #undef TARGET_BUILD_BUILTIN_VA_LIST
17769 #define TARGET_BUILD_BUILTIN_VA_LIST aarch64_build_builtin_va_list
17771 #undef TARGET_CALLEE_COPIES
17772 #define TARGET_CALLEE_COPIES hook_bool_CUMULATIVE_ARGS_mode_tree_bool_false
17774 #undef TARGET_CAN_ELIMINATE
17775 #define TARGET_CAN_ELIMINATE aarch64_can_eliminate
17777 #undef TARGET_CAN_INLINE_P
17778 #define TARGET_CAN_INLINE_P aarch64_can_inline_p
17780 #undef TARGET_CANNOT_FORCE_CONST_MEM
17781 #define TARGET_CANNOT_FORCE_CONST_MEM aarch64_cannot_force_const_mem
17783 #undef TARGET_CASE_VALUES_THRESHOLD
17784 #define TARGET_CASE_VALUES_THRESHOLD aarch64_case_values_threshold
17786 #undef TARGET_CONDITIONAL_REGISTER_USAGE
17787 #define TARGET_CONDITIONAL_REGISTER_USAGE aarch64_conditional_register_usage
17789 /* Only the least significant bit is used for initialization guard
17790 variables. */
17791 #undef TARGET_CXX_GUARD_MASK_BIT
17792 #define TARGET_CXX_GUARD_MASK_BIT hook_bool_void_true
17794 #undef TARGET_C_MODE_FOR_SUFFIX
17795 #define TARGET_C_MODE_FOR_SUFFIX aarch64_c_mode_for_suffix
17797 #ifdef TARGET_BIG_ENDIAN_DEFAULT
17798 #undef TARGET_DEFAULT_TARGET_FLAGS
17799 #define TARGET_DEFAULT_TARGET_FLAGS (MASK_BIG_END)
17800 #endif
17802 #undef TARGET_CLASS_MAX_NREGS
17803 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
17805 #undef TARGET_BUILTIN_DECL
17806 #define TARGET_BUILTIN_DECL aarch64_builtin_decl
17808 #undef TARGET_BUILTIN_RECIPROCAL
17809 #define TARGET_BUILTIN_RECIPROCAL aarch64_builtin_reciprocal
17811 #undef TARGET_C_EXCESS_PRECISION
17812 #define TARGET_C_EXCESS_PRECISION aarch64_excess_precision
17814 #undef TARGET_EXPAND_BUILTIN
17815 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
17817 #undef TARGET_EXPAND_BUILTIN_VA_START
17818 #define TARGET_EXPAND_BUILTIN_VA_START aarch64_expand_builtin_va_start
17820 #undef TARGET_FOLD_BUILTIN
17821 #define TARGET_FOLD_BUILTIN aarch64_fold_builtin
17823 #undef TARGET_FUNCTION_ARG
17824 #define TARGET_FUNCTION_ARG aarch64_function_arg
17826 #undef TARGET_FUNCTION_ARG_ADVANCE
17827 #define TARGET_FUNCTION_ARG_ADVANCE aarch64_function_arg_advance
17829 #undef TARGET_FUNCTION_ARG_BOUNDARY
17830 #define TARGET_FUNCTION_ARG_BOUNDARY aarch64_function_arg_boundary
17832 #undef TARGET_FUNCTION_ARG_PADDING
17833 #define TARGET_FUNCTION_ARG_PADDING aarch64_function_arg_padding
17835 #undef TARGET_GET_RAW_RESULT_MODE
17836 #define TARGET_GET_RAW_RESULT_MODE aarch64_get_reg_raw_mode
17837 #undef TARGET_GET_RAW_ARG_MODE
17838 #define TARGET_GET_RAW_ARG_MODE aarch64_get_reg_raw_mode
17840 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
17841 #define TARGET_FUNCTION_OK_FOR_SIBCALL aarch64_function_ok_for_sibcall
17843 #undef TARGET_FUNCTION_VALUE
17844 #define TARGET_FUNCTION_VALUE aarch64_function_value
17846 #undef TARGET_FUNCTION_VALUE_REGNO_P
17847 #define TARGET_FUNCTION_VALUE_REGNO_P aarch64_function_value_regno_p
17849 #undef TARGET_GIMPLE_FOLD_BUILTIN
17850 #define TARGET_GIMPLE_FOLD_BUILTIN aarch64_gimple_fold_builtin
17852 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
17853 #define TARGET_GIMPLIFY_VA_ARG_EXPR aarch64_gimplify_va_arg_expr
17855 #undef TARGET_INIT_BUILTINS
17856 #define TARGET_INIT_BUILTINS aarch64_init_builtins
17858 #undef TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS
17859 #define TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS \
17860 aarch64_ira_change_pseudo_allocno_class
17862 #undef TARGET_LEGITIMATE_ADDRESS_P
17863 #define TARGET_LEGITIMATE_ADDRESS_P aarch64_legitimate_address_hook_p
17865 #undef TARGET_LEGITIMATE_CONSTANT_P
17866 #define TARGET_LEGITIMATE_CONSTANT_P aarch64_legitimate_constant_p
17868 #undef TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT
17869 #define TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT \
17870 aarch64_legitimize_address_displacement
17872 #undef TARGET_LIBGCC_CMP_RETURN_MODE
17873 #define TARGET_LIBGCC_CMP_RETURN_MODE aarch64_libgcc_cmp_return_mode
17875 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
17876 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
17877 aarch64_libgcc_floating_mode_supported_p
17879 #undef TARGET_MANGLE_TYPE
17880 #define TARGET_MANGLE_TYPE aarch64_mangle_type
17882 #undef TARGET_MEMORY_MOVE_COST
17883 #define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
17885 #undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
17886 #define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
17888 #undef TARGET_MUST_PASS_IN_STACK
17889 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
17891 /* This target hook should return true if accesses to volatile bitfields
17892 should use the narrowest mode possible. It should return false if these
17893 accesses should use the bitfield container type. */
17894 #undef TARGET_NARROW_VOLATILE_BITFIELD
17895 #define TARGET_NARROW_VOLATILE_BITFIELD hook_bool_void_false
17897 #undef TARGET_OPTION_OVERRIDE
17898 #define TARGET_OPTION_OVERRIDE aarch64_override_options
17900 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
17901 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE \
17902 aarch64_override_options_after_change
17904 #undef TARGET_OPTION_SAVE
17905 #define TARGET_OPTION_SAVE aarch64_option_save
17907 #undef TARGET_OPTION_RESTORE
17908 #define TARGET_OPTION_RESTORE aarch64_option_restore
17910 #undef TARGET_OPTION_PRINT
17911 #define TARGET_OPTION_PRINT aarch64_option_print
17913 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
17914 #define TARGET_OPTION_VALID_ATTRIBUTE_P aarch64_option_valid_attribute_p
17916 #undef TARGET_SET_CURRENT_FUNCTION
17917 #define TARGET_SET_CURRENT_FUNCTION aarch64_set_current_function
17919 #undef TARGET_PASS_BY_REFERENCE
17920 #define TARGET_PASS_BY_REFERENCE aarch64_pass_by_reference
17922 #undef TARGET_PREFERRED_RELOAD_CLASS
17923 #define TARGET_PREFERRED_RELOAD_CLASS aarch64_preferred_reload_class
17925 #undef TARGET_SCHED_REASSOCIATION_WIDTH
17926 #define TARGET_SCHED_REASSOCIATION_WIDTH aarch64_reassociation_width
17928 #undef TARGET_PROMOTED_TYPE
17929 #define TARGET_PROMOTED_TYPE aarch64_promoted_type
17931 #undef TARGET_SECONDARY_RELOAD
17932 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
17934 #undef TARGET_SHIFT_TRUNCATION_MASK
17935 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
17937 #undef TARGET_SETUP_INCOMING_VARARGS
17938 #define TARGET_SETUP_INCOMING_VARARGS aarch64_setup_incoming_varargs
17940 #undef TARGET_STRUCT_VALUE_RTX
17941 #define TARGET_STRUCT_VALUE_RTX aarch64_struct_value_rtx
17943 #undef TARGET_REGISTER_MOVE_COST
17944 #define TARGET_REGISTER_MOVE_COST aarch64_register_move_cost
17946 #undef TARGET_RETURN_IN_MEMORY
17947 #define TARGET_RETURN_IN_MEMORY aarch64_return_in_memory
17949 #undef TARGET_RETURN_IN_MSB
17950 #define TARGET_RETURN_IN_MSB aarch64_return_in_msb
17952 #undef TARGET_RTX_COSTS
17953 #define TARGET_RTX_COSTS aarch64_rtx_costs_wrapper
17955 #undef TARGET_SCALAR_MODE_SUPPORTED_P
17956 #define TARGET_SCALAR_MODE_SUPPORTED_P aarch64_scalar_mode_supported_p
17958 #undef TARGET_SCHED_ISSUE_RATE
17959 #define TARGET_SCHED_ISSUE_RATE aarch64_sched_issue_rate
17961 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
17962 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
17963 aarch64_sched_first_cycle_multipass_dfa_lookahead
17965 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
17966 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD \
17967 aarch64_first_cycle_multipass_dfa_lookahead_guard
17969 #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
17970 #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS \
17971 aarch64_get_separate_components
17973 #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
17974 #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB \
17975 aarch64_components_for_bb
17977 #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
17978 #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS \
17979 aarch64_disqualify_components
17981 #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
17982 #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
17983 aarch64_emit_prologue_components
17985 #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
17986 #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
17987 aarch64_emit_epilogue_components
17989 #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
17990 #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS \
17991 aarch64_set_handled_components
17993 #undef TARGET_TRAMPOLINE_INIT
17994 #define TARGET_TRAMPOLINE_INIT aarch64_trampoline_init
17996 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
17997 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P aarch64_use_blocks_for_constant_p
17999 #undef TARGET_VECTOR_MODE_SUPPORTED_P
18000 #define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
18002 #undef TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT
18003 #define TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT \
18004 aarch64_builtin_support_vector_misalignment
18006 #undef TARGET_ARRAY_MODE
18007 #define TARGET_ARRAY_MODE aarch64_array_mode
18009 #undef TARGET_ARRAY_MODE_SUPPORTED_P
18010 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
18012 #undef TARGET_VECTORIZE_ADD_STMT_COST
18013 #define TARGET_VECTORIZE_ADD_STMT_COST aarch64_add_stmt_cost
18015 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
18016 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
18017 aarch64_builtin_vectorization_cost
18019 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
18020 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
18022 #undef TARGET_VECTORIZE_BUILTINS
18023 #define TARGET_VECTORIZE_BUILTINS
18025 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
18026 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
18027 aarch64_builtin_vectorized_function
18029 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
18030 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
18031 aarch64_autovectorize_vector_sizes
18033 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
18034 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV \
18035 aarch64_atomic_assign_expand_fenv
18037 /* Section anchor support. */
18039 #undef TARGET_MIN_ANCHOR_OFFSET
18040 #define TARGET_MIN_ANCHOR_OFFSET -256
18042 /* Limit the maximum anchor offset to 4k-1, since that's the limit for a
18043 byte offset; we can do much more for larger data types, but have no way
18044 to determine the size of the access. We assume accesses are aligned. */
18045 #undef TARGET_MAX_ANCHOR_OFFSET
18046 #define TARGET_MAX_ANCHOR_OFFSET 4095
18048 #undef TARGET_VECTOR_ALIGNMENT
18049 #define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
18051 #undef TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
18052 #define TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT \
18053 aarch64_vectorize_preferred_vector_alignment
18054 #undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
18055 #define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
18056 aarch64_simd_vector_alignment_reachable
18058 /* vec_perm support. */
18060 #undef TARGET_VECTORIZE_VEC_PERM_CONST
18061 #define TARGET_VECTORIZE_VEC_PERM_CONST \
18062 aarch64_vectorize_vec_perm_const
18064 #undef TARGET_VECTORIZE_GET_MASK_MODE
18065 #define TARGET_VECTORIZE_GET_MASK_MODE aarch64_get_mask_mode
18066 #undef TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE
18067 #define TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE \
18068 aarch64_empty_mask_is_expensive
18070 #undef TARGET_INIT_LIBFUNCS
18071 #define TARGET_INIT_LIBFUNCS aarch64_init_libfuncs
18073 #undef TARGET_FIXED_CONDITION_CODE_REGS
18074 #define TARGET_FIXED_CONDITION_CODE_REGS aarch64_fixed_condition_code_regs
18076 #undef TARGET_FLAGS_REGNUM
18077 #define TARGET_FLAGS_REGNUM CC_REGNUM
18079 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
18080 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
18082 #undef TARGET_ASAN_SHADOW_OFFSET
18083 #define TARGET_ASAN_SHADOW_OFFSET aarch64_asan_shadow_offset
18085 #undef TARGET_LEGITIMIZE_ADDRESS
18086 #define TARGET_LEGITIMIZE_ADDRESS aarch64_legitimize_address
18088 #undef TARGET_SCHED_CAN_SPECULATE_INSN
18089 #define TARGET_SCHED_CAN_SPECULATE_INSN aarch64_sched_can_speculate_insn
18091 #undef TARGET_CAN_USE_DOLOOP_P
18092 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
18094 #undef TARGET_SCHED_ADJUST_PRIORITY
18095 #define TARGET_SCHED_ADJUST_PRIORITY aarch64_sched_adjust_priority
18097 #undef TARGET_SCHED_MACRO_FUSION_P
18098 #define TARGET_SCHED_MACRO_FUSION_P aarch64_macro_fusion_p
18100 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
18101 #define TARGET_SCHED_MACRO_FUSION_PAIR_P aarch_macro_fusion_pair_p
18103 #undef TARGET_SCHED_FUSION_PRIORITY
18104 #define TARGET_SCHED_FUSION_PRIORITY aarch64_sched_fusion_priority
18106 #undef TARGET_UNSPEC_MAY_TRAP_P
18107 #define TARGET_UNSPEC_MAY_TRAP_P aarch64_unspec_may_trap_p
18109 #undef TARGET_USE_PSEUDO_PIC_REG
18110 #define TARGET_USE_PSEUDO_PIC_REG aarch64_use_pseudo_pic_reg
18112 #undef TARGET_PRINT_OPERAND
18113 #define TARGET_PRINT_OPERAND aarch64_print_operand
18115 #undef TARGET_PRINT_OPERAND_ADDRESS
18116 #define TARGET_PRINT_OPERAND_ADDRESS aarch64_print_operand_address
18118 #undef TARGET_OPTAB_SUPPORTED_P
18119 #define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p
18121 #undef TARGET_OMIT_STRUCT_RETURN_REG
18122 #define TARGET_OMIT_STRUCT_RETURN_REG true
18124 #undef TARGET_DWARF_POLY_INDETERMINATE_VALUE
18125 #define TARGET_DWARF_POLY_INDETERMINATE_VALUE \
18126 aarch64_dwarf_poly_indeterminate_value
18128 /* The architecture reserves bits 0 and 1 so use bit 2 for descriptors. */
18129 #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
18130 #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 4
18132 #undef TARGET_HARD_REGNO_NREGS
18133 #define TARGET_HARD_REGNO_NREGS aarch64_hard_regno_nregs
18134 #undef TARGET_HARD_REGNO_MODE_OK
18135 #define TARGET_HARD_REGNO_MODE_OK aarch64_hard_regno_mode_ok
18137 #undef TARGET_MODES_TIEABLE_P
18138 #define TARGET_MODES_TIEABLE_P aarch64_modes_tieable_p
18140 #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
18141 #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
18142 aarch64_hard_regno_call_part_clobbered
18144 #undef TARGET_CONSTANT_ALIGNMENT
18145 #define TARGET_CONSTANT_ALIGNMENT aarch64_constant_alignment
18147 #undef TARGET_COMPUTE_PRESSURE_CLASSES
18148 #define TARGET_COMPUTE_PRESSURE_CLASSES aarch64_compute_pressure_classes
18150 #undef TARGET_CAN_CHANGE_MODE_CLASS
18151 #define TARGET_CAN_CHANGE_MODE_CLASS aarch64_can_change_mode_class
18153 #undef TARGET_SELECT_EARLY_REMAT_MODES
18154 #define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
18156 #if CHECKING_P
18157 #undef TARGET_RUN_TARGET_SELFTESTS
18158 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests