| blob | eed37dbd91f4c75f741bb7b9276dc5193f02d256 |
1 /* Machine description for AArch64 architecture.
2 Copyright (C) 2009-2015 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
100 /* Defined for convenience. */
101 #define POINTER_BYTES (POINTER_SIZE / BITS_PER_UNIT)
103 /* Classifies an address.
105 ADDRESS_REG_IMM
106 A simple base register plus immediate offset.
108 ADDRESS_REG_WB
109 A base register indexed by immediate offset with writeback.
111 ADDRESS_REG_REG
112 A base register indexed by (optionally scaled) register.
114 ADDRESS_REG_UXTW
115 A base register indexed by (optionally scaled) zero-extended register.
117 ADDRESS_REG_SXTW
118 A base register indexed by (optionally scaled) sign-extended register.
120 ADDRESS_LO_SUM
121 A LO_SUM rtx with a base register and "LO12" symbol relocation.
123 ADDRESS_SYMBOLIC:
124 A constant symbolic address, in pc-relative literal pool. */
127 ADDRESS_REG_IMM,
128 ADDRESS_REG_WB,
129 ADDRESS_REG_REG,
130 ADDRESS_REG_UXTW,
131 ADDRESS_REG_SXTW,
132 ADDRESS_LO_SUM,
133 ADDRESS_SYMBOLIC
134 };
138 rtx base;
139 rtx offset;
142 };
144 struct simd_immediate_info
145 {
146 rtx value;
151 };
153 /* The current code model. */
156 #ifdef HAVE_AS_TLS
157 #undef TARGET_HAVE_TLS
158 #define TARGET_HAVE_TLS 1
159 #endif
163 const_tree,
175 /* Major revision number of the ARM Architecture implemented by the target. */
178 /* The processor for which instructions should be scheduled. */
181 /* The current tuning set. */
184 /* Mask to specify which instructions we are allowed to generate. */
187 /* Mask to specify which instruction scheduling options should be used. */
190 /* Tuning parameters. */
193 {
194 {
199 },
205 };
208 {
209 {
214 },
220 };
223 {
224 {
229 },
235 };
238 {
240 /* Avoid the use of slow int<->fp moves for spilling by setting
241 their cost higher than memmov_cost. */
245 };
248 {
250 /* Avoid the use of slow int<->fp moves for spilling by setting
251 their cost higher than memmov_cost. */
255 };
258 {
260 /* Avoid the use of slow int<->fp moves for spilling by setting
261 their cost higher than memmov_cost. */
265 };
268 {
273 };
276 {
278 /* Avoid the use of slow int<->fp moves for spilling by setting
279 their cost higher than memmov_cost. */
283 };
285 /* Generic costs for vector insn classes. */
287 {
300 };
302 /* Generic costs for vector insn classes. */
304 {
317 };
319 /* Generic costs for vector insn classes. */
321 {
334 };
336 #define AARCH64_FUSE_NOTHING (0)
337 #define AARCH64_FUSE_MOV_MOVK (1 << 0)
338 #define AARCH64_FUSE_ADRP_ADD (1 << 1)
339 #define AARCH64_FUSE_MOVK_MOVK (1 << 2)
340 #define AARCH64_FUSE_ADRP_LDR (1 << 3)
341 #define AARCH64_FUSE_CMP_BRANCH (1 << 4)
344 {
358 };
361 {
376 };
379 {
394 };
397 {
411 };
414 {
428 };
430 /* A processor implementing AArch64. */
431 struct processor
432 {
439 };
441 /* Processor cores implementing AArch64. */
443 {
444 #define AARCH64_CORE(NAME, IDENT, SCHED, ARCH, FLAGS, COSTS, IMP, PART) \
445 {NAME, SCHED, #ARCH, ARCH, FLAGS, &COSTS##_tunings},
447 #undef AARCH64_CORE
450 };
452 /* Architectures implementing AArch64. */
454 {
455 #define AARCH64_ARCH(NAME, CORE, ARCH, FLAGS) \
456 {NAME, CORE, #ARCH, ARCH, FLAGS, NULL},
458 #undef AARCH64_ARCH
460 };
462 /* Target specification. These are populated as commandline arguments
463 are processed, or NULL if not specified. */
468 #define AARCH64_CPU_DEFAULT_FLAGS ((selected_cpu) ? selected_cpu->flags : 0)
470 /* An ISA extension in the co-processor and main instruction set space. */
471 struct aarch64_option_extension
472 {
476 };
478 /* ISA extensions in AArch64. */
480 {
481 #define AARCH64_OPT_EXTENSION(NAME, FLAGS_ON, FLAGS_OFF, FEATURE_STRING) \
482 {NAME, FLAGS_ON, FLAGS_OFF},
484 #undef AARCH64_OPT_EXTENSION
486 };
488 /* Used to track the size of an address when generating a pre/post
489 increment address. */
492 /* A table of valid AArch64 "bitmask immediate" values for
493 logical instructions. */
495 #define AARCH64_NUM_BITMASKS 5334
499 {
503 }
504 aarch64_cc;
506 #define AARCH64_INVERSE_CONDITION_CODE(X) ((aarch64_cc) (((int) X) ^ 1))
508 /* The condition codes of the processor, and the inverse function. */
510 {
513 };
515 static unsigned int
517 {
519 }
521 static int
524 {
532 }
534 /* Provide a mapping from gcc register numbers to dwarf register numbers. */
535 unsigned
537 {
545 /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
546 equivalent DWARF register. */
548 }
550 /* Return TRUE if MODE is any of the large INT modes. */
551 static bool
553 {
555 }
557 /* Return TRUE if MODE is any of the vector modes. */
558 static bool
560 {
563 }
565 /* Implement target hook TARGET_ARRAY_MODE_SUPPORTED_P. */
566 static bool
569 {
576 }
578 /* Implement HARD_REGNO_NREGS. */
580 int
582 {
584 {
590 }
592 }
594 /* Implement HARD_REGNO_MODE_OK. */
596 int
598 {
603 /* The purpose of comparing with ptr_mode is to support the
604 global register variable associated with the stack pointer
605 register via the syntax of asm ("wsp") in ILP32. */
615 {
617 return
619 else
621 }
624 }
626 /* Implement HARD_REGNO_CALLER_SAVE_MODE. */
627 machine_mode
629 machine_mode mode)
630 {
631 /* Handle modes that fit within single registers. */
633 {
636 else
638 }
639 /* Fall back to generic for multi-reg and very large modes. */
640 else
642 }
644 /* Return true if calls to DECL should be treated as
645 long-calls (ie called via a register). */
646 static bool
648 {
650 }
652 /* Return true if calls to symbol-ref SYM should be treated as
653 long-calls (ie called via a register). */
654 bool
656 {
658 }
660 /* Return true if the offsets to a zero/sign-extract operation
661 represent an expression that matches an extend operation. The
662 operands represent the paramters from
664 (extract:MODE (mult (reg) (MULT_IMM)) (EXTRACT_IMM) (const_int 0)). */
665 bool
667 rtx extract_imm)
668 {
685 }
687 /* Emit an insn that's a simple single-set. Both the operands must be
688 known to be valid. */
691 {
693 }
695 /* X and Y are two things to compare using CODE. Emit the compare insn and
696 return the rtx for register 0 in the proper mode. */
697 rtx
699 {
705 }
707 /* Build the SYMBOL_REF for __tls_get_addr. */
711 rtx
713 {
717 }
719 /* Return the TLS model to use for ADDR. */
721 static enum tls_model
723 {
728 {
732 }
737 }
739 /* We'll allow lo_sum's in addresses in our legitimate addresses
740 so that combine would take care of combining addresses where
741 necessary, but for generation purposes, we'll generate the address
742 as :
743 RTL Absolute
744 tmp = hi (symbol_ref); adrp x1, foo
745 dest = lo_sum (tmp, symbol_ref); add dest, x1, :lo_12:foo
746 nop
748 PIC TLS
749 adrp x1, :got:foo adrp tmp, :tlsgd:foo
750 ldr x1, [:got_lo12:foo] add dest, tmp, :tlsgd_lo12:foo
751 bl __tls_get_addr
752 nop
754 Load TLS symbol, depending on TLS mechanism and TLS access model.
756 Global Dynamic - Traditional TLS:
757 adrp tmp, :tlsgd:imm
758 add dest, tmp, #:tlsgd_lo12:imm
759 bl __tls_get_addr
761 Global Dynamic - TLS Descriptors:
762 adrp dest, :tlsdesc:imm
763 ldr tmp, [dest, #:tlsdesc_lo12:imm]
764 add dest, dest, #:tlsdesc_lo12:imm
765 blr tmp
766 mrs tp, tpidr_el0
767 add dest, dest, tp
769 Initial Exec:
770 mrs tp, tpidr_el0
771 adrp tmp, :gottprel:imm
772 ldr dest, [tmp, #:gottprel_lo12:imm]
773 add dest, dest, tp
775 Local Exec:
776 mrs tp, tpidr_el0
777 add t0, tp, #:tprel_hi12:imm, lsl #12
778 add t0, t0, #:tprel_lo12_nc:imm
779 */
781 static void
784 {
786 {
788 {
789 /* In ILP32, the mode of dest can be either SImode or DImode. */
801 }
808 {
809 /* In ILP32, the mode of dest can be either SImode or DImode,
810 while the got entry is always of SImode size. The mode of
811 dest depends on how dest is used: if dest is assigned to a
812 pointer (e.g. in the memory), it has SImode; it may have
813 DImode if dest is dereferenced to access the memeory.
814 This is why we have to handle three different ldr_got_small
815 patterns here (two patterns for ILP32). */
824 {
827 else
829 }
830 else
831 {
834 }
837 }
840 {
852 }
855 {
858 rtx tp;
862 /* In ILP32, the got entry is always of SImode size. Unlike
863 small GOT, the dest is fixed at reg 0. */
866 else
876 }
879 {
880 /* In ILP32, the mode of dest can be either SImode or DImode,
881 while the got entry is always of SImode size. The mode of
882 dest depends on how dest is used: if dest is assigned to a
883 pointer (e.g. in the memory), it has SImode; it may have
884 DImode if dest is dereferenced to access the memeory.
885 This is why we have to handle three different tlsie_small
886 patterns here (two patterns for ILP32). */
892 {
895 else
896 {
899 }
900 }
901 else
902 {
905 }
910 }
913 {
922 }
930 }
931 }
933 /* Emit a move from SRC to DEST. Assume that the move expanders can
934 handle all moves if !can_create_pseudo_p (). The distinction is
935 important because, unlike emit_move_insn, the move expanders know
936 how to force Pmode objects into the constant pool even when the
937 constant pool address is not itself legitimate. */
938 static rtx
940 {
944 }
946 /* Split a 128-bit move operation into two 64-bit move operations,
947 taking care to handle partial overlap of register to register
948 copies. Special cases are needed when moving between GP regs and
949 FP regs. SRC can be a register, constant or memory; DST a register
950 or memory. If either operand is memory it must not have any side
951 effects. */
952 void
954 {
965 {
969 /* Handle FP <-> GP regs. */
971 {
976 {
979 }
980 else
981 {
984 }
986 }
988 {
993 {
996 }
997 else
998 {
1001 }
1003 }
1004 }
1011 /* At most one pairing may overlap. */
1013 {
1016 }
1017 else
1018 {
1021 }
1022 }
1024 bool
1026 {
1029 }
1031 /* Split a complex SIMD combine. */
1033 void
1035 {
1042 {
1046 {
1067 }
1071 }
1072 }
1074 /* Split a complex SIMD move. */
1076 void
1078 {
1085 {
1091 {
1112 }
1116 }
1117 }
1119 static rtx
1121 {
1124 else
1125 {
1128 }
1129 }
1132 static rtx
1134 {
1136 {
1137 rtx high;
1138 /* Load the full offset into a register. This
1139 might be improvable in the future. */
1145 }
1147 }
1149 static int
1151 machine_mode mode)
1152 {
1158 rtx subtarget;
1163 {
1166 num_insns++;
1168 }
1171 {
1172 /* We know we can't do this in 1 insn, and we must be able to do it
1173 in two; so don't mess around looking for sequences that don't buy
1174 us anything. */
1176 {
1181 }
1184 }
1186 /* Remaining cases are all for DImode. */
1197 {
1199 one_match++;
1200 else
1201 {
1205 zero_match++;
1206 }
1207 }
1210 {
1211 /* Set one of the quarters and then insert back into result. */
1214 {
1219 }
1222 }
1229 {
1233 {
1235 {
1241 }
1244 }
1246 {
1248 {
1254 }
1257 }
1259 {
1261 {
1267 }
1270 }
1272 {
1274 {
1280 }
1283 }
1284 }
1286 /* See if we can do it by arithmetically combining two
1287 immediates. */
1289 {
1295 {
1297 {
1303 }
1306 }
1309 {
1311 {
1313 {
1318 }
1321 }
1322 }
1323 }
1325 /* See if we can do it by logically combining two immediates. */
1327 {
1329 {
1334 {
1336 {
1342 }
1345 }
1346 }
1348 {
1353 {
1355 {
1361 }
1364 }
1365 }
1366 }
1369 {
1370 /* Set either first three quarters or all but the third. */
1375 num_insns ++;
1377 /* Now insert other two quarters. */
1380 {
1382 {
1386 num_insns ++;
1387 }
1388 }
1390 }
1392 simple_sequence:
1396 {
1398 {
1400 {
1404 num_insns ++;
1406 }
1407 else
1408 {
1412 num_insns ++;
1413 }
1414 }
1415 }
1418 }
1421 void
1423 {
1428 /* Check on what type of symbol it is. */
1432 {
1436 /* If we have (const (plus symbol offset)), separate out the offset
1437 before we start classifying the symbol. */
1442 {
1446 {
1452 }
1466 {
1472 }
1473 /* FALLTHRU */
1483 }
1484 }
1487 {
1490 else
1491 {
1495 }
1498 }
1501 }
1503 static bool
1505 tree exp ATTRIBUTE_UNUSED)
1506 {
1507 /* Currently, always true. */
1509 }
1511 /* Implement TARGET_PASS_BY_REFERENCE. */
1513 static bool
1515 machine_mode mode,
1516 const_tree type,
1518 {
1519 HOST_WIDE_INT size;
1520 machine_mode dummymode;
1523 /* GET_MODE_SIZE (BLKmode) is useless since it is 0. */
1527 /* Aggregates are passed by reference based on their size. */
1529 {
1531 }
1533 /* Variable sized arguments are always returned by reference. */
1537 /* Can this be a candidate to be passed in fp/simd register(s)? */
1540 NULL))
1543 /* Arguments which are variable sized or larger than 2 registers are
1544 passed by reference unless they are a homogenous floating point
1545 aggregate. */
1547 }
1549 /* Return TRUE if VALTYPE is padded to its least significant bits. */
1550 static bool
1552 {
1553 machine_mode dummy_mode;
1556 /* Never happens in little-endian mode. */
1560 /* Only composite types smaller than or equal to 16 bytes can
1561 be potentially returned in registers. */
1567 /* But not a composite that is an HFA (Homogeneous Floating-point Aggregate)
1568 or an HVA (Homogeneous Short-Vector Aggregate); such a special composite
1569 is always passed/returned in the least significant bits of fp/simd
1570 register(s). */
1576 }
1578 /* Implement TARGET_FUNCTION_VALUE.
1579 Define how to find the value returned by a function. */
1581 static rtx
1584 {
1585 machine_mode mode;
1588 machine_mode ag_mode;
1595 {
1599 {
1602 }
1603 }
1607 {
1609 {
1612 }
1613 else
1614 {
1616 rtx par;
1620 {
1625 }
1627 }
1628 }
1629 else
1631 }
1633 /* Implements TARGET_FUNCTION_VALUE_REGNO_P.
1634 Return true if REGNO is the number of a hard register in which the values
1635 of called function may come back. */
1637 static bool
1639 {
1640 /* Maximum of 16 bytes can be returned in the general registers. Examples
1641 of 16-byte return values are: 128-bit integers and 16-byte small
1642 structures (excluding homogeneous floating-point aggregates). */
1646 /* Up to four fp/simd registers can return a function value, e.g. a
1647 homogeneous floating-point aggregate having four members. */
1652 }
1654 /* Implement TARGET_RETURN_IN_MEMORY.
1656 If the type T of the result of a function is such that
1657 void func (T arg)
1658 would require that arg be passed as a value in a register (or set of
1659 registers) according to the parameter passing rules, then the result
1660 is returned in the same registers as would be used for such an
1661 argument. */
1663 static bool
1665 {
1666 HOST_WIDE_INT size;
1667 machine_mode ag_mode;
1673 /* Simple scalar types always returned in registers. */
1677 type,
1680 NULL))
1683 /* Types larger than 2 registers returned in memory. */
1686 }
1688 static bool
1691 {
1694 type,
1696 nregs,
1697 NULL);
1698 }
1700 /* Given MODE and TYPE of a function argument, return the alignment in
1701 bits. The idea is to suppress any stronger alignment requested by
1702 the user and opt for the natural alignment (specified in AAPCS64 \S 4.1).
1703 This is a helper function for local use only. */
1705 static unsigned int
1707 {
1711 {
1713 {
1716 else
1718 }
1719 else
1721 }
1722 else
1726 }
1728 /* Layout a function argument according to the AAPCS64 rules. The rule
1729 numbers refer to the rule numbers in the AAPCS64. */
1731 static void
1733 const_tree type,
1735 {
1739 HOST_WIDE_INT size;
1741 /* We need to do this once per argument. */
1747 /* Size in bytes, rounded to the nearest multiple of 8 bytes. */
1748 size
1750 UNITS_PER_WORD);
1754 mode,
1755 type,
1758 /* allocate_ncrn may be false-positive, but allocate_nvrn is quite reliable.
1759 The following code thus handles passing by SIMD/FP registers first. */
1763 /* C1 - C5 for floating point, homogenous floating point aggregates (HFA)
1764 and homogenous short-vector aggregates (HVA). */
1766 {
1768 {
1771 {
1774 }
1775 else
1776 {
1777 rtx par;
1781 {
1784 tmp = gen_rtx_EXPR_LIST
1788 }
1790 }
1792 }
1793 else
1794 {
1795 /* C.3 NSRN is set to 8. */
1798 }
1799 }
1804 /* C6 - C9. though the sign and zero extension semantics are
1805 handled elsewhere. This is the case where the argument fits
1806 entirely general registers. */
1808 {
1813 /* C.8 if the argument has an alignment of 16 then the NGRN is
1814 rounded up to the next even number. */
1816 {
1819 }
1820 /* NREGS can be 0 when e.g. an empty structure is to be passed.
1821 A reg is still generated for it, but the caller should be smart
1822 enough not to use it. */
1824 {
1826 }
1827 else
1828 {
1829 rtx par;
1834 {
1839 }
1841 }
1845 }
1847 /* C.11 */
1850 /* The argument is passed on stack; record the needed number of words for
1851 this argument and align the total size if necessary. */
1852 on_stack:
1858 }
1860 /* Implement TARGET_FUNCTION_ARG. */
1862 static rtx
1865 {
1874 }
1876 void
1878 const_tree fntype ATTRIBUTE_UNUSED,
1879 rtx libname ATTRIBUTE_UNUSED,
1880 const_tree fndecl ATTRIBUTE_UNUSED,
1882 {
1894 }
1896 static void
1898 machine_mode mode,
1899 const_tree type,
1901 {
1904 {
1914 }
1915 }
1917 bool
1919 {
1922 }
1924 /* Implement FUNCTION_ARG_BOUNDARY. Every parameter gets at least
1925 PARM_BOUNDARY bits of alignment, but will be given anything up
1926 to STACK_BOUNDARY bits if the type requires it. This makes sure
1927 that both before and after the layout of each argument, the Next
1928 Stacked Argument Address (NSAA) will have a minimum alignment of
1929 8 bytes. */
1931 static unsigned int
1933 {
1941 }
1943 /* For use by FUNCTION_ARG_PADDING (MODE, TYPE).
1945 Return true if an argument passed on the stack should be padded upwards,
1946 i.e. if the least-significant byte of the stack slot has useful data.
1948 Small aggregate types are placed in the lowest memory address.
1950 The related parameter passing rules are B.4, C.3, C.5 and C.14. */
1952 bool
1954 {
1955 /* On little-endian targets, the least significant byte of every stack
1956 argument is passed at the lowest byte address of the stack slot. */
1960 /* Otherwise, integral, floating-point and pointer types are padded downward:
1961 the least significant byte of a stack argument is passed at the highest
1962 byte address of the stack slot. */
1969 /* Everything else padded upward, i.e. data in first byte of stack slot. */
1971 }
1973 /* Similarly, for use by BLOCK_REG_PADDING (MODE, TYPE, FIRST).
1975 It specifies padding for the last (may also be the only)
1976 element of a block move between registers and memory. If
1977 assuming the block is in the memory, padding upward means that
1978 the last element is padded after its highest significant byte,
1979 while in downward padding, the last element is padded at the
1980 its least significant byte side.
1982 Small aggregates and small complex types are always padded
1983 upwards.
1985 We don't need to worry about homogeneous floating-point or
1986 short-vector aggregates; their move is not affected by the
1987 padding direction determined here. Regardless of endianness,
1988 each element of such an aggregate is put in the least
1989 significant bits of a fp/simd register.
1991 Return !BYTES_BIG_ENDIAN if the least significant byte of the
1992 register has useful data, and return the opposite if the most
1993 significant byte does. */
1995 bool
1998 {
2000 /* Small composite types are always padded upward. */
2002 {
2007 }
2009 /* Otherwise, use the default padding. */
2011 }
2013 static machine_mode
2015 {
2017 }
2019 static bool
2021 {
2022 /* In aarch64_override_options_after_change
2023 flag_omit_leaf_frame_pointer turns off the frame pointer by
2024 default. Turn it back on now if we've not got a leaf
2025 function. */
2031 }
2033 /* Mark the registers that need to be saved by the callee and calculate
2034 the size of the callee-saved registers area and frame record (both FP
2035 and LR may be omitted). */
2036 static void
2038 {
2045 #define SLOT_NOT_REQUIRED (-2)
2046 #define SLOT_REQUIRED (-1)
2051 /* First mark all the registers that really need to be saved... */
2058 /* ... that includes the eh data registers (if needed)... */
2064 /* ... and any callee saved register that dataflow says is live. */
2077 {
2078 /* FP and LR are placed in the linkage record. */
2085 }
2087 /* Now assign stack slots for them. */
2090 {
2097 }
2101 {
2109 }
2129 }
2131 static bool
2133 {
2135 }
2137 static unsigned
2139 {
2141 regno ++;
2143 }
2145 static void
2147 HOST_WIDE_INT adjustment)
2148 {
2159 }
2161 static rtx
2163 HOST_WIDE_INT adjustment)
2164 {
2166 {
2177 }
2178 }
2180 static void
2183 {
2193 }
2195 static rtx
2197 HOST_WIDE_INT adjustment)
2198 {
2200 {
2209 }
2210 }
2212 static rtx
2214 rtx reg2)
2215 {
2217 {
2226 }
2227 }
2229 static rtx
2231 rtx mem2)
2232 {
2234 {
2243 }
2244 }
2247 static void
2250 {
2260 {
2262 HOST_WIDE_INT offset;
2272 offset));
2280 {
2282 rtx mem2;
2286 offset));
2288 reg2));
2290 /* The first part of a frame-related parallel insn is
2291 always assumed to be relevant to the frame
2292 calculations; subsequent parts, are only
2293 frame-related if explicitly marked. */
2296 }
2297 else
2301 }
2302 }
2304 static void
2308 {
2314 HOST_WIDE_INT offset;
2319 {
2336 {
2338 rtx mem2;
2346 }
2347 else
2350 }
2351 }
2353 /* AArch64 stack frames generated by this compiler look like:
2355 +-------------------------------+
2356 | |
2357 | incoming stack arguments |
2358 | |
2359 +-------------------------------+
2360 | | <-- incoming stack pointer (aligned)
2361 | callee-allocated save area |
2362 | for register varargs |
2363 | |
2364 +-------------------------------+
2365 | local variables | <-- frame_pointer_rtx
2366 | |
2367 +-------------------------------+
2368 | padding0 | \
2369 +-------------------------------+ |
2370 | callee-saved registers | | frame.saved_regs_size
2371 +-------------------------------+ |
2372 | LR' | |
2373 +-------------------------------+ |
2374 | FP' | / <- hard_frame_pointer_rtx (aligned)
2375 +-------------------------------+
2376 | dynamic allocation |
2377 +-------------------------------+
2378 | padding |
2379 +-------------------------------+
2380 | outgoing stack arguments | <-- arg_pointer
2381 | |
2382 +-------------------------------+
2383 | | <-- stack_pointer_rtx (aligned)
2385 Dynamic stack allocations via alloca() decrease stack_pointer_rtx
2386 but leave frame_pointer_rtx and hard_frame_pointer_rtx
2387 unchanged. */
2389 /* Generate the prologue instructions for entry into a function.
2390 Establish the stack frame by decreasing the stack pointer with a
2391 properly calculated size and, if necessary, create a frame record
2392 filled with the values of LR and previous frame pointer. The
2393 current FP is also set up if it is in use. */
2395 void
2397 {
2398 /* sub sp, sp, #<frame_size>
2399 stp {fp, lr}, [sp, #<frame_size> - 16]
2400 add fp, sp, #<frame_size> - hardfp_offset
2401 stp {cs_reg}, [fp, #-16] etc.
2403 sub sp, sp, <final_adjustment_if_any>
2404 */
2407 HOST_WIDE_INT hard_fp_offset;
2419 /* Store pairs and load pairs have a range only -512 to 504. */
2421 {
2422 /* When the frame has a large size, an initial decrease is done on
2423 the stack pointer to jump over the callee-allocated save area for
2424 register varargs, the local variable area and/or the callee-saved
2425 register area. This will allow the pre-index write-back
2426 store pair instructions to be used for setting up the stack frame
2427 efficiently. */
2436 {
2446 }
2448 {
2453 {
2457 }
2459 {
2463 }
2464 }
2465 }
2466 else
2470 {
2474 {
2478 {
2485 }
2486 else
2489 /* Set up frame pointer to point to the location of the
2490 previous frame pointer on the stack. */
2492 stack_pointer_rtx,
2496 }
2497 else
2498 {
2506 {
2510 }
2511 else
2512 {
2519 else
2521 }
2522 }
2525 skip_wb);
2527 skip_wb);
2528 }
2530 /* when offset >= 512,
2531 sub sp, sp, #<outgoing_args_size> */
2533 {
2535 {
2540 }
2541 }
2542 }
2544 /* Return TRUE if we can use a simple_return insn.
2546 This function checks whether the callee saved stack is empty, which
2547 means no restore actions are need. The pro_and_epilogue will use
2548 this to check whether shrink-wrapping opt is feasible. */
2550 bool
2552 {
2562 }
2564 /* Generate the epilogue instructions for returning from a function. */
2565 void
2567 {
2569 HOST_WIDE_INT fp_offset;
2570 HOST_WIDE_INT hard_fp_offset;
2572 /* We need to add memory barrier to prevent read from deallocated stack. */
2582 /* Store pairs and load pairs have a range only -512 to 504. */
2584 {
2592 {
2597 }
2598 }
2599 else
2602 /* If there were outgoing arguments or we've done dynamic stack
2603 allocation, then restore the stack pointer from the frame
2604 pointer. This is at most one insn and more efficient than using
2605 GCC's internal mechanism. */
2608 {
2613 hard_frame_pointer_rtx,
2616 }
2619 {
2642 {
2648 {
2653 }
2654 else
2655 {
2662 }
2663 }
2664 else
2665 {
2668 }
2670 /* Reset the CFA to be SP + FRAME_SIZE. */
2677 }
2680 {
2685 {
2689 }
2690 else
2691 {
2696 {
2701 }
2704 }
2706 /* Reset the CFA to be SP + 0. */
2709 }
2711 /* Stack adjustment for exception handler. */
2713 {
2714 /* We need to unwind the stack by the offset computed by
2715 EH_RETURN_STACKADJ_RTX. We have already reset the CFA
2716 to be SP; letting the CFA move during this adjustment
2717 is just as correct as retaining the CFA from the body
2718 of the function. Therefore, do nothing special. */
2720 }
2725 }
2727 /* Return the place to copy the exception unwinding return address to.
2728 This will probably be a stack slot, but could (in theory be the
2729 return register). */
2730 rtx
2732 {
2733 HOST_WIDE_INT fp_offset;
2743 /* DSE and CSELIB do not detect an alias between sp+k1 and fp+k2. This can
2744 result in a store to save LR introduced by builtin_eh_return () being
2745 incorrectly deleted because the alias is not detected.
2746 So in the calculation of the address to copy the exception unwinding
2747 return address to, we note 2 cases.
2748 If FP is needed and the fp_offset is 0, it means that SP = FP and hence
2749 we return a SP-relative location since all the addresses are SP-relative
2750 in this case. This prevents the store from being optimized away.
2751 If the fp_offset is not 0, then the addresses will be FP-relative and
2752 therefore we return a FP-relative location. */
2755 {
2759 else
2762 }
2764 /* If FP is not needed, we calculate the location of LR, which would be
2765 at the top of the saved registers block. */
2769 stack_pointer_rtx,
2770 fp_offset
2773 }
2775 /* Possibly output code to build up a constant in a register. For
2776 the benefit of the costs infrastructure, returns the number of
2777 instructions which would be emitted. GENERATE inhibits or
2778 enables code generation. */
2780 static int
2782 {
2786 {
2790 }
2791 else
2792 {
2797 HOST_WIDE_INT valm;
2798 HOST_WIDE_INT tval;
2801 {
2811 }
2813 /* zcount contains the number of additional MOVK instructions
2814 required if the constant is built up with an initial MOVZ instruction,
2815 while ncount is the number of MOVK instructions required if starting
2816 with a MOVN instruction. Choose the sequence that yields the fewest
2817 number of instructions, preferring MOVZ instructions when they are both
2818 the same. */
2820 {
2825 insns++;
2826 }
2827 else
2828 {
2833 insns++;
2834 }
2839 {
2841 {
2846 insns++;
2847 }
2849 }
2850 }
2852 }
2854 static void
2856 {
2865 {
2868 }
2870 {
2872 {
2878 else
2881 }
2883 {
2887 }
2888 }
2889 }
2891 /* Output code to add DELTA to the first argument, and then jump
2892 to FUNCTION. Used for C++ multiple inheritance. */
2893 static void
2895 HOST_WIDE_INT delta,
2896 HOST_WIDE_INT vcall_offset,
2897 tree function)
2898 {
2899 /* The this pointer is always in x0. Note that this differs from
2900 Arm where the this pointer maybe bumped to r1 if r0 is required
2901 to return a pointer to an aggregate. On AArch64 a result value
2902 pointer will be in x8. */
2912 else
2913 {
2922 {
2926 else
2928 }
2932 else
2939 else
2940 {
2943 }
2947 else
2953 }
2955 /* Generate a tail call to the target function. */
2957 {
2960 }
2972 /* Stop pretending to be a post-reload pass. */
2974 }
2976 static bool
2978 {
2983 {
2987 /* Don't recurse into UNSPEC_TLS looking for TLS symbols; these are
2988 TLS offsets, not real symbol references. */
2991 }
2993 }
2996 static int
2998 {
3007 }
3010 static void
3012 {
3018 {
3022 else
3025 {
3027 {
3028 /* set s consecutive bits to 1 (s < 64) */
3030 /* rotate right by r */
3033 /* replicate the constant depending on SIMD size */
3044 }
3047 }
3048 }
3049 }
3053 aarch64_bitmasks_cmp);
3054 }
3057 /* Return true if val can be encoded as a 12-bit unsigned immediate with
3058 a left shift of 0 or 12 bits. */
3059 bool
3061 {
3064 );
3065 }
3068 /* Return true if val is an immediate that can be loaded into a
3069 register by a MOVZ instruction. */
3070 static bool
3072 {
3074 {
3078 }
3079 else
3080 {
3081 /* Ignore sign extension. */
3083 }
3086 }
3089 /* Return true if val is a valid bitmask immediate. */
3090 bool
3092 {
3094 {
3095 /* Replicate bit pattern. */
3098 }
3101 }
3104 /* Return true if val is an immediate that can be loaded into a
3105 register in a single instruction. */
3106 bool
3108 {
3112 }
3114 static bool
3116 {
3124 {
3128 else
3129 /* Avoid generating a 64-bit relocation in ILP32; leave
3130 to aarch64_expand_mov_immediate to handle it properly. */
3132 }
3135 }
3137 /* Return true if register REGNO is a valid index register.
3138 STRICT_P is true if REG_OK_STRICT is in effect. */
3140 bool
3142 {
3144 {
3152 }
3154 }
3156 /* Return true if register REGNO is a valid base register for mode MODE.
3157 STRICT_P is true if REG_OK_STRICT is in effect. */
3159 bool
3161 {
3163 {
3171 }
3173 /* The fake registers will be eliminated to either the stack or
3174 hard frame pointer, both of which are usually valid base registers.
3175 Reload deals with the cases where the eliminated form isn't valid. */
3180 }
3182 /* Return true if X is a valid base register for mode MODE.
3183 STRICT_P is true if REG_OK_STRICT is in effect. */
3185 static bool
3187 {
3192 }
3194 /* Return true if address offset is a valid index. If it is, fill in INFO
3195 appropriately. STRICT_P is true if REG_OK_STRICT is in effect. */
3197 static bool
3200 {
3202 rtx index;
3205 /* (reg:P) */
3208 {
3212 }
3213 /* (sign_extend:DI (reg:SI)) */
3218 {
3223 }
3224 /* (mult:DI (sign_extend:DI (reg:SI)) (const_int scale)) */
3231 {
3236 }
3237 /* (ashift:DI (sign_extend:DI (reg:SI)) (const_int shift)) */
3244 {
3249 }
3250 /* (sign_extract:DI (mult:DI (reg:DI) (const_int scale)) 32+shift 0) */
3257 {
3265 }
3266 /* (and:DI (mult:DI (reg:DI) (const_int scale))
3267 (const_int 0xffffffff<<shift)) */
3274 {
3280 }
3281 /* (sign_extract:DI (ashift:DI (reg:DI) (const_int shift)) 32+shift 0) */
3288 {
3296 }
3297 /* (and:DI (ashift:DI (reg:DI) (const_int shift))
3298 (const_int 0xffffffff<<shift)) */
3305 {
3311 }
3312 /* (mult:P (reg:P) (const_int scale)) */
3317 {
3321 }
3322 /* (ashift:P (reg:P) (const_int shift)) */
3327 {
3331 }
3332 else
3343 {
3348 }
3351 }
3353 bool
3355 {
3359 }
3363 HOST_WIDE_INT offset)
3364 {
3366 }
3370 {
3374 }
3376 /* Return true if X is a valid address for machine mode MODE. If it is,
3377 fill in INFO appropriately. STRICT_P is true if REG_OK_STRICT is in
3378 effect. OUTER_CODE is PARALLEL for a load/store pair. */
3380 static bool
3384 {
3388 /* On BE, we use load/store pair for all large int mode load/stores. */
3390 || (BYTES_BIG_ENDIAN
3394 !load_store_pair_p
3398 /* On LE, for AdvSIMD, don't support anything other than POST_INC or
3399 REG addressing. */
3405 {
3423 {
3429 }
3434 {
3441 /* TImode and TFmode values are allowed in both pairs of X
3442 registers and individual Q registers. The available
3443 address modes are:
3444 X,X: 7-bit signed scaled offset
3445 Q: 9-bit signed offset
3446 We conservatively require an offset representable in either mode.
3447 */
3452 /* A 7bit offset check because OImode will emit a ldp/stp
3453 instruction (only big endian will get here).
3454 For ldp/stp instructions, the offset is scaled for the size of a
3455 single element of the pair. */
3459 /* Three 9/12 bit offsets checks because CImode will emit three
3460 ldr/str instructions (only big endian will get here). */
3467 /* Two 7bit offsets checks because XImode will emit two ldp/stp
3468 instructions (only big endian will get here). */
3477 else
3480 }
3483 {
3484 /* Look for base + (scaled/extended) index register. */
3487 {
3490 }
3493 {
3496 }
3497 }
3518 {
3519 HOST_WIDE_INT offset;
3523 /* TImode and TFmode values are allowed in both pairs of X
3524 registers and individual Q registers. The available
3525 address modes are:
3526 X,X: 7-bit signed scaled offset
3527 Q: 9-bit signed offset
3528 We conservatively require an offset representable in either mode.
3529 */
3537 else
3539 }
3545 /* load literal: pc-relative constant pool entry. Only supported
3546 for SI mode or larger. */
3550 {
3557 }
3566 {
3572 {
3573 /* The symbol and offset must be aligned to the access size. */
3580 {
3584 }
3590 else
3599 }
3600 }
3605 }
3606 }
3608 bool
3610 {
3611 rtx offset;
3615 }
3617 /* Classify the base of symbolic expression X, given that X appears in
3618 context CONTEXT. */
3620 enum aarch64_symbol_type
3623 {
3624 rtx offset;
3628 }
3631 /* Return TRUE if X is a legitimate address for accessing memory in
3632 mode MODE. */
3633 static bool
3635 {
3639 }
3641 /* Return TRUE if X is a legitimate address for accessing memory in
3642 mode MODE. OUTER_CODE will be PARALLEL if this is a load/store
3643 pair operation. */
3644 bool
3647 {
3651 }
3653 /* Return TRUE if rtx X is immediate constant 0.0 */
3654 bool
3656 {
3657 REAL_VALUE_TYPE r;
3666 }
3668 /* Return the fixed registers used for condition codes. */
3670 static bool
3672 {
3676 }
3678 /* Emit call insn with PAT and do aarch64-specific handling. */
3680 void
3682 {
3688 }
3690 machine_mode
3692 {
3693 /* All floating point compares return CCFP if it is an equality
3694 comparison, and CCFPE otherwise. */
3696 {
3698 {
3719 }
3720 }
3729 /* A compare with a shifted operand. Because of canonicalization,
3730 the comparison will have to be swapped when we emit the assembly
3731 code. */
3739 /* Similarly for a negated operand, but we can only do this for
3740 equalities. */
3747 /* A compare of a mode narrower than SI mode against zero can be done
3748 by extending the value in the comparison. */
3751 /* Only use sign-extension if we really need it. */
3755 /* For everything else, return CCmode. */
3757 }
3759 static int
3762 int
3764 {
3771 }
3773 static int
3775 {
3778 {
3782 {
3796 }
3851 {
3863 }
3870 {
3882 }
3887 {
3893 }
3898 {
3902 }
3908 }
3917 }
3919 bool
3921 HOST_WIDE_INT minval,
3922 HOST_WIDE_INT maxval)
3923 {
3924 HOST_WIDE_INT firstval;
3941 }
3943 bool
3945 {
3947 }
3949 static unsigned
3951 {
3955 {
3956 count++;
3958 }
3961 }
3963 /* N Z C V. */
3964 #define AARCH64_CC_V 1
3965 #define AARCH64_CC_C (1 << 1)
3966 #define AARCH64_CC_Z (1 << 2)
3967 #define AARCH64_CC_N (1 << 3)
3969 /* N Z C V flags for ccmp. The first code is for AND op and the other
3970 is for IOR op. Indexed by AARCH64_COND_CODE. */
3972 {
3989 };
3991 int
3993 {
3995 {
4028 }
4029 }
4032 void
4034 {
4036 {
4037 /* An integer or symbol address without a preceding # sign. */
4040 {
4052 {
4055 }
4056 /* Fall through. */
4060 }
4064 /* Print the sign/zero-extend size as a character 8->b, 16->h, 32->w. */
4065 {
4070 {
4073 }
4076 {
4089 }
4090 }
4094 {
4097 /* Print N such that 2^N == X. */
4099 {
4102 }
4105 }
4109 /* Print the number of non-zero bits in X (a const_int). */
4111 {
4114 }
4120 /* Print the higher numbered register of a pair (TImode) of regs. */
4122 {
4125 }
4131 {
4133 /* Print a condition (eq, ne, etc). */
4135 /* CONST_TRUE_RTX means always -- that's the default. */
4140 {
4143 }
4148 }
4152 {
4154 /* Print the inverse of a condition (eq <-> ne, etc). */
4156 /* CONST_TRUE_RTX means never -- that's the default. */
4158 {
4161 }
4164 {
4167 }
4172 }
4180 /* Print a scalar FP/SIMD register name. */
4182 {
4183 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
4185 }
4193 /* Print the first FP/SIMD register name in a list. */
4195 {
4196 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
4198 }
4203 /* Print a scalar FP/SIMD register name + 1. */
4205 {
4206 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
4208 }
4213 /* Print bottom 16 bits of integer constant in hex. */
4215 {
4218 }
4224 /* Print a general register name or the zero register (32-bit or
4225 64-bit). */
4228 {
4231 }
4234 {
4237 }
4240 {
4243 }
4245 /* Fall through */
4248 /* Print a normal operand, if it's a general register, then we
4249 assume DImode. */
4251 {
4254 }
4257 {
4278 {
4281 HOST_WIDE_INT_MIN,
4282 HOST_WIDE_INT_MAX));
4284 }
4286 {
4288 }
4289 else
4294 /* CONST_DOUBLE can represent a double-width integer.
4295 In this case, the mode of x is VOIDmode. */
4299 {
4302 }
4304 {
4305 #define buf_size 20
4307 REAL_VALUE_TYPE r;
4314 #undef buf_size
4315 }
4321 }
4329 {
4356 }
4362 {
4389 }
4396 {
4402 }
4407 {
4409 /* Print nzcv. */
4412 {
4415 }
4420 }
4424 {
4426 /* Print nzcv. */
4429 {
4432 }
4437 }
4443 }
4444 }
4446 void
4448 {
4454 {
4458 else
4467 else
4476 else
4485 else
4492 {
4519 }
4530 }
4533 }
4535 bool
4537 {
4544 /* UNSPEC_TLS entries for a symbol include a LABEL_REF for the
4545 referencing instruction, but they are constant offsets, not
4546 symbols. */
4552 {
4554 {
4560 }
4563 }
4566 }
4568 /* Implement REGNO_REG_CLASS. */
4570 enum reg_class
4572 {
4587 }
4589 static rtx
4591 {
4592 /* Try to split X+CONST into Y=X+(CONST & ~mask), Y+(CONST&mask),
4593 where mask is selected by alignment and size of the offset.
4594 We try to pick as large a range for the offset as possible to
4595 maximize the chance of a CSE. However, for aligned addresses
4596 we limit the range to 4k so that structures with different sized
4597 elements are likely to use the same base. */
4600 {
4602 HOST_WIDE_INT base_offset;
4604 /* Does it look like we'll need a load/store-pair operation? */
4609 /* For offsets aren't a multiple of the access size, the limit is
4610 -256...255. */
4613 else
4622 NULL_RTX);
4625 }
4628 }
4630 /* Try a machine-dependent way of reloading an illegitimate address
4631 operand. If we find one, push the reload and return the new rtx. */
4633 rtx
4635 machine_mode mode,
4638 {
4641 /* Do not allow mem (plus (reg, const)) if vector struct mode. */
4646 {
4653 }
4655 /* We must recognize output that we have already generated ourselves. */
4661 {
4666 }
4668 /* We wish to handle large displacements off a base register by splitting
4669 the addend across an add and the mem insn. This can cut the number of
4670 extra insns needed from 3 to 1. It is only useful for load/store of a
4671 single register with 12 bit offset field. */
4679 {
4683 HOST_WIDE_INT offs;
4684 rtx cst;
4687 /* In ILP32, xmode can be either DImode or SImode. */
4690 /* Reload non-zero BLKmode offsets. This is because we cannot ascertain
4691 BLKmode alignment. */
4697 /* Align misaligned offset by adjusting high part to compensate. */
4699 {
4701 {
4702 /* Align down. */
4705 }
4706 else
4707 {
4708 /* Align up. */
4713 }
4714 }
4716 /* Check for overflow. */
4724 /* Reload high part into base reg, leaving the low part
4725 in the mem instruction.
4726 Note that replacing this gen_rtx_PLUS with plus_constant is
4727 wrong in this case because we rely on the
4728 (plus (plus reg c1) c2) structure being preserved so that
4729 XEXP (*p, 0) in push_reload below uses the correct term. */
4738 }
4741 }
4744 static reg_class_t
4746 reg_class_t rclass,
4747 machine_mode mode,
4749 {
4750 /* Without the TARGET_SIMD instructions we cannot move a Q register
4751 to a Q register directly. We need a scratch. */
4755 {
4761 }
4763 /* A TFmode or TImode memory access should be handled via an FP_REGS
4764 because AArch64 has richer addressing modes for LDR/STR instructions
4765 than LDP/STP instructions. */
4774 }
4776 static bool
4778 {
4779 /* If we need a frame pointer, we must eliminate FRAME_POINTER_REGNUM into
4780 HARD_FRAME_POINTER_REGNUM and not into STACK_POINTER_REGNUM. */
4783 {
4795 }
4796 else
4797 {
4798 /* If we decided that we didn't need a leaf frame pointer but then used
4799 LR in the function, then we'll want a frame pointer after all, so
4800 prevent this elimination to ensure a frame pointer is used. */
4802 && flag_omit_leaf_frame_pointer
4805 }
4808 }
4810 HOST_WIDE_INT
4812 {
4816 {
4823 }
4826 {
4830 }
4833 }
4835 /* Implement RETURN_ADDR_RTX. We do not support moving back to a
4836 previous frame. */
4838 rtx
4840 {
4844 }
4847 static void
4849 {
4851 {
4854 }
4855 else
4856 {
4859 }
4864 }
4866 static void
4868 {
4872 /* Don't need to copy the trailing D-words, we fill those in below. */
4884 /* XXX We should really define a "clear_cache" pattern and use
4885 gen_clear_cache(). */
4890 ptr_mode);
4891 }
4893 static unsigned char
4895 {
4897 {
4904 return
4915 }
4917 }
4919 static reg_class_t
4921 {
4926 {
4932 }
4934 /* If it's an integer immediate that MOVI can't handle, then
4935 FP_REGS is not an option, so we return NO_REGS instead. */
4940 /* Register eliminiation can result in a request for
4941 SP+constant->FP_REGS. We cannot support such operations which
4942 use SP as source and an FP_REG as destination, so reject out
4943 right now. */
4945 {
4948 /* Look through a possible SUBREG introduced by ILP32. */
4954 POINTER_REGS));
4956 }
4959 }
4961 void
4963 {
4965 }
4967 static void
4969 {
4972 else
4973 {
4981 }
4982 }
4984 static void
4986 {
4989 else
4990 {
4998 }
4999 }
5003 {
5009 {
5010 {
5013 },
5014 {
5017 },
5018 {
5021 },
5022 /* We assume that DImode is only generated when not optimizing and
5023 that we don't really need 64-bit address offsets. That would
5024 imply an object file with 8GB of code in a single function! */
5025 {
5028 }
5029 };
5037 /* Need to implement table size reduction, by chaning the code below. */
5047 }
5050 /* Return size in bits of an arithmetic operand which is shifted/scaled and
5051 masked such that it is suitable for a UXTB, UXTH, or UXTW extend
5052 operator. */
5054 int
5056 {
5058 {
5061 {
5065 }
5066 }
5068 }
5070 static bool
5072 const_rtx x ATTRIBUTE_UNUSED)
5073 {
5074 /* We can't use blocks for constants when we're using a per-function
5075 constant pool. */
5077 }
5081 rtx x ATTRIBUTE_UNUSED,
5083 {
5084 /* Force all constant pool entries into the current function section. */
5086 }
5089 /* Costs. */
5091 /* Helper function for rtx cost calculation. Strip a shift expression
5092 from X. Returns the inner operand if successful, or the original
5093 expression on failure. */
5094 static rtx
5096 {
5099 /* We accept both ROTATERT and ROTATE: since the RHS must be a constant
5100 we can convert both to ROR during final output. */
5115 }
5117 /* Helper function for rtx cost calculation. Strip an extend
5118 expression from X. Returns the inner operand if successful, or the
5119 original expression on failure. We deal with a number of possible
5120 canonicalization variations here. */
5121 static rtx
5123 {
5126 /* Zero and sign extraction of a widened value. */
5134 /* It can also be represented (for zero-extend) as an AND with an
5135 immediate. */
5144 /* Now handle extended register, as this may also have an optional
5145 left shift by 1..4. */
5159 }
5161 /* Return true iff CODE is a shift supported in combination
5162 with arithmetic instructions. */
5164 static bool
5166 {
5168 }
5170 /* Helper function for rtx cost calculation. Calculate the cost of
5171 a MULT or ASHIFT, which may be part of a compound PLUS/MINUS rtx.
5172 Return the calculated cost of the expression, recursing manually in to
5173 operands where needed. */
5175 static int
5177 {
5193 /* Integer multiply/fma. */
5195 {
5196 /* The multiply will be canonicalized as a shift, cost it as such. */
5200 {
5204 {
5206 {
5208 /* ARITH + shift-by-register. */
5211 /* ARITH + extended register. We don't have a cost field
5212 for ARITH+EXTEND+SHIFT, so use extend_arith here. */
5214 else
5215 /* ARITH + shift-by-immediate. */
5217 }
5218 else
5219 /* LSL (immediate). */
5222 }
5223 /* Strip extends as we will have costed them in the case above. */
5230 }
5232 /* MNEG or [US]MNEGL. Extract the NEG operand and indicate that it's a
5233 compound and let the below cases handle it. After all, MNEG is a
5234 special-case alias of MSUB. */
5236 {
5239 }
5241 /* Integer multiplies or FMAs have zero/sign extending variants. */
5246 {
5251 {
5253 /* SMADDL/UMADDL/UMSUBL/SMSUBL. */
5255 else
5256 /* MUL/SMULL/UMULL. */
5258 }
5261 }
5263 /* This is either an integer multiply or a MADD. In both cases
5264 we want to recurse and cost the operands. */
5269 {
5271 /* MADD/MSUB. */
5273 else
5274 /* MUL. */
5276 }
5279 }
5280 else
5281 {
5283 {
5284 /* Floating-point FMA/FMUL can also support negations of the
5285 operands. */
5292 /* FMADD/FNMADD/FNMSUB/FMSUB. */
5294 else
5295 /* FMUL/FNMUL. */
5297 }
5302 }
5303 }
5305 static int
5307 machine_mode mode,
5308 addr_space_t as ATTRIBUTE_UNUSED,
5310 {
5318 {
5320 {
5321 /* This is a CONST or SYMBOL ref which will be split
5322 in a different way depending on the code model in use.
5323 Cost it through the generic infrastructure. */
5325 /* Divide through by the cost of one instruction to
5326 bring it to the same units as the address costs. */
5328 /* The cost is then the cost of preparing the address,
5329 followed by an immediate (possibly 0) offset. */
5331 }
5332 else
5333 {
5334 /* This is most likely a jump table from a case
5335 statement. */
5337 }
5338 }
5341 {
5353 else
5369 }
5373 {
5374 /* For the sake of calculating the cost of the shifted register
5375 component, we can treat same sized modes in the same way. */
5377 {
5390 /* We can't tell, or this is a 128-bit vector. */
5394 }
5395 }
5398 }
5400 /* Return true if the RTX X in mode MODE is a zero or sign extract
5401 usable in an ADD or SUB (extended register) instruction. */
5402 static bool
5404 {
5405 /* Catch add with a sign extract.
5406 This is add_<optab><mode>_multp2. */
5409 {
5420 op1))
5421 {
5423 }
5424 }
5427 }
5429 static bool
5431 {
5433 {
5445 }
5446 }
5448 /* Return true iff X is an rtx that will match an extr instruction
5449 i.e. as described in the *extr<mode>5_insn family of patterns.
5450 OP0 and OP1 will be set to the operands of the shifts involved
5451 on success and will be NULL_RTX otherwise. */
5453 static bool
5455 {
5470 {
5471 /* Canonicalise locally to ashift in op0, lshiftrt in op1. */
5483 {
5487 }
5488 }
5491 }
5493 /* Calculate the cost of calculating (if_then_else (OP0) (OP1) (OP2)),
5494 storing it in *COST. Result is true if the total cost of the operation
5495 has now been calculated. */
5496 static bool
5498 {
5499 rtx inner;
5500 rtx comparator;
5504 {
5508 }
5509 else
5510 {
5514 }
5517 {
5518 /* Conditional branch. */
5521 else
5522 {
5524 {
5526 {
5527 /* TBZ/TBNZ/CBZ/CBNZ. */
5529 /* TBZ/TBNZ. */
5532 else
5533 /* CBZ/CBNZ. */
5537 }
5538 }
5540 {
5541 /* TBZ/TBNZ. */
5544 }
5545 }
5546 }
5548 {
5549 /* It's a conditional operation based on the status flags,
5550 so it must be some flavor of CSEL. */
5552 /* CSNEG, CSINV, and CSINC are handled for free as part of CSEL. */
5561 }
5563 /* We don't know what this is, cost all operands. */
5565 }
5567 /* Calculate the cost of calculating X, storing it in *COST. Result
5568 is true if the total cost of the operation has now been calculated. */
5569 static bool
5572 {
5578 /* By default, assume that everything has equivalent cost to the
5579 cheapest instruction. Any additional costs are applied as a delta
5580 above this default. */
5583 /* TODO: The cost infrastructure currently does not handle
5584 vector operations. Assume that all vector operations
5585 are equally expensive. */
5587 {
5591 }
5594 {
5596 /* The cost depends entirely on the operands to SET. */
5602 {
5605 {
5617 }
5626 /* Fall through. */
5628 /* const0_rtx is in general free, but we will use an
5629 instruction to set a register to 0. */
5631 {
5632 /* The cost is 1 per register copied. */
5636 }
5637 else
5638 /* Cost is just the cost of the RHS of the set. */
5644 /* Bit-field insertion. Strip any redundant widening of
5645 the RHS to meet the width of the target. */
5656 {
5657 /* MOV immediate is assumed to always be cheap. */
5659 }
5660 else
5661 {
5662 /* BFM. */
5666 }
5671 /* We can't make sense of this, assume default cost. */
5674 }
5678 /* If an instruction can incorporate a constant within the
5679 instruction, the instruction's expression avoids calling
5680 rtx_cost() on the constant. If rtx_cost() is called on a
5681 constant, then it is usually because the constant must be
5682 moved into a register by one or more instructions.
5684 The exception is constant 0, which can be expressed
5685 as XZR/WZR and is therefore free. The exception to this is
5686 if we have (set (reg) (const0_rtx)) in which case we must cost
5687 the move. However, we can catch that when we cost the SET, so
5688 we don't need to consider that here. */
5691 else
5692 {
5693 /* To an approximation, building any other constant is
5694 proportionally expensive to the number of instructions
5695 required to build that constant. This is true whether we
5696 are compiling for SPEED or otherwise. */
5699 }
5704 {
5705 /* mov[df,sf]_aarch64. */
5707 /* FMOV (scalar immediate). */
5710 {
5711 /* This will be a load from memory. */
5714 else
5716 }
5717 else
5718 /* Otherwise this is +0.0. We get this using MOVI d0, #0
5719 or MOV v0.s[0], wzr - neither of which are modeled by the
5720 cost tables. Just use the default cost. */
5721 {
5722 }
5723 }
5729 {
5730 /* For loads we want the base cost of a load, plus an
5731 approximation for the additional cost of the addressing
5732 mode. */
5744 }
5752 {
5755 {
5756 /* CSETM. */
5759 }
5761 /* Cost this as SUB wzr, X. */
5765 }
5768 {
5769 /* Support (neg(fma...)) as a single instruction only if
5770 sign of zeros is unimportant. This matches the decision
5771 making in aarch64.md. */
5773 {
5774 /* FNMADD. */
5777 }
5779 /* FNEG. */
5782 }
5799 {
5802 }
5805 {
5806 /* TODO: A write to the CC flags possibly costs extra, this
5807 needs encoding in the cost tables. */
5809 /* CC_ZESWPmode supports zero extend for free. */
5813 /* ANDS. */
5815 {
5818 }
5821 {
5822 /* ADDS (and CMN alias). */
5825 }
5828 {
5829 /* SUBS. */
5832 }
5835 {
5836 /* CMN. */
5843 }
5845 /* CMP.
5847 Compare can freely swap the order of operands, and
5848 canonicalization puts the more complex operation first.
5849 But the integer MINUS logic expects the shift/extend
5850 operation in op1. */
5853 {
5856 }
5858 }
5861 {
5862 /* FCMP. */
5867 {
5869 /* FCMP supports constant 0.0 for no extra cost. */
5871 }
5873 }
5878 {
5882 cost_minus:
5885 /* Detect valid immediates. */
5891 {
5893 /* SUB(S) (immediate). */
5896 }
5898 /* Look for SUB (extended register). */
5900 {
5908 }
5912 /* Cost this as an FMA-alike operation. */
5916 {
5919 speed);
5921 }
5926 {
5928 /* SUB(S). */
5931 /* FSUB. */
5933 }
5935 }
5938 {
5939 rtx new_op0;
5944 cost_plus:
5947 {
5948 /* CSINC. */
5952 }
5957 {
5961 /* ADD (immediate). */
5964 }
5968 /* Look for ADD (extended register). */
5970 {
5978 }
5980 /* Strip any extend, leave shifts behind as we will
5981 cost them through mult_cost. */
5986 {
5988 speed);
5990 }
5995 {
5997 /* ADD. */
6000 /* FADD. */
6002 }
6004 }
6016 {
6023 }
6026 {
6033 }
6034 /* Fall through. */
6037 cost_logic:
6047 {
6048 /* This is a UBFM/SBFM. */
6053 }
6056 {
6057 /* We possibly get the immediate for free, this is not
6058 modelled. */
6061 {
6068 }
6069 else
6070 {
6073 /* Handle ORN, EON, or BIC. */
6079 /* If we had a shift on op0 then this is a logical-shift-
6080 by-register/immediate operation. Otherwise, this is just
6081 a logical operation. */
6083 {
6085 {
6086 /* Shift by immediate. */
6089 else
6091 }
6092 else
6094 }
6096 /* In both cases we want to cost both operands. */
6101 }
6102 }
6109 /* MVN-shifted-reg. */
6111 {
6118 }
6119 /* EON can have two forms: (xor (not a) b) but also (not (xor a b)).
6120 Handle the second form here taking care that 'a' in the above can
6121 be a shift. */
6123 {
6132 {
6135 else
6137 }
6140 }
6141 /* MVN. */
6150 /* If a value is written in SI mode, then zero extended to DI
6151 mode, the operation will in general be free as a write to
6152 a 'w' register implicitly zeroes the upper bits of an 'x'
6153 register. However, if this is
6155 (set (reg) (zero_extend (reg)))
6157 we must cost the explicit register move. */
6161 {
6165 /* MOV. */
6167 else
6168 /* Free, the cost is that of the SI mode operation. */
6172 }
6174 {
6175 /* All loads can zero extend to any size for free. */
6178 }
6180 /* UXTB/UXTH. */
6188 {
6189 /* LDRSH. */
6191 {
6198 }
6200 }
6211 {
6212 /* LSL (immediate), UBMF, UBFIZ and friends. These are all
6213 aliases. */
6217 /* We can incorporate zero/sign extend for free. */
6224 }
6225 else
6226 {
6227 /* LSLV. */
6232 }
6242 {
6243 /* ASR (immediate) and friends. */
6249 }
6250 else
6251 {
6253 /* ASR (register) and friends. */
6258 }
6263 {
6264 /* LDR. */
6267 }
6270 {
6271 /* ADRP, followed by ADD. */
6275 }
6278 {
6279 /* ADR. */
6282 }
6285 {
6286 /* One extra load instruction, after accessing the GOT. */
6290 }
6295 /* ADRP/ADD (immediate). */
6302 /* UBFX/SBFX. */
6306 /* We can trust that the immediates used will be correct (there
6307 are no by-register forms), so we need only cost op0. */
6313 /* aarch64_rtx_mult_cost always handles recursion to its
6314 operands. */
6320 {
6330 }
6337 {
6339 /* There is no integer SQRT, so only DIV and UDIV can get
6340 here. */
6342 else
6344 }
6372 /* FMSUB, FNMADD, and FNMSUB are free. */
6379 /* aarch64_fnma4_elt_to_64v2df has the NEG as operand 1,
6380 and the by-element operand as operand 0. */
6384 /* Catch vector-by-element operations. The by-element operand can
6385 either be (vec_duplicate (vec_select (x))) or just
6386 (vec_select (x)), depending on whether we are multiplying by
6387 a vector or a scalar.
6389 Canonicalization is not very good in these cases, FMA4 will put the
6390 by-element operand as operand 0, FNMA4 will have it as operand 1. */
6401 /* If the remaining parameters are not registers,
6402 get the cost to put them into registers. */
6421 /* Strip the rounding part. They will all be implemented
6422 by the fcvt* family of instructions anyway. */
6424 {
6433 }
6443 {
6446 /* FABD, which is analogous to FADD. */
6448 {
6455 }
6456 /* Simple FABS is analogous to FNEG. */
6459 }
6460 else
6461 {
6462 /* Integer ABS will either be split to
6463 two arithmetic instructions, or will be an ABS
6464 (scalar), which we don't model. */
6468 }
6474 {
6475 /* FMAXNM/FMINNM/FMAX/FMIN.
6476 TODO: This may not be accurate for all implementations, but
6477 we do not model this in the cost tables. */
6479 }
6483 /* The floating point round to integer frint* instructions. */
6485 {
6490 }
6493 {
6498 }
6503 /* Decompose <su>muldi3_highpart. */
6505 mode == DImode
6506 /* (lshiftrt:TI */
6509 /* (mult:TI */
6511 /* (ANY_EXTEND:TI (reg:DI))
6512 (ANY_EXTEND:TI (reg:DI))) */
6519 /* (const_int 64) */
6522 {
6523 /* UMULH/SMULH. */
6531 }
6533 /* Fall through. */
6536 }
6543 }
6545 /* Wrapper around aarch64_rtx_costs, dumps the partial, or total cost
6546 calculated for X. This cost is stored in *COST. Returns true
6547 if the total cost of X was calculated. */
6548 static bool
6551 {
6555 {
6560 }
6563 }
6565 static int
6568 {
6574 /* Caller save and pointer regs are equivalent to GENERAL_REGS. */
6581 /* Moving between GPR and stack cost is the same as GP2GP. */
6586 /* To/From the stack register, we move via the gprs. */
6592 {
6593 /* 128-bit operations on general registers require 2 instructions. */
6601 /* When AdvSIMD instructions are disabled it is not possible to move
6602 a 128-bit value directly between Q registers. This is handled in
6603 secondary reload. A general register is used as a scratch to move
6604 the upper DI value and the lower DI value is moved directly,
6605 hence the cost is the sum of three moves. */
6610 }
6620 }
6622 static int
6624 reg_class_t rclass ATTRIBUTE_UNUSED,
6626 {
6628 }
6630 /* Return the number of instructions that can be issued per cycle. */
6631 static int
6633 {
6635 }
6637 static int
6639 {
6643 }
6645 /* Vectorizer cost model target hooks. */
6647 /* Implement targetm.vectorize.builtin_vectorization_cost. */
6648 static int
6650 tree vectype,
6652 {
6656 {
6703 }
6704 }
6706 /* Implement targetm.vectorize.add_stmt_cost. */
6707 static unsigned
6711 {
6716 {
6721 /* Statements in an inner loop relative to the loop being
6722 vectorized are weighted more heavily. The value here is
6723 a function (linear for now) of the loop nest level. */
6725 {
6731 }
6735 }
6738 }
6742 /* Parse the architecture extension string. */
6744 static void
6746 {
6747 /* The extension string is parsed left to right. */
6750 /* Flag to say whether we are adding or removing an extension. */
6754 {
6758 str++;
6763 else
6767 {
6771 }
6776 {
6780 }
6782 /* Scan over the extensions table trying to find an exact match. */
6784 {
6786 {
6787 /* Add or remove the extension. */
6790 else
6793 }
6794 }
6797 {
6798 /* Extension not found in list. */
6801 }
6804 };
6807 }
6809 /* Parse the ARCH string. */
6811 static void
6813 {
6825 else
6829 {
6832 }
6834 /* Loop through the list of supported ARCHs to find a match. */
6836 {
6838 {
6846 {
6847 /* ARCH string contains at least one extension. */
6849 }
6852 {
6855 }
6858 }
6859 }
6861 /* ARCH name not found in list. */
6864 }
6866 /* Parse the CPU string. */
6868 static void
6870 {
6882 else
6886 {
6889 }
6891 /* Loop through the list of supported CPUs to find a match. */
6893 {
6895 {
6900 {
6901 /* CPU string contains at least one extension. */
6903 }
6906 }
6907 }
6909 /* CPU name not found in list. */
6912 }
6914 /* Parse the TUNE string. */
6916 static void
6918 {
6923 /* Loop through the list of supported CPUs to find a match. */
6925 {
6927 {
6930 }
6931 }
6933 /* CPU name not found in list. */
6936 }
6939 /* Implement TARGET_OPTION_OVERRIDE. */
6941 static void
6943 {
6944 /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
6945 If either of -march or -mtune is given, they override their
6946 respective component of -mcpu.
6948 So, first parse AARCH64_CPU_STRING, then the others, be careful
6949 with -march as, if -mcpu is not present on the command line, march
6950 must set a sensible default CPU. */
6952 {
6954 }
6957 {
6959 }
6962 {
6964 }
6966 #ifndef HAVE_AS_MABI_OPTION
6967 /* The compiler may have been configured with 2.23.* binutils, which does
6968 not have support for ILP32. */
6971 #endif
6977 /* This target defaults to strict volatile bitfields. */
6981 /* If the user did not specify a processor, choose the default
6982 one for them. This will be the CPU set during configuration using
6983 --with-cpu, otherwise it is "generic". */
6985 {
6988 }
7001 {
7002 #ifdef TARGET_FIX_ERR_A53_835769_DEFAULT
7004 #else
7006 #endif
7007 }
7009 /* If not opzimizing for size, set the default
7010 alignment to what the target wants */
7012 {
7019 }
7025 }
7027 /* Implement targetm.override_options_after_change. */
7029 static void
7031 {
7036 }
7040 {
7044 }
7046 void
7048 {
7050 }
7052 /* A checking mechanism for the implementation of the various code models. */
7053 static void
7055 {
7057 {
7059 {
7071 }
7072 }
7073 else
7075 }
7077 /* Return true if SYMBOL_REF X binds locally. */
7079 static bool
7081 {
7085 }
7087 /* Return true if SYMBOL_REF X is thread local */
7088 static bool
7090 {
7098 }
7100 /* Classify a TLS symbol into one of the TLS kinds. */
7101 enum aarch64_symbol_type
7103 {
7107 {
7124 }
7125 }
7127 /* Return the method that should be used to access SYMBOL_REF or
7128 LABEL_REF X in context CONTEXT. */
7130 enum aarch64_symbol_type
7133 {
7135 {
7137 {
7151 }
7152 }
7155 {
7163 {
7165 /* When we retreive symbol + offset address, we have to make sure
7166 the offset does not cause overflow of the final address. But
7167 we have no way of knowing the address of symbol at compile time
7168 so we can't accurately say if the distance between the PC and
7169 symbol + offset is outside the addressible range of +/-1M in the
7170 TINY code model. So we rely on images not being greater than
7171 1M and cap the offset at 1M and anything beyond 1M will have to
7172 be loaded using an alternative mechanism. */
7179 /* Same reasoning as the tiny code model, but the offset cap here is
7180 4G. */
7199 }
7200 }
7202 /* By default push everything into the constant pool. */
7204 }
7206 bool
7208 {
7210 }
7212 bool
7214 {
7222 }
7224 /* Return true if X holds either a quarter-precision or
7225 floating-point +0.0 constant. */
7226 static bool
7228 {
7232 /* TODO: We could handle moving 0.0 to a TFmode register,
7233 but first we would like to refactor the movtf_aarch64
7234 to be more amicable to split moves properly and
7235 correctly gate on TARGET_SIMD. For now - reject all
7236 constants which are not to SFmode or DFmode registers. */
7243 }
7245 static bool
7247 {
7248 /* Do not allow vector struct mode constants. We could support
7249 0 and -1 easily, but they need support in aarch64-simd.md. */
7253 /* This could probably go away because
7254 we now decompose CONST_INTs according to expand_mov_immediate. */
7265 }
7267 rtx
7269 {
7275 /* Can return in any reg. */
7278 }
7280 /* On AAPCS systems, this is the "struct __va_list". */
7283 /* Implement TARGET_BUILD_BUILTIN_VA_LIST.
7284 Return the type to use as __builtin_va_list.
7286 AAPCS64 \S 7.1.4 requires that va_list be a typedef for a type defined as:
7288 struct __va_list
7289 {
7290 void *__stack;
7291 void *__gr_top;
7292 void *__vr_top;
7293 int __gr_offs;
7294 int __vr_offs;
7295 }; */
7297 static tree
7299 {
7300 tree va_list_name;
7303 /* Create the type. */
7305 /* Give it the required name. */
7307 TYPE_DECL,
7309 va_list_type);
7314 /* Create the fields. */
7317 ptr_type_node);
7320 ptr_type_node);
7323 ptr_type_node);
7326 integer_type_node);
7329 integer_type_node);
7349 /* Compute its layout. */
7353 }
7355 /* Implement TARGET_EXPAND_BUILTIN_VA_START. */
7356 static void
7358 {
7362 tree t;
7368 gr_save_area_size
7370 vr_save_area_size
7374 {
7379 }
7388 NULL_TREE);
7390 NULL_TREE);
7392 NULL_TREE);
7394 NULL_TREE);
7396 NULL_TREE);
7398 /* Emit code to initialize STACK, which points to the next varargs stack
7399 argument. CUM->AAPCS_STACK_SIZE gives the number of stack words used
7400 by named arguments. STACK is 8-byte aligned. */
7407 /* Emit code to initialize GRTOP, the top of the GR save area.
7408 virtual_incoming_args_rtx should have been 16 byte aligned. */
7413 /* Emit code to initialize VRTOP, the top of the VR save area.
7414 This address is gr_save_area_bytes below GRTOP, rounded
7415 down to the next 16-byte boundary. */
7425 /* Emit code to initialize GROFF, the offset from GRTOP of the
7426 next GPR argument. */
7431 /* Likewise emit code to initialize VROFF, the offset from FTOP
7432 of the next VR argument. */
7436 }
7438 /* Implement TARGET_GIMPLIFY_VA_ARG_EXPR. */
7440 static tree
7443 {
7444 tree addr;
7450 machine_mode mode;
7477 type,
7481 {
7482 /* TYPE passed in fp/simd registers. */
7495 {
7498 }
7501 {
7503 }
7504 }
7505 else
7506 {
7507 /* TYPE passed in general registers. */
7520 {
7522 }
7523 }
7525 /* Get a local temporary for the field value. */
7528 /* Emit code to branch if off >= 0. */
7534 {
7535 /* Emit: offs = (offs + 15) & -16. */
7541 }
7542 else
7545 /* Update ap.__[g|v]r_offs */
7550 /* String up. */
7554 /* [cond2] if (ap.__[g|v]r_offs > 0) */
7559 /* String up: make sure the assignment happens before the use. */
7563 /* Prepare the trees handling the argument that is passed on the stack;
7564 the top level node will store in ON_STACK. */
7567 {
7568 /* if (alignof(type) > 8) (arg = arg + 15) & -16; */
7576 }
7577 else
7579 /* Advance ap.__stack */
7587 /* String up roundup and advance. */
7590 /* String up with arg */
7592 /* Big-endianness related address adjustment. */
7595 {
7599 }
7604 /* Adjustment to OFFSET in the case of BIG_ENDIAN. */
7614 {
7615 /* type ha; // treat as "struct {ftype field[n];}"
7616 ... [computing offs]
7617 for (i = 0; i <nregs; ++i, offs += 16)
7618 ha.field[i] = *((ftype *)(ap.__vr_top + offs));
7619 return ha; */
7623 /* Declare a local variable. */
7627 /* Establish the base type. */
7629 {
7642 /* The half precision and quad precision are not fully supported yet. Enable
7643 the following code after the support is complete. Need to find the correct
7644 type node for __fp16 *. */
7645 #if 0
7650 #endif
7653 {
7657 }
7661 }
7663 /* *(field_ptr_t)&ha = *((field_ptr_t)vr_saved_area */
7671 /* ha.field[i] = *((field_ptr_t)vr_saved_area + i) */
7673 {
7683 }
7687 }
7697 }
7699 /* Implement TARGET_SETUP_INCOMING_VARARGS. */
7701 static void
7705 {
7707 CUMULATIVE_ARGS local_cum;
7710 /* The caller has advanced CUM up to, but not beyond, the last named
7711 argument. Advance a local copy of CUM past the last "real" named
7712 argument, to find out how many registers are left over. */
7716 /* Found out how many registers we need to save. */
7721 {
7726 }
7729 {
7731 {
7734 /* virtual_incoming_args_rtx should have been 16-byte aligned. */
7742 }
7744 {
7745 /* We can't use move_block_from_reg, because it will use
7746 the wrong mode, storing D regs only. */
7750 /* Set OFF to the offset from virtual_incoming_args_rtx of
7751 the first vector register. The VR save area lies below
7752 the GR one, and is aligned to 16 bytes. */
7758 {
7766 }
7767 }
7768 }
7770 /* We don't save the size into *PRETEND_SIZE because we want to avoid
7771 any complication of having crtl->args.pretend_args_size changed. */
7776 }
7778 static void
7780 {
7783 {
7785 {
7788 }
7789 }
7790 }
7792 /* Walk down the type tree of TYPE counting consecutive base elements.
7793 If *MODEP is VOIDmode, then set it to the first valid floating point
7794 type. If a non-floating point type is found, or if a floating point
7795 type that doesn't match a non-VOIDmode *MODEP is found, then return -1,
7796 otherwise return the count in the sub-tree. */
7797 static int
7799 {
7800 machine_mode mode;
7801 HOST_WIDE_INT size;
7804 {
7832 /* Use V2SImode and V4SImode as representatives of all 64-bit
7833 and 128-bit vector types. */
7836 {
7845 }
7850 /* Vector modes are considered to be opaque: two vectors are
7851 equivalent for the purposes of being homogeneous aggregates
7852 if they are the same size. */
7859 {
7863 /* Can't handle incomplete types nor sizes that are not
7864 fixed. */
7871 || !index
7882 /* There must be no padding. */
7887 }
7890 {
7893 tree field;
7895 /* Can't handle incomplete types nor sizes that are not
7896 fixed. */
7902 {
7910 }
7912 /* There must be no padding. */
7917 }
7921 {
7922 /* These aren't very interesting except in a degenerate case. */
7925 tree field;
7927 /* Can't handle incomplete types nor sizes that are not
7928 fixed. */
7934 {
7942 }
7944 /* There must be no padding. */
7949 }
7953 }
7956 }
7958 /* Return TRUE if the type, as described by TYPE and MODE, is a composite
7959 type as described in AAPCS64 \S 4.3. This includes aggregate, union and
7960 array types. The C99 floating-point complex types are also considered
7961 as composite types, according to AAPCS64 \S 7.1.1. The complex integer
7962 types, which are GCC extensions and out of the scope of AAPCS64, are
7963 treated as composite types here as well.
7965 Note that MODE itself is not sufficient in determining whether a type
7966 is such a composite type or not. This is because
7967 stor-layout.c:compute_record_mode may have already changed the MODE
7968 (BLKmode) of a RECORD_TYPE TYPE to some other mode. For example, a
7969 structure with only one field may have its MODE set to the mode of the
7970 field. Also an integer mode whose size matches the size of the
7971 RECORD_TYPE type may be used to substitute the original mode
7972 (i.e. BLKmode) in certain circumstances. In other words, MODE cannot be
7973 solely relied on. */
7975 static bool
7977 machine_mode mode)
7978 {
7988 }
7990 /* Return TRUE if the type, as described by TYPE and MODE, is a short vector
7991 type as described in AAPCS64 \S 4.1.2.
7993 See the comment above aarch64_composite_type_p for the notes on MODE. */
7995 static bool
7997 machine_mode mode)
7998 {
8009 }
8011 /* Return TRUE if an argument, whose type is described by TYPE and MODE,
8012 shall be passed or returned in simd/fp register(s) (providing these
8013 parameter passing registers are available).
8015 Upon successful return, *COUNT returns the number of needed registers,
8016 *BASE_MODE returns the mode of the individual register and when IS_HAF
8017 is not NULL, *IS_HA indicates whether or not the argument is a homogeneous
8018 floating-point aggregate or a homogeneous short-vector aggregate. */
8020 static bool
8022 const_tree type,
8026 {
8034 {
8037 }
8039 {
8043 }
8045 {
8049 {
8052 }
8053 else
8055 }
8056 else
8061 }
8063 /* Implement TARGET_STRUCT_VALUE_RTX. */
8065 static rtx
8068 {
8070 }
8072 /* Implements target hook vector_mode_supported_p. */
8073 static bool
8075 {
8086 }
8088 /* Return appropriate SIMD container
8089 for MODE within a vector of WIDTH bits. */
8090 static machine_mode
8092 {
8095 {
8098 {
8113 }
8114 else
8116 {
8127 }
8128 }
8130 }
8132 /* Return 128-bit container as the preferred SIMD mode for MODE. */
8133 static machine_mode
8135 {
8137 }
8139 /* Return the bitmask of possible vector sizes for the vectorizer
8140 to iterate over. */
8141 static unsigned int
8143 {
8145 }
8147 /* Implement TARGET_MANGLE_TYPE. */
8151 {
8152 /* The AArch64 ABI documents say that "__va_list" has to be
8153 managled as if it is in the "std" namespace. */
8157 /* Mangle AArch64-specific internal types. TYPE_NAME is non-NULL_TREE for
8158 builtin types. */
8162 /* Use the default mangling. */
8164 }
8167 /* Return true if the rtx_insn contains a MEM RTX somewhere
8168 in it. */
8170 static bool
8172 {
8179 }
8181 /* Find the first rtx_insn before insn that will generate an assembly
8182 instruction. */
8186 {
8190 do
8191 {
8193 }
8197 }
8199 static bool
8201 {
8203 /* A number of these may be AArch32 only. */
8208 };
8211 {
8214 }
8217 }
8219 /* Check if there is a register dependency between a load and the insn
8220 for which we hold recog_data. */
8222 static bool
8224 {
8225 rtx load_reg;
8235 {
8241 }
8243 }
8246 /* When working around the Cortex-A53 erratum 835769,
8247 given rtx_insn INSN, return true if it is a 64-bit multiply-accumulate
8248 instruction and has a preceding memory instruction such that a NOP
8249 should be inserted between them. */
8251 bool
8253 {
8256 rtx body;
8269 /* aarch64_prev_real_insn can call recog_memoized on insns other than INSN.
8270 Restore recog state to INSN to avoid state corruption. */
8278 /* If the previous insn is a memory op and there is no dependency between
8279 it and the DImode madd, emit a NOP between them. If body is NULL then we
8280 have a complex memory operation, probably a load/store pair.
8281 Be conservative for now and emit a NOP. */
8288 }
8291 /* Implement FINAL_PRESCAN_INSN. */
8293 void
8295 {
8298 }
8301 /* Return the equivalent letter for size. */
8302 static char
8304 {
8306 {
8312 }
8313 }
8315 /* Return true iff x is a uniform vector of floating-point
8316 constants, and the constant can be represented in
8317 quarter-precision form. Note, as aarch64_float_const_representable
8318 rejects both +0.0 and -0.0, we will also reject +0.0 and -0.0. */
8319 static bool
8321 {
8336 {
8344 }
8347 }
8349 /* Return true for valid and false for invalid. */
8350 bool
8353 {
8354 #define CHECK(STRIDE, ELSIZE, CLASS, TEST, SHIFT, NEG) \
8355 matches = 1; \
8356 for (i = 0; i < idx; i += (STRIDE)) \
8357 if (!(TEST)) \
8358 matches = 0; \
8359 if (matches) \
8360 { \
8361 immtype = (CLASS); \
8362 elsize = (ELSIZE); \
8363 eshift = (SHIFT); \
8364 emvn = (NEG); \
8365 break; \
8366 }
8376 {
8382 {
8387 }
8390 }
8392 /* Splat vector constant out into a byte vector. */
8394 {
8395 /* The vector is provided in gcc endian-neutral fashion. For aarch64_be,
8396 it must be laid out in the vector register in reverse order. */
8402 {
8405 }
8407 {
8410 }
8411 else
8415 {
8418 {
8421 }
8424 }
8425 }
8427 /* Sanity check. */
8430 do
8431 {
8480 }
8487 {
8497 /* Un-invert bytes of recognized vector, if necessary. */
8503 {
8504 /* FIXME: Broken on 32-bit H_W_I hosts. */
8513 }
8514 else
8515 {
8519 /* Construct 'abcdefgh' because the assembler cannot handle
8520 generic constants. */
8525 }
8526 }
8529 #undef CHECK
8530 }
8532 /* Check of immediate shift constants are within range. */
8533 bool
8535 {
8539 else
8541 }
8543 /* Return true if X is a uniform vector where all elements
8544 are either the floating-point constant 0.0 or the
8545 integer constant 0. */
8546 bool
8548 {
8550 }
8552 bool
8554 {
8559 {
8564 }
8567 }
8569 bool
8572 machine_mode mode)
8573 {
8586 }
8588 /* Return a const_int vector of VAL. */
8589 rtx
8591 {
8600 }
8602 /* Check OP is a legal scalar immediate for the MOVI instruction. */
8604 bool
8606 {
8607 machine_mode vmode;
8613 }
8615 /* Construct and return a PARALLEL RTX vector with elements numbering the
8616 lanes of either the high (HIGH == TRUE) or low (HIGH == FALSE) half of
8617 the vector - from the perspective of the architecture. This does not
8618 line up with GCC's perspective on lane numbers, so we end up with
8619 different masks depending on our target endian-ness. The diagram
8620 below may help. We must draw the distinction when building masks
8621 which select one half of the vector. An instruction selecting
8622 architectural low-lanes for a big-endian target, must be described using
8623 a mask selecting GCC high-lanes.
8625 Big-Endian Little-Endian
8627 GCC 0 1 2 3 3 2 1 0
8628 | x | x | x | x | | x | x | x | x |
8629 Architecture 3 2 1 0 3 2 1 0
8631 Low Mask: { 2, 3 } { 0, 1 }
8632 High Mask: { 0, 1 } { 2, 3 }
8633 */
8635 rtx
8637 {
8643 rtx t1;
8648 else
8656 }
8658 /* Check OP for validity as a PARALLEL RTX vector with elements
8659 numbering the lanes of either the high (HIGH == TRUE) or low lanes,
8660 from the perspective of the architecture. See the diagram above
8661 aarch64_simd_vect_par_cnst_half for more details. */
8663 bool
8666 {
8679 {
8686 }
8688 }
8690 /* Bounds-check lanes. Ensure OPERAND lies between LOW (inclusive) and
8691 HIGH (exclusive). */
8692 void
8694 const_tree exp)
8695 {
8696 HOST_WIDE_INT lane;
8701 {
8704 else
8706 }
8707 }
8709 /* Return TRUE if OP is a valid vector addressing mode. */
8710 bool
8712 {
8715 }
8717 /* Emit a register copy from operand to operand, taking care not to
8718 early-clobber source registers in the process.
8720 COUNT is the number of components into which the copy needs to be
8721 decomposed. */
8722 void
8725 {
8735 else
8739 }
8741 /* Compute and return the length of aarch64_simd_mov<mode>, where <mode> is
8742 one of VSTRUCT modes: OI, CI or XI. */
8743 int
8745 {
8746 machine_mode mode;
8751 {
8754 {
8763 }
8764 }
8766 }
8768 /* Compute and return the length of aarch64_simd_reglist<mode>, where <mode> is
8769 one of VSTRUCT modes: OI, CI, EI, or XI. */
8770 int
8772 {
8774 }
8776 /* Implement target hook TARGET_VECTOR_ALIGNMENT. The AAPCS64 sets the maximum
8777 alignment of a vector to 128 bits. */
8778 static HOST_WIDE_INT
8780 {
8783 }
8785 /* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE. */
8786 static bool
8788 {
8792 /* We guarantee alignment for vectors up to 128-bits. */
8797 /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned. */
8799 }
8801 /* If VALS is a vector constant that can be loaded into a register
8802 using DUP, generate instructions to do so and return an RTX to
8803 assign to the register. Otherwise return NULL_RTX. */
8804 static rtx
8806 {
8811 rtx x;
8818 {
8822 }
8827 /* We can load this constant by using DUP and a constant in a
8828 single ARM register. This will be cheaper than a vector
8829 load. */
8832 }
8835 /* Generate code to load VALS, which is a PARALLEL containing only
8836 constants (for vec_init) or CONST_VECTOR, efficiently into a
8837 register. Returns an RTX to copy into the register, or NULL_RTX
8838 for a PARALLEL that can not be converted into a CONST_VECTOR. */
8839 static rtx
8841 {
8843 rtx const_dup;
8852 {
8853 /* A CONST_VECTOR must contain only CONST_INTs and
8854 CONST_DOUBLEs, but CONSTANT_P allows more (e.g. SYMBOL_REF).
8855 Only store valid constants in a CONST_VECTOR. */
8857 {
8860 n_const++;
8861 }
8864 }
8865 else
8870 /* Load using MOVI/MVNI. */
8873 /* Loaded using DUP. */
8876 /* Load from constant pool. We can not take advantage of single-cycle
8877 LD1 because we need a PC-relative addressing mode. */
8879 else
8880 /* A PARALLEL containing something not valid inside CONST_VECTOR.
8881 We can not construct an initializer. */
8883 }
8885 void
8887 {
8896 {
8900 else
8905 }
8908 {
8911 {
8914 }
8915 }
8917 /* Splat a single non-constant element if we can. */
8919 {
8923 }
8925 /* Half the fields (or less) are non-constant. Load constant then overwrite
8926 varying fields. Hope that this is more efficient than using the stack. */
8928 {
8931 /* Load constant part of vector. We really don't care what goes into the
8932 parts we will overwrite, but we're more likely to be able to load the
8933 constant efficiently if it has fewer, larger, repeating parts
8934 (see aarch64_simd_valid_immediate). */
8936 {
8942 {
8943 /* Look in the copied vector, as more elements are const. */
8946 {
8949 }
8950 }
8952 }
8955 /* Insert variables. */
8960 {
8966 }
8968 }
8970 /* Construct the vector in memory one field at a time
8971 and load the whole vector. */
8979 }
8981 static unsigned HOST_WIDE_INT
8983 {
8984 return
8987 }
8989 #ifndef TLS_SECTION_ASM_FLAG
8991 #endif
8993 void
8995 tree decl ATTRIBUTE_UNUSED)
8996 {
8999 /* If we have already declared this section, we can use an
9000 abbreviated form to switch back to it -- unless this section is
9001 part of a COMDAT groups, in which case GAS requires the full
9002 declaration every time. */
9005 {
9008 }
9031 {
9037 else
9040 #ifdef TYPE_OPERAND_FMT
9042 #else
9044 #endif
9051 {
9054 else
9057 }
9058 }
9061 }
9063 /* Select a format to encode pointers in exception handling data. */
9064 int
9066 {
9069 {
9074 /* text+got+data < 4Gb. 4-byte signed relocs are sufficient
9075 for everything. */
9079 /* No assumptions here. 8-byte relocs required. */
9082 }
9084 }
9086 /* Emit load exclusive. */
9088 static void
9091 {
9095 {
9102 }
9105 }
9107 /* Emit store exclusive. */
9109 static void
9112 {
9116 {
9123 }
9126 }
9128 /* Mark the previous jump instruction as unlikely. */
9130 static void
9132 {
9137 }
9139 /* Expand a compare and swap pattern. */
9141 void
9143 {
9159 /* Normally the succ memory model must be stronger than fail, but in the
9160 unlikely event of fail being ACQUIRE and succ being RELEASE we need to
9161 promote succ to ACQ_REL so that we don't lose the acquire semantics. */
9168 {
9171 /* For short modes, we're going to perform the comparison in SImode,
9172 so do the zero-extension now. */
9176 /* Fall through. */
9180 /* Force the value into a register if needed. */
9187 }
9190 {
9197 }
9207 }
9209 /* Split a compare and swap pattern. */
9211 void
9213 {
9215 machine_mode mode;
9230 {
9233 }
9247 {
9252 }
9253 else
9254 {
9258 }
9261 }
9263 /* Split an atomic operation. */
9265 void
9268 {
9272 rtx x;
9281 else
9288 {
9302 {
9305 }
9306 /* Fall through. */
9312 }
9321 }
9323 static void
9325 {
9333 }
9335 static void
9337 {
9339 {
9342 }
9344 {
9349 }
9351 }
9353 /* Target hook for c_mode_for_suffix. */
9354 static machine_mode
9356 {
9361 }
9363 /* We can only represent floating point constants which will fit in
9364 "quarter-precision" values. These values are characterised by
9365 a sign bit, a 4-bit mantissa and a 3-bit exponent. And are given
9366 by:
9368 (-1)^s * (n/16) * 2^r
9370 Where:
9371 's' is the sign bit.
9372 'n' is an integer in the range 16 <= n <= 31.
9373 'r' is an integer in the range -3 <= r <= 4. */
9375 /* Return true iff X can be represented by a quarter-precision
9376 floating point immediate operand X. Note, we cannot represent 0.0. */
9377 bool
9379 {
9380 /* This represents our current view of how many bits
9381 make up the mantissa. */
9396 /* We cannot represent infinities, NaNs or +/-zero. We won't
9397 know if we have +zero until we analyse the mantissa, but we
9398 can reject the other invalid values. */
9403 /* Extract exponent. */
9407 /* For the mantissa, we expand into two HOST_WIDE_INTS, apart from the
9408 highest (sign) bit, with a fixed binary point at bit point_pos.
9409 m1 holds the low part of the mantissa, m2 the high part.
9410 WARNING: If we ever have a representation using more than 2 * H_W_I - 1
9411 bits for the mantissa, this can fail (low bits will be lost). */
9415 /* If the low part of the mantissa has bits set we cannot represent
9416 the value. */
9419 /* We have rejected the lower HOST_WIDE_INT, so update our
9420 understanding of how many bits lie in the mantissa and
9421 look only at the high HOST_WIDE_INT. */
9425 /* We can only represent values with a mantissa of the form 1.xxxx. */
9430 /* Having filtered unrepresentable values, we may now remove all
9431 but the highest 5 bits. */
9434 /* We cannot represent the value 0.0, so reject it. This is handled
9435 elsewhere. */
9439 /* Then, as bit 4 is always set, we can mask it off, leaving
9440 the mantissa in the range [0, 15]. */
9444 /* GCC internally does not use IEEE754-like encoding (where normalized
9445 significands are in the range [1, 2). GCC uses [0.5, 1) (see real.c).
9446 Our mantissa values are shifted 4 places to the left relative to
9447 normalized IEEE754 so we must modify the exponent returned by REAL_EXP
9448 by 5 places to correct for GCC's representation. */
9452 }
9456 machine_mode mode,
9458 {
9468 /* This will return true to show const_vector is legal for use as either
9469 a AdvSIMD MOVI instruction (or, implicitly, MVNI) immediate. It will
9470 also update INFO to show how the immediate should be generated. */
9479 {
9483 else
9484 {
9485 #define buf_size 20
9486 REAL_VALUE_TYPE r;
9490 #undef buf_size
9494 else
9498 }
9499 }
9511 else
9515 }
9519 machine_mode mode)
9520 {
9521 machine_mode vmode;
9527 }
9529 /* Split operands into moves from op[1] + op[2] into op[0]. */
9531 void
9533 {
9544 {
9545 /* No-op move. Can't split to nothing; emit something. */
9548 }
9550 /* Preserve register attributes for variable tracking. */
9555 /* Special case of reversed high/low parts. */
9558 {
9562 }
9564 {
9565 /* Try to avoid unnecessary moves if part of the result
9566 is in the right place already. */
9571 }
9572 else
9573 {
9578 }
9579 }
9581 /* vec_perm support. */
9583 #define MAX_VECT_LEN 16
9585 struct expand_vec_perm_d
9586 {
9589 machine_mode vmode;
9593 };
9595 /* Generate a variable permutation. */
9597 static void
9599 {
9610 {
9612 {
9613 /* Expand the argument to a V16QI mode by duplicating it. */
9617 }
9618 else
9619 {
9621 }
9622 }
9623 else
9624 {
9625 rtx pair;
9628 {
9632 }
9633 else
9634 {
9638 }
9639 }
9640 }
9642 void
9644 {
9648 rtx mask;
9650 /* The TBL instruction does not use a modulo index, so we must take care
9651 of that ourselves. */
9656 /* For big-endian, we also need to reverse the index within the vector
9657 (but not which vector). */
9659 {
9660 /* If one_vector_p, mask is a vector of (nelt - 1)'s already. */
9665 }
9667 }
9669 /* Recognize patterns suitable for the TRN instructions. */
9670 static bool
9672 {
9681 /* Note that these are little-endian tests.
9682 We correct for big-endian later. */
9687 else
9692 {
9697 }
9699 /* Success! */
9706 {
9709 }
9713 {
9715 {
9728 }
9729 }
9730 else
9731 {
9733 {
9746 }
9747 }
9751 }
9753 /* Recognize patterns suitable for the UZP instructions. */
9754 static bool
9756 {
9765 /* Note that these are little-endian tests.
9766 We correct for big-endian later. */
9771 else
9776 {
9780 }
9782 /* Success! */
9789 {
9792 }
9796 {
9798 {
9811 }
9812 }
9813 else
9814 {
9816 {
9829 }
9830 }
9834 }
9836 /* Recognize patterns suitable for the ZIP instructions. */
9837 static bool
9839 {
9848 /* Note that these are little-endian tests.
9849 We correct for big-endian later. */
9852 /* Do Nothing. */
9853 ;
9856 else
9861 {
9868 }
9870 /* Success! */
9877 {
9880 }
9884 {
9886 {
9899 }
9900 }
9901 else
9902 {
9904 {
9917 }
9918 }
9922 }
9924 /* Recognize patterns for the EXT insn. */
9926 static bool
9928 {
9931 rtx offset;
9935 /* Check if the extracted indices are increasing by one. */
9937 {
9940 {
9941 /* We'll pass the same vector in twice, so allow indices to wrap. */
9943 }
9946 }
9949 {
9962 }
9964 /* Success! */
9968 /* The case where (location == 0) is a no-op for both big- and little-endian,
9969 and is removed by the mid-end at optimization levels -O1 and higher. */
9972 {
9973 /* After setup, we want the high elements of the first vector (stored
9974 at the LSB end of the register), and the low elements of the second
9975 vector (stored at the MSB end of the register). So swap. */
9977 /* location != 0 (above), so safe to assume (nelt - location) < nelt. */
9979 }
9984 }
9986 /* Recognize patterns for the REV insns. */
9988 static bool
9990 {
9999 {
10002 {
10007 }
10011 {
10018 }
10022 {
10033 }
10037 }
10041 {
10042 /* This is guaranteed to be true as the value of diff
10043 is 7, 3, 1 and we should have enough elements in the
10044 queue to generate this. Getting a vector mask with a
10045 value of diff other than these values implies that
10046 something is wrong by the time we get here. */
10050 }
10052 /* Success! */
10058 }
10060 static bool
10062 {
10065 rtx in0;
10068 rtx lane;
10072 {
10075 }
10077 /* The generic preparation in aarch64_expand_vec_perm_const_1
10078 swaps the operand order and the permute indices if it finds
10079 d->perm[0] to be in the second operand. Thus, we can always
10080 use d->op0 and need not do any extra arithmetic to get the
10081 correct lane number. */
10086 {
10099 }
10103 }
10105 static bool
10107 {
10115 /* Generic code will try constant permutation twice. Once with the
10116 original mode and again with the elements lowered to QImode.
10117 So wait and don't do the selector expansion ourselves. */
10122 {
10125 /* If big-endian and two vectors we end up with a weird mixed-endian
10126 mode on NEON. Reverse the index within each word but not the word
10127 itself. */
10130 }
10136 }
10138 static bool
10140 {
10141 /* The pattern matching functions above are written to look for a small
10142 number to begin the sequence (0, 1, N/2). If we begin with an index
10143 from the second operand, we can swap the operands. */
10145 {
10153 }
10156 {
10170 }
10172 }
10174 /* Expand a vec_perm_const pattern. */
10176 bool
10178 {
10192 {
10197 }
10200 {
10209 /* The elements of PERM do not suggest that only the first operand
10210 is used, but both operands are identical. Allow easier matching
10211 of the permutation by folding the permutation into the single
10212 input vector. */
10213 /* Fall Through. */
10225 }
10228 }
10230 static bool
10233 {
10243 /* Calculate whether all elements are in one vector. */
10245 {
10249 }
10251 /* If all elements are from the second vector, reindex as if from the
10252 first vector. */
10257 /* Check whether the mask can be applied to a single vector. */
10270 }
10272 rtx
10274 {
10275 /* We have to reverse each vector because we dont have
10276 a permuted load that can reverse-load according to ABI rules. */
10277 rtx mask;
10291 }
10293 /* Implement MODES_TIEABLE_P. */
10295 bool
10297 {
10301 /* We specifically want to allow elements of "structure" modes to
10302 be tieable to the structure. This more general condition allows
10303 other rarer situations too. */
10310 }
10312 /* Return a new RTX holding the result of moving POINTER forward by
10313 AMOUNT bytes. */
10315 static rtx
10317 {
10322 }
10324 /* Return a new RTX holding the result of moving POINTER forward by the
10325 size of the mode it points to. */
10327 static rtx
10329 {
10333 }
10335 /* Copy one MODE sized block from SRC to DST, then progress SRC and DST by
10336 MODE bytes. */
10338 static void
10340 machine_mode mode)
10341 {
10344 /* "Cast" the pointers to the correct mode. */
10347 /* Emit the memcpy. */
10350 /* Move the pointers forward. */
10353 }
10355 /* Expand movmem, as if from a __builtin_memcpy. Return true if
10356 we succeed, otherwise return false. */
10358 bool
10360 {
10364 rtx base;
10367 /* When optimizing for size, give a better estimate of the length of a
10368 memcpy call, but use the default otherwise. */
10371 /* We can't do anything smart if the amount to copy is not constant. */
10377 /* Try to keep the number of instructions low. For cases below 16 bytes we
10378 need to make at most two moves. For cases above 16 bytes it will be one
10379 move for each 16 byte chunk, then at most two additional moves. */
10389 /* Simple cases. Copy 0-3 bytes, as (if applicable) a 2-byte, then a
10390 1-byte chunk. */
10392 {
10394 {
10397 }
10403 }
10405 /* Copy 4-8 bytes. First a 4-byte chunk, then (if applicable) a second
10406 4-byte chunk, partially overlapping with the previously copied chunk. */
10408 {
10412 {
10418 }
10420 }
10422 /* Copy more than 8 bytes. Copy chunks of 16 bytes until we run out of
10423 them, then (if applicable) an 8-byte chunk. */
10425 {
10427 {
10430 }
10431 else
10432 {
10435 }
10436 }
10438 /* Finish the final bytes of the copy. We can always do this in one
10439 instruction. We either copy the exact amount we need, or partially
10440 overlap with the previous chunk we copied and copy 8-bytes. */
10449 else
10450 {
10452 {
10456 }
10457 else
10458 {
10464 }
10465 }
10468 }
10470 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
10472 static unsigned HOST_WIDE_INT
10474 {
10476 }
10478 static bool
10483 {
10484 /* STORE_BY_PIECES can be used when copying a constant string, but
10485 in that case each 64-bit chunk takes 5 insns instead of 2 (LDR/STR).
10486 For now we always fail this and let the move_by_pieces code copy
10487 the string from read-only memory. */
10492 }
10494 static enum machine_mode
10496 {
10498 {
10531 }
10532 }
10534 static rtx
10537 {
10556 {
10572 }
10577 {
10580 }
10594 {
10597 }
10602 }
10604 static rtx
10607 {
10626 {
10644 }
10649 {
10652 }
10669 {
10672 }
10678 }
10680 #undef TARGET_GEN_CCMP_FIRST
10681 #define TARGET_GEN_CCMP_FIRST aarch64_gen_ccmp_first
10683 #undef TARGET_GEN_CCMP_NEXT
10684 #define TARGET_GEN_CCMP_NEXT aarch64_gen_ccmp_next
10686 /* Implement TARGET_SCHED_MACRO_FUSION_P. Return true if target supports
10687 instruction fusion of some sort. */
10689 static bool
10691 {
10693 }
10696 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P. Return true if PREV and CURR
10697 should be kept together during scheduling. */
10699 static bool
10701 {
10702 rtx set_dest;
10705 /* prev and curr are simple SET insns i.e. no flag setting or branching. */
10713 {
10714 /* We are trying to match:
10715 prev (mov) == (set (reg r0) (const_int imm16))
10716 curr (movk) == (set (zero_extract (reg r0)
10717 (const_int 16)
10718 (const_int 16))
10719 (const_int imm16_1)) */
10731 {
10733 }
10734 }
10738 {
10740 /* We're trying to match:
10741 prev (adrp) == (set (reg r1)
10742 (high (symbol_ref ("SYM"))))
10743 curr (add) == (set (reg r0)
10744 (lo_sum (reg r1)
10745 (symbol_ref ("SYM"))))
10746 Note that r0 need not necessarily be the same as r1, especially
10747 during pre-regalloc scheduling. */
10751 {
10759 }
10760 }
10764 {
10766 /* We're trying to match:
10767 prev (movk) == (set (zero_extract (reg r0)
10768 (const_int 16)
10769 (const_int 32))
10770 (const_int imm16_1))
10771 curr (movk) == (set (zero_extract (reg r0)
10772 (const_int 16)
10773 (const_int 48))
10774 (const_int imm16_2)) */
10790 }
10793 {
10794 /* We're trying to match:
10795 prev (adrp) == (set (reg r0)
10796 (high (symbol_ref ("SYM"))))
10797 curr (ldr) == (set (reg r1)
10798 (mem (lo_sum (reg r0)
10799 (symbol_ref ("SYM")))))
10800 or
10801 curr (ldr) == (set (reg r1)
10802 (zero_extend (mem
10803 (lo_sum (reg r0)
10804 (symbol_ref ("SYM")))))) */
10807 {
10820 }
10821 }
10825 {
10828 /* FIXME: this misses some which is considered simple arthematic
10829 instructions for ThunderX. Simple shifts are missed here. */
10835 }
10838 }
10840 /* If MEM is in the form of [base+offset], extract the two parts
10841 of address and set to BASE and OFFSET, otherwise return false
10842 after clearing BASE and OFFSET. */
10844 bool
10846 {
10847 rtx addr;
10854 {
10858 }
10862 {
10866 }
10872 }
10874 /* Types for scheduling fusion. */
10875 enum sched_fusion_type
10876 {
10878 SCHED_FUSION_LD_SIGN_EXTEND,
10879 SCHED_FUSION_LD_ZERO_EXTEND,
10880 SCHED_FUSION_LD,
10881 SCHED_FUSION_ST,
10882 SCHED_FUSION_NUM
10883 };
10885 /* If INSN is a load or store of address in the form of [base+offset],
10886 extract the two parts and set to BASE and OFFSET. Return scheduling
10887 fusion type this INSN is. */
10889 static enum sched_fusion_type
10891 {
10908 {
10913 }
10915 {
10920 }
10925 {
10928 }
10929 else
10936 }
10938 /* Implement the TARGET_SCHED_FUSION_PRIORITY hook.
10940 Currently we only support to fuse ldr or str instructions, so FUSION_PRI
10941 and PRI are only calculated for these instructions. For other instruction,
10942 FUSION_PRI and PRI are simply set to MAX_PRI - 1. In the future, other
10943 type instruction fusion can be added by returning different priorities.
10945 It's important that irrelevant instructions get the largest FUSION_PRI. */
10947 static void
10950 {
10960 {
10964 }
10966 /* Set FUSION_PRI according to fusion type and base register. */
10969 /* Calculate PRI. */
10972 /* INSN with smaller offset goes first. */
10976 else
10981 }
10983 /* Given OPERANDS of consecutive load/store, check if we can merge
10984 them into ldp/stp. LOAD is true if they are load instructions.
10985 MODE is the mode of memory operands. */
10987 bool
10990 {
10996 {
11004 }
11005 else
11006 {
11011 }
11013 /* The mems cannot be volatile. */
11017 /* Check if the addresses are in the form of [base+offset]. */
11025 /* Check if the bases are same. */
11032 /* Check if the offsets are consecutive. */
11036 /* Check if the addresses are clobbered by load. */
11038 {
11042 /* In increasing order, the last load can clobber the address. */
11045 }
11049 else
11054 else
11057 /* Check if the registers are of same class. */
11062 }
11064 /* Given OPERANDS of consecutive load/store, check if we can merge
11065 them into ldp/stp by adjusting the offset. LOAD is true if they
11066 are load instructions. MODE is the mode of memory operands.
11068 Given below consecutive stores:
11070 str w1, [xb, 0x100]
11071 str w1, [xb, 0x104]
11072 str w1, [xb, 0x108]
11073 str w1, [xb, 0x10c]
11075 Though the offsets are out of the range supported by stp, we can
11076 still pair them after adjusting the offset, like:
11078 add scratch, xb, 0x100
11079 stp w1, w1, [scratch]
11080 stp w1, w1, [scratch, 0x8]
11082 The peephole patterns detecting this opportunity should guarantee
11083 the scratch register is avaliable. */
11085 bool
11088 {
11095 {
11108 }
11109 else
11110 {
11119 }
11120 /* Skip if memory operand is by itslef valid for ldp/stp. */
11124 /* The mems cannot be volatile. */
11129 /* Check if the addresses are in the form of [base+offset]. */
11143 /* Check if the bases are same. */
11154 /* Check if the offsets are consecutive. */
11163 /* Check if the addresses are clobbered by load. */
11165 {
11171 /* In increasing order, the last load can clobber the address. */
11174 }
11178 else
11183 else
11188 else
11193 else
11196 /* Check if the registers are of same class. */
11201 }
11203 /* Given OPERANDS of consecutive load/store, this function pairs them
11204 into ldp/stp after adjusting the offset. It depends on the fact
11205 that addresses of load/store instructions are in increasing order.
11206 MODE is the mode of memory operands. CODE is the rtl operator
11207 which should be applied to all memory operands, it's SIGN_EXTEND,
11208 ZERO_EXTEND or UNKNOWN. */
11210 bool
11213 {
11219 {
11224 }
11225 else
11226 {
11232 }
11237 /* Adjust offset thus it can fit in ldp/stp instruction. */
11245 /* Further adjust to make sure all offsets are OK. */
11247 {
11250 }
11252 /* Make sure the adjustment can be done with ADD/SUB instructions. */
11257 {
11260 }
11262 /* Create new memory references. */
11266 /* Check if the adjusted address is OK for ldp/stp. */
11285 {
11290 }
11292 {
11297 }
11300 {
11305 }
11306 else
11307 {
11312 }
11314 /* Emit adjusting instruction. */
11317 /* Emit ldp/stp instructions. */
11325 }
11327 #undef TARGET_ADDRESS_COST
11328 #define TARGET_ADDRESS_COST aarch64_address_cost
11330 /* This hook will determines whether unnamed bitfields affect the alignment
11331 of the containing structure. The hook returns true if the structure
11332 should inherit the alignment requirements of an unnamed bitfield's
11333 type. */
11334 #undef TARGET_ALIGN_ANON_BITFIELD
11335 #define TARGET_ALIGN_ANON_BITFIELD hook_bool_void_true
11337 #undef TARGET_ASM_ALIGNED_DI_OP
11340 #undef TARGET_ASM_ALIGNED_HI_OP
11343 #undef TARGET_ASM_ALIGNED_SI_OP
11346 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
11347 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK \
11348 hook_bool_const_tree_hwi_hwi_const_tree_true
11350 #undef TARGET_ASM_FILE_START
11351 #define TARGET_ASM_FILE_START aarch64_start_file
11353 #undef TARGET_ASM_OUTPUT_MI_THUNK
11354 #define TARGET_ASM_OUTPUT_MI_THUNK aarch64_output_mi_thunk
11356 #undef TARGET_ASM_SELECT_RTX_SECTION
11357 #define TARGET_ASM_SELECT_RTX_SECTION aarch64_select_rtx_section
11359 #undef TARGET_ASM_TRAMPOLINE_TEMPLATE
11360 #define TARGET_ASM_TRAMPOLINE_TEMPLATE aarch64_asm_trampoline_template
11362 #undef TARGET_BUILD_BUILTIN_VA_LIST
11363 #define TARGET_BUILD_BUILTIN_VA_LIST aarch64_build_builtin_va_list
11365 #undef TARGET_CALLEE_COPIES
11366 #define TARGET_CALLEE_COPIES hook_bool_CUMULATIVE_ARGS_mode_tree_bool_false
11368 #undef TARGET_CAN_ELIMINATE
11369 #define TARGET_CAN_ELIMINATE aarch64_can_eliminate
11371 #undef TARGET_CANNOT_FORCE_CONST_MEM
11372 #define TARGET_CANNOT_FORCE_CONST_MEM aarch64_cannot_force_const_mem
11374 #undef TARGET_CONDITIONAL_REGISTER_USAGE
11375 #define TARGET_CONDITIONAL_REGISTER_USAGE aarch64_conditional_register_usage
11377 /* Only the least significant bit is used for initialization guard
11378 variables. */
11379 #undef TARGET_CXX_GUARD_MASK_BIT
11380 #define TARGET_CXX_GUARD_MASK_BIT hook_bool_void_true
11382 #undef TARGET_C_MODE_FOR_SUFFIX
11383 #define TARGET_C_MODE_FOR_SUFFIX aarch64_c_mode_for_suffix
11385 #ifdef TARGET_BIG_ENDIAN_DEFAULT
11386 #undef TARGET_DEFAULT_TARGET_FLAGS
11387 #define TARGET_DEFAULT_TARGET_FLAGS (MASK_BIG_END)
11388 #endif
11390 #undef TARGET_CLASS_MAX_NREGS
11391 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
11393 #undef TARGET_BUILTIN_DECL
11394 #define TARGET_BUILTIN_DECL aarch64_builtin_decl
11396 #undef TARGET_EXPAND_BUILTIN
11397 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
11399 #undef TARGET_EXPAND_BUILTIN_VA_START
11400 #define TARGET_EXPAND_BUILTIN_VA_START aarch64_expand_builtin_va_start
11402 #undef TARGET_FOLD_BUILTIN
11403 #define TARGET_FOLD_BUILTIN aarch64_fold_builtin
11405 #undef TARGET_FUNCTION_ARG
11406 #define TARGET_FUNCTION_ARG aarch64_function_arg
11408 #undef TARGET_FUNCTION_ARG_ADVANCE
11409 #define TARGET_FUNCTION_ARG_ADVANCE aarch64_function_arg_advance
11411 #undef TARGET_FUNCTION_ARG_BOUNDARY
11412 #define TARGET_FUNCTION_ARG_BOUNDARY aarch64_function_arg_boundary
11414 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
11415 #define TARGET_FUNCTION_OK_FOR_SIBCALL aarch64_function_ok_for_sibcall
11417 #undef TARGET_FUNCTION_VALUE
11418 #define TARGET_FUNCTION_VALUE aarch64_function_value
11420 #undef TARGET_FUNCTION_VALUE_REGNO_P
11421 #define TARGET_FUNCTION_VALUE_REGNO_P aarch64_function_value_regno_p
11423 #undef TARGET_FRAME_POINTER_REQUIRED
11424 #define TARGET_FRAME_POINTER_REQUIRED aarch64_frame_pointer_required
11426 #undef TARGET_GIMPLE_FOLD_BUILTIN
11427 #define TARGET_GIMPLE_FOLD_BUILTIN aarch64_gimple_fold_builtin
11429 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
11430 #define TARGET_GIMPLIFY_VA_ARG_EXPR aarch64_gimplify_va_arg_expr
11432 #undef TARGET_INIT_BUILTINS
11433 #define TARGET_INIT_BUILTINS aarch64_init_builtins
11435 #undef TARGET_LEGITIMATE_ADDRESS_P
11436 #define TARGET_LEGITIMATE_ADDRESS_P aarch64_legitimate_address_hook_p
11438 #undef TARGET_LEGITIMATE_CONSTANT_P
11439 #define TARGET_LEGITIMATE_CONSTANT_P aarch64_legitimate_constant_p
11441 #undef TARGET_LIBGCC_CMP_RETURN_MODE
11442 #define TARGET_LIBGCC_CMP_RETURN_MODE aarch64_libgcc_cmp_return_mode
11444 #undef TARGET_LRA_P
11445 #define TARGET_LRA_P hook_bool_void_true
11447 #undef TARGET_MANGLE_TYPE
11448 #define TARGET_MANGLE_TYPE aarch64_mangle_type
11450 #undef TARGET_MEMORY_MOVE_COST
11451 #define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
11453 #undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
11454 #define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
11456 #undef TARGET_MUST_PASS_IN_STACK
11457 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
11459 /* This target hook should return true if accesses to volatile bitfields
11460 should use the narrowest mode possible. It should return false if these
11461 accesses should use the bitfield container type. */
11462 #undef TARGET_NARROW_VOLATILE_BITFIELD
11463 #define TARGET_NARROW_VOLATILE_BITFIELD hook_bool_void_false
11465 #undef TARGET_OPTION_OVERRIDE
11466 #define TARGET_OPTION_OVERRIDE aarch64_override_options
11468 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
11469 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE \
11470 aarch64_override_options_after_change
11472 #undef TARGET_PASS_BY_REFERENCE
11473 #define TARGET_PASS_BY_REFERENCE aarch64_pass_by_reference
11475 #undef TARGET_PREFERRED_RELOAD_CLASS
11476 #define TARGET_PREFERRED_RELOAD_CLASS aarch64_preferred_reload_class
11478 #undef TARGET_SCHED_REASSOCIATION_WIDTH
11479 #define TARGET_SCHED_REASSOCIATION_WIDTH aarch64_reassociation_width
11481 #undef TARGET_SECONDARY_RELOAD
11482 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
11484 #undef TARGET_SHIFT_TRUNCATION_MASK
11485 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
11487 #undef TARGET_SETUP_INCOMING_VARARGS
11488 #define TARGET_SETUP_INCOMING_VARARGS aarch64_setup_incoming_varargs
11490 #undef TARGET_STRUCT_VALUE_RTX
11491 #define TARGET_STRUCT_VALUE_RTX aarch64_struct_value_rtx
11493 #undef TARGET_REGISTER_MOVE_COST
11494 #define TARGET_REGISTER_MOVE_COST aarch64_register_move_cost
11496 #undef TARGET_RETURN_IN_MEMORY
11497 #define TARGET_RETURN_IN_MEMORY aarch64_return_in_memory
11499 #undef TARGET_RETURN_IN_MSB
11500 #define TARGET_RETURN_IN_MSB aarch64_return_in_msb
11502 #undef TARGET_RTX_COSTS
11503 #define TARGET_RTX_COSTS aarch64_rtx_costs_wrapper
11505 #undef TARGET_SCHED_ISSUE_RATE
11506 #define TARGET_SCHED_ISSUE_RATE aarch64_sched_issue_rate
11508 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
11509 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
11510 aarch64_sched_first_cycle_multipass_dfa_lookahead
11512 #undef TARGET_TRAMPOLINE_INIT
11513 #define TARGET_TRAMPOLINE_INIT aarch64_trampoline_init
11515 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
11516 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P aarch64_use_blocks_for_constant_p
11518 #undef TARGET_VECTOR_MODE_SUPPORTED_P
11519 #define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
11521 #undef TARGET_ARRAY_MODE_SUPPORTED_P
11522 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
11524 #undef TARGET_VECTORIZE_ADD_STMT_COST
11525 #define TARGET_VECTORIZE_ADD_STMT_COST aarch64_add_stmt_cost
11527 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
11528 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
11529 aarch64_builtin_vectorization_cost
11531 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
11532 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
11534 #undef TARGET_VECTORIZE_BUILTINS
11535 #define TARGET_VECTORIZE_BUILTINS
11537 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
11538 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
11539 aarch64_builtin_vectorized_function
11541 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
11542 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
11543 aarch64_autovectorize_vector_sizes
11545 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
11546 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV \
11547 aarch64_atomic_assign_expand_fenv
11549 /* Section anchor support. */
11551 #undef TARGET_MIN_ANCHOR_OFFSET
11552 #define TARGET_MIN_ANCHOR_OFFSET -256
11554 /* Limit the maximum anchor offset to 4k-1, since that's the limit for a
11555 byte offset; we can do much more for larger data types, but have no way
11556 to determine the size of the access. We assume accesses are aligned. */
11557 #undef TARGET_MAX_ANCHOR_OFFSET
11558 #define TARGET_MAX_ANCHOR_OFFSET 4095
11560 #undef TARGET_VECTOR_ALIGNMENT
11561 #define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
11563 #undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
11564 #define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
11565 aarch64_simd_vector_alignment_reachable
11567 /* vec_perm support. */
11569 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
11570 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
11571 aarch64_vectorize_vec_perm_const_ok
11574 #undef TARGET_FIXED_CONDITION_CODE_REGS
11575 #define TARGET_FIXED_CONDITION_CODE_REGS aarch64_fixed_condition_code_regs
11577 #undef TARGET_FLAGS_REGNUM
11578 #define TARGET_FLAGS_REGNUM CC_REGNUM
11580 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
11581 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
11583 #undef TARGET_ASAN_SHADOW_OFFSET
11584 #define TARGET_ASAN_SHADOW_OFFSET aarch64_asan_shadow_offset
11586 #undef TARGET_LEGITIMIZE_ADDRESS
11587 #define TARGET_LEGITIMIZE_ADDRESS aarch64_legitimize_address
11589 #undef TARGET_USE_BY_PIECES_INFRASTRUCTURE_P
11590 #define TARGET_USE_BY_PIECES_INFRASTRUCTURE_P \
11591 aarch64_use_by_pieces_infrastructure_p
11593 #undef TARGET_CAN_USE_DOLOOP_P
11594 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
11596 #undef TARGET_SCHED_MACRO_FUSION_P
11597 #define TARGET_SCHED_MACRO_FUSION_P aarch64_macro_fusion_p
11599 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
11600 #define TARGET_SCHED_MACRO_FUSION_PAIR_P aarch_macro_fusion_pair_p
11602 #undef TARGET_SCHED_FUSION_PRIORITY
11603 #define TARGET_SCHED_FUSION_PRIORITY aarch64_sched_fusion_priority