1 /* Copyright (C) 2012-2017 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library. If not, see
17 <http://www.gnu.org/licenses/>. */
23 * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
26 /* To test the page crossing code path more thoroughly, compile with
27 -DTEST_PAGE_CROSS - this will force all calls through the slower
28 entry path. This option is not intended for production use. */
30 /* Arguments and results. */
34 /* Locals and temporaries. */
46 /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
47 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
48 can be done in parallel across the entire word. A faster check
49 (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives
50 false hits for characters 129..255. */
52 #define REP8_01 0x0101010101010101
53 #define REP8_7f 0x7f7f7f7f7f7f7f7f
54 #define REP8_80 0x8080808080808080
56 #ifdef TEST_PAGE_CROSS
57 # define MIN_PAGE_SIZE 15
59 # define MIN_PAGE_SIZE 4096
62 /* Since strings are short on average, we check the first 16 bytes
63 of the string for a NUL character. In order to do an unaligned ldp
64 safely we have to do a page cross check first. If there is a NUL
65 byte we calculate the length from the 2 8-byte words using
66 conditional select to reduce branch mispredictions (it is unlikely
67 strlen will be repeatedly called on strings with the same length).
69 If the string is longer than 16 bytes, we align src so don't need
70 further page cross checks, and process 32 bytes per iteration
71 using the fast NUL check. If we encounter non-ASCII characters,
72 fallback to a second loop using the full NUL check.
74 If the page cross check fails, we read 16 bytes from an aligned
75 address, remove any characters before the string, and continue
76 in the main loop using aligned loads. Since strings crossing a
77 page in the first 16 bytes are rare (probability of
78 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized.
80 AArch64 systems have a minimum page size of 4k. We don't bother
81 checking for larger page sizes - the cost of setting up the correct
82 page size is just not worth the extra gain from a small reduction in
83 the cases taking the slow path. Note that we only care about
84 whether the first fetch, which may be misaligned, crosses a page
87 ENTRY_ALIGN (__strlen, 6)
90 and tmp1, srcin, MIN_PAGE_SIZE - 1
92 cmp tmp1, MIN_PAGE_SIZE - 16
94 ldp data1, data2, [srcin]
96 /* For big-endian, carry propagation (if the final byte in the
97 string is 0x01) means we cannot use has_nul1/2 directly.
98 Since we expect strings to be small and early-exit,
99 byte-swap the data now so has_null1/2 will be correct. */
103 sub tmp1, data1, zeroones
104 orr tmp2, data1, REP8_7f
105 sub tmp3, data2, zeroones
106 orr tmp4, data2, REP8_7f
107 bics has_nul1, tmp1, tmp2
108 bic has_nul2, tmp3, tmp4
109 ccmp has_nul2, 0, 0, eq
110 beq L(main_loop_entry)
112 /* Enter with C = has_nul1 == 0. */
113 csel has_nul1, has_nul1, has_nul2, cc
115 rev has_nul1, has_nul1
117 csel len, xzr, len, cc
118 add len, len, tmp1, lsr 3
121 /* The inner loop processes 32 bytes per iteration and uses the fast
122 NUL check. If we encounter non-ASCII characters, use a second
123 loop with the accurate NUL check. */
129 ldp data1, data2, [src, 32]!
131 sub tmp1, data1, zeroones
132 sub tmp3, data2, zeroones
134 tst tmp2, zeroones, lsl 7
136 ldp data1, data2, [src, 16]
137 sub tmp1, data1, zeroones
138 sub tmp3, data2, zeroones
140 tst tmp2, zeroones, lsl 7
144 /* The fast check failed, so do the slower, accurate NUL check. */
145 orr tmp2, data1, REP8_7f
146 orr tmp4, data2, REP8_7f
147 bics has_nul1, tmp1, tmp2
148 bic has_nul2, tmp3, tmp4
149 ccmp has_nul2, 0, 0, eq
152 /* Enter with C = has_nul1 == 0. */
155 /* For big-endian, carry propagation (if the final byte in the
156 string is 0x01) means we cannot use has_nul1/2 directly. The
157 easiest way to get the correct byte is to byte-swap the data
158 and calculate the syndrome a second time. */
159 csel data1, data1, data2, cc
161 sub tmp1, data1, zeroones
162 orr tmp2, data1, REP8_7f
163 bic has_nul1, tmp1, tmp2
165 csel has_nul1, has_nul1, has_nul2, cc
168 rev has_nul1, has_nul1
171 csel len, len, tmp2, cc
172 add len, len, tmp1, lsr 3
176 ldp data1, data2, [src, 16]!
177 sub tmp1, data1, zeroones
178 orr tmp2, data1, REP8_7f
179 sub tmp3, data2, zeroones
180 orr tmp4, data2, REP8_7f
181 bics has_nul1, tmp1, tmp2
182 bic has_nul2, tmp3, tmp4
183 ccmp has_nul2, 0, 0, eq
185 ldp data1, data2, [src, 16]!
186 sub tmp1, data1, zeroones
187 orr tmp2, data1, REP8_7f
188 sub tmp3, data2, zeroones
189 orr tmp4, data2, REP8_7f
190 bics has_nul1, tmp1, tmp2
191 bic has_nul2, tmp3, tmp4
192 ccmp has_nul2, 0, 0, eq
196 /* Load 16 bytes from [srcin & ~15] and force the bytes that precede
197 srcin to 0x7f, so we ignore any NUL bytes before the string.
198 Then continue in the aligned loop. */
201 ldp data1, data2, [src]
205 /* Big-endian. Early bytes are at MSB. */
206 lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
208 /* Little-endian. Early bytes are at LSB. */
209 lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
211 orr tmp1, tmp1, REP8_80
212 orn data1, data1, tmp1
213 orn tmp2, data2, tmp1
215 csel data1, data1, tmp4, eq
216 csel data2, data2, tmp2, eq
217 b L(page_cross_entry)
219 weak_alias (__strlen, strlen)
220 libc_hidden_builtin_def (strlen)