1 /* Optimized memchr with sse2
2 Copyright (C) 2011-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
23 # define CFI_PUSH(REG) \
24 cfi_adjust_cfa_offset (4); \
25 cfi_rel_offset (REG, 0)
27 # define CFI_POP(REG) \
28 cfi_adjust_cfa_offset (-4); \
31 # define PUSH(REG) pushl REG; CFI_PUSH (REG)
32 # define POP(REG) popl REG; CFI_POP (REG)
38 # ifndef USE_AS_RAWMEMCHR
40 # define RETURN POP(%edi); ret; CFI_PUSH(%edi);
44 # define MEMCHR __memchr_sse2_bsf
51 movd STR2(%esp), %xmm1
53 # ifndef USE_AS_RAWMEMCHR
60 punpcklbw %xmm1, %xmm1
61 punpcklbw %xmm1, %xmm1
64 pshufd $0, %xmm1, %xmm1
71 /* Check if there is a match. */
74 je L(unaligned_no_match_1)
75 /* Check which byte is a match. */
78 # ifndef USE_AS_RAWMEMCHR
86 L(unaligned_no_match_1):
87 # ifndef USE_AS_RAWMEMCHR
106 # ifndef USE_AS_RAWMEMCHR
112 /* Handle unaligned string. */
114 # ifndef USE_AS_RAWMEMCHR
127 /* Check if there is a match. */
129 /* Remove the leading bytes. */
132 je L(unaligned_no_match)
133 /* Check which byte is a match. */
136 # ifndef USE_AS_RAWMEMCHR
149 L(unaligned_no_match):
150 # ifndef USE_AS_RAWMEMCHR
151 /* Calculate the last acceptable address and check for possible
152 addition overflow by using satured math:
154 edx |= -(edx < ecx) */
166 /* Loop start on aligned string. */
168 # ifndef USE_AS_RAWMEMCHR
180 # ifndef USE_AS_RAWMEMCHR
181 movdqa 16(%edi), %xmm2
183 movdqa 16(%edx), %xmm2
190 # ifndef USE_AS_RAWMEMCHR
191 movdqa 32(%edi), %xmm3
193 movdqa 32(%edx), %xmm3
200 # ifndef USE_AS_RAWMEMCHR
201 movdqa 48(%edi), %xmm4
203 movdqa 48(%edx), %xmm4
207 # ifndef USE_AS_RAWMEMCHR
216 # ifndef USE_AS_RAWMEMCHR
223 # ifndef USE_AS_RAWMEMCHR
235 # ifndef USE_AS_RAWMEMCHR
236 movdqa 16(%edi), %xmm2
238 movdqa 16(%edx), %xmm2
245 # ifndef USE_AS_RAWMEMCHR
246 movdqa 32(%edi), %xmm3
248 movdqa 32(%edx), %xmm3
255 # ifndef USE_AS_RAWMEMCHR
256 movdqa 48(%edi), %xmm3
258 movdqa 48(%edx), %xmm3
263 # ifndef USE_AS_RAWMEMCHR
271 # ifndef USE_AS_RAWMEMCHR
282 # ifndef USE_AS_RAWMEMCHR
286 movdqa 16(%edi), %xmm2
287 movdqa 32(%edi), %xmm3
288 movdqa 48(%edi), %xmm4
291 movdqa 16(%edx), %xmm2
292 movdqa 32(%edx), %xmm3
293 movdqa 48(%edx), %xmm4
305 # ifndef USE_AS_RAWMEMCHR
314 # ifndef USE_AS_RAWMEMCHR
328 # ifndef USE_AS_RAWMEMCHR
329 movdqa 32(%edi), %xmm3
331 movdqa 32(%edx), %xmm3
336 # ifndef USE_AS_RAWMEMCHR
337 pcmpeqb 48(%edi), %xmm1
339 pcmpeqb 48(%edx), %xmm1
348 # ifndef USE_AS_RAWMEMCHR
349 lea 48(%edi, %eax), %eax
352 lea 48(%edx, %eax), %eax
356 # ifndef USE_AS_RAWMEMCHR
369 movdqa 16(%edi), %xmm2
375 movdqa 32(%edi), %xmm3
383 pcmpeqb 48(%edi), %xmm1
400 pcmpeqb 16(%edi), %xmm1
410 # ifndef USE_AS_RAWMEMCHR
411 lea -16(%eax, %edi), %eax
414 lea -16(%eax, %edx), %eax
421 # ifndef USE_AS_RAWMEMCHR
432 # ifndef USE_AS_RAWMEMCHR
433 lea 16(%eax, %edi), %eax
436 lea 16(%eax, %edx), %eax
443 # ifndef USE_AS_RAWMEMCHR
444 lea 32(%eax, %edi), %eax
447 lea 32(%eax, %edx), %eax
451 # ifndef USE_AS_RAWMEMCHR
468 lea 16(%edi, %eax), %eax
478 lea 32(%edi, %eax), %eax
488 lea 48(%edi, %eax), %eax
494 # ifndef USE_AS_RAWMEMCHR