| blob | 8879855d92b80b3d3f54f5b229eb7bf063440c6a |
1 /* strcmp with SSE4.2
2 Copyright (C) 2009, 2010 Free Software Foundation, Inc.
3 Contributed by Intel Corporation.
4 This file is part of the GNU C Library.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, write to the Free
18 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
19 02111-1307 USA. */
21 #include <sysdep.h>
22 #include <init-arch.h>
24 #ifdef USE_AS_STRNCMP
25 /* Since the counter, %r11, is unsigned, we branch to strcmp_exitz
26 if the new counter > the old one or is 0. */
27 # define UPDATE_STRNCMP_COUNTER \
28 /* calculate left number to compare */ \
29 lea -16(%rcx, %r11), %r9; \
30 cmp %r9, %r11; \
31 jb LABEL(strcmp_exitz_sse4_2); \
32 test %r9, %r9; \
33 je LABEL(strcmp_exitz_sse4_2); \
34 mov %r9, %r11
36 # define STRCMP_SSE42 __strncmp_sse42
37 # define STRCMP_SSSE3 __strncmp_ssse3
38 # define STRCMP_SSE2 __strncmp_sse2
39 # define __GI_STRCMP __GI_strncmp
40 #elif defined USE_AS_STRCASECMP_L
41 # include "locale-defines.h"
43 # define UPDATE_STRNCMP_COUNTER
45 # define STRCMP_SSE42 __strcasecmp_l_sse42
46 # define STRCMP_SSSE3 __strcasecmp_l_ssse3
47 # define STRCMP_SSE2 __strcasecmp_l_sse2
48 # define __GI_STRCMP __GI___strcasecmp_l
49 #elif defined USE_AS_STRNCASECMP_L
50 # include "locale-defines.h"
52 /* Since the counter, %r11, is unsigned, we branch to strcmp_exitz
53 if the new counter > the old one or is 0. */
54 # define UPDATE_STRNCMP_COUNTER \
55 /* calculate left number to compare */ \
56 lea -16(%rcx, %r11), %r9; \
57 cmp %r9, %r11; \
58 jb LABEL(strcmp_exitz_sse4_2); \
59 test %r9, %r9; \
60 je LABEL(strcmp_exitz_sse4_2); \
61 mov %r9, %r11
63 # define STRCMP_SSE42 __strncasecmp_l_sse42
64 # define STRCMP_SSSE3 __strncasecmp_l_ssse3
65 # define STRCMP_SSE2 __strncasecmp_l_sse2
66 # define __GI_STRCMP __GI___strncasecmp_l
67 #else
68 # define UPDATE_STRNCMP_COUNTER
69 # ifndef STRCMP
70 # define STRCMP strcmp
71 # define STRCMP_SSE42 __strcmp_sse42
72 # define STRCMP_SSSE3 __strcmp_ssse3
73 # define STRCMP_SSE2 __strcmp_sse2
74 # define __GI_STRCMP __GI_strcmp
75 # endif
76 #endif
78 #ifndef LABEL
79 # define LABEL(l) L(l)
80 #endif
82 /* Define multiple versions only for the definition in libc. Don't
83 define multiple versions for strncmp in static library since we
84 need strncmp before the initialization happened. */
85 #if (defined SHARED || !defined USE_AS_STRNCMP) && !defined NOT_IN_libc
86 .text
87 ENTRY(STRCMP)
88 .type STRCMP, @gnu_indirect_function
89 cmpl $0, __cpu_features+KIND_OFFSET(%rip)
90 jne 1f
91 call __init_cpu_features
92 1:
93 leaq STRCMP_SSE42(%rip), %rax
94 testl $bit_SSE4_2, __cpu_features+CPUID_OFFSET+index_SSE4_2(%rip)
95 jnz 2f
96 leaq STRCMP_SSSE3(%rip), %rax
97 testl $bit_SSSE3, __cpu_features+CPUID_OFFSET+index_SSSE3(%rip)
98 jnz 2f
99 leaq STRCMP_SSE2(%rip), %rax
100 2: ret
101 END(STRCMP)
103 # ifdef USE_AS_STRCASECMP_L
104 ENTRY(__strcasecmp)
105 .type __strcasecmp, @gnu_indirect_function
106 cmpl $0, __cpu_features+KIND_OFFSET(%rip)
107 jne 1f
108 call __init_cpu_features
109 1:
110 leaq __strcasecmp_sse42(%rip), %rax
111 testl $bit_SSE4_2, __cpu_features+CPUID_OFFSET+index_SSE4_2(%rip)
112 jnz 2f
113 leaq __strcasecmp_ssse3(%rip), %rax
114 testl $bit_SSSE3, __cpu_features+CPUID_OFFSET+index_SSSE3(%rip)
115 jnz 2f
116 leaq __strcasecmp_sse2(%rip), %rax
117 2: ret
118 END(__strcasecmp)
119 weak_alias (__strcasecmp, strcasecmp)
120 # endif
121 # ifdef USE_AS_STRNCASECMP_L
122 ENTRY(__strncasecmp)
123 .type __strncasecmp, @gnu_indirect_function
124 cmpl $0, __cpu_features+KIND_OFFSET(%rip)
125 jne 1f
126 call __init_cpu_features
127 1:
128 leaq __strncasecmp_sse42(%rip), %rax
129 testl $bit_SSE4_2, __cpu_features+CPUID_OFFSET+index_SSE4_2(%rip)
130 jnz 2f
131 leaq __strncasecmp_ssse3(%rip), %rax
132 testl $bit_SSSE3, __cpu_features+CPUID_OFFSET+index_SSSE3(%rip)
133 jnz 2f
134 leaq __strncasecmp_sse2(%rip), %rax
135 2: ret
136 END(__strncasecmp)
137 weak_alias (__strncasecmp, strncasecmp)
138 # endif
140 /* We use 0x1a:
141 _SIDD_SBYTE_OPS
142 | _SIDD_CMP_EQUAL_EACH
143 | _SIDD_NEGATIVE_POLARITY
144 | _SIDD_LEAST_SIGNIFICANT
145 on pcmpistri to find out if two 16byte data elements are the same
146 and the offset of the first different byte. There are 4 cases:
148 1. Both 16byte data elements are valid and identical.
149 2. Both 16byte data elements have EOS and identical.
150 3. Both 16byte data elements are valid and they differ at offset X.
151 4. At least one 16byte data element has EOS at offset X. Two 16byte
152 data elements must differ at or before offset X.
154 Here is the table of ECX, CFlag, ZFlag and SFlag for 4 cases:
156 case ECX CFlag ZFlag SFlag
157 1 16 0 0 0
158 2 16 0 1 1
159 3 X 1 0 0
160 4 0 <= X 1 0/1 0/1
162 We exit from the loop for cases 2, 3 and 4 with jbe which branches
163 when either CFlag or ZFlag is 1. If CFlag == 0, we return 0 for
164 case 2. */
166 /* Put all SSE 4.2 functions together. */
167 .section .text.sse4.2,"ax",@progbits
168 .align 16
169 .type STRCMP_SSE42, @function
170 # ifdef USE_AS_STRCASECMP_L
171 ENTRY (__strcasecmp_sse42)
172 movq __libc_tsd_LOCALE@gottpoff(%rip),%rax
173 movq %fs:(%rax),%rdx
175 // XXX 5 byte should be before the function
176 /* 5-byte NOP. */
177 .byte 0x0f,0x1f,0x44,0x00,0x00
178 END (__strcasecmp_sse42)
179 /* FALLTHROUGH to strcasecmp_l. */
180 # endif
181 # ifdef USE_AS_STRNCASECMP_L
182 ENTRY (__strncasecmp_sse42)
183 movq __libc_tsd_LOCALE@gottpoff(%rip),%rax
184 movq %fs:(%rax),%rcx
186 // XXX 5 byte should be before the function
187 /* 5-byte NOP. */
188 .byte 0x0f,0x1f,0x44,0x00,0x00
189 END (__strncasecmp_sse42)
190 /* FALLTHROUGH to strncasecmp_l. */
191 # endif
193 STRCMP_SSE42:
194 cfi_startproc
195 CALL_MCOUNT
197 /*
198 * This implementation uses SSE to compare up to 16 bytes at a time.
199 */
200 # ifdef USE_AS_STRCASECMP_L
201 /* We have to fall back on the C implementation for locales
202 with encodings not matching ASCII for single bytes. */
203 # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0
204 movq LOCALE_T___LOCALES+LC_CTYPE*8(%rdx), %rax
205 # else
206 movq (%rdx), %rax
207 # endif
208 testl $0, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax)
209 jne __strcasecmp_l_nonascii
210 # endif
211 # ifdef USE_AS_STRNCASECMP_L
212 /* We have to fall back on the C implementation for locales
213 with encodings not matching ASCII for single bytes. */
214 # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0
215 movq LOCALE_T___LOCALES+LC_CTYPE*8(%rcx), %rax
216 # else
217 movq (%rcx), %rax
218 # endif
219 testl $0, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax)
220 jne __strncasecmp_l_nonascii
221 # endif
223 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
224 test %rdx, %rdx
225 je LABEL(strcmp_exitz_sse4_2)
226 cmp $1, %rdx
227 je LABEL(Byte0_sse4_2)
228 mov %rdx, %r11
229 # endif
230 mov %esi, %ecx
231 mov %edi, %eax
232 /* Use 64bit AND here to avoid long NOP padding. */
233 and $0x3f, %rcx /* rsi alignment in cache line */
234 and $0x3f, %rax /* rdi alignment in cache line */
235 # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
236 .section .rodata.cst16,"aM",@progbits,16
237 .align 16
238 .Lbelowupper_sse4:
239 .quad 0x4040404040404040
240 .quad 0x4040404040404040
241 .Ltopupper_sse4:
242 .quad 0x5b5b5b5b5b5b5b5b
243 .quad 0x5b5b5b5b5b5b5b5b
244 .Ltouppermask_sse4:
245 .quad 0x2020202020202020
246 .quad 0x2020202020202020
247 .previous
248 movdqa .Lbelowupper_sse4(%rip), %xmm4
249 # define UCLOW_reg %xmm4
250 movdqa .Ltopupper_sse4(%rip), %xmm5
251 # define UCHIGH_reg %xmm5
252 movdqa .Ltouppermask_sse4(%rip), %xmm6
253 # define LCQWORD_reg %xmm6
254 # endif
255 cmp $0x30, %ecx
256 ja LABEL(crosscache_sse4_2)/* rsi: 16-byte load will cross cache line */
257 cmp $0x30, %eax
258 ja LABEL(crosscache_sse4_2)/* rdi: 16-byte load will cross cache line */
259 movdqu (%rdi), %xmm1
260 movdqu (%rsi), %xmm2
261 # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
262 # define TOLOWER(reg1, reg2) \
263 movdqa reg1, %xmm7; \
264 movdqa UCHIGH_reg, %xmm8; \
265 movdqa reg2, %xmm9; \
266 movdqa UCHIGH_reg, %xmm10; \
267 pcmpgtb UCLOW_reg, %xmm7; \
268 pcmpgtb reg1, %xmm8; \
269 pcmpgtb UCLOW_reg, %xmm9; \
270 pcmpgtb reg2, %xmm10; \
271 pand %xmm8, %xmm7; \
272 pand %xmm10, %xmm9; \
273 pand LCQWORD_reg, %xmm7; \
274 pand LCQWORD_reg, %xmm9; \
275 por %xmm7, reg1; \
276 por %xmm9, reg2
277 TOLOWER (%xmm1, %xmm2)
278 # else
279 # define TOLOWER(reg1, reg2)
280 # endif
281 pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */
282 pcmpeqb %xmm1, %xmm0 /* Any null chars? */
283 pcmpeqb %xmm2, %xmm1 /* compare first 16 bytes for equality */
284 psubb %xmm0, %xmm1 /* packed sub of comparison results*/
285 pmovmskb %xmm1, %edx
286 sub $0xffff, %edx /* if first 16 bytes are same, edx == 0xffff */
287 jnz LABEL(less16bytes_sse4_2)/* If not, find different value or null char */
288 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
289 sub $16, %r11
290 jbe LABEL(strcmp_exitz_sse4_2)/* finish comparision */
291 # endif
292 add $16, %rsi /* prepare to search next 16 bytes */
293 add $16, %rdi /* prepare to search next 16 bytes */
295 /*
296 * Determine source and destination string offsets from 16-byte alignment.
297 * Use relative offset difference between the two to determine which case
298 * below to use.
299 */
300 .p2align 4
301 LABEL(crosscache_sse4_2):
302 and $0xfffffffffffffff0, %rsi /* force %rsi is 16 byte aligned */
303 and $0xfffffffffffffff0, %rdi /* force %rdi is 16 byte aligned */
304 mov $0xffff, %edx /* for equivalent offset */
305 xor %r8d, %r8d
306 and $0xf, %ecx /* offset of rsi */
307 and $0xf, %eax /* offset of rdi */
308 cmp %eax, %ecx
309 je LABEL(ashr_0_sse4_2) /* rsi and rdi relative offset same */
310 ja LABEL(bigger_sse4_2)
311 mov %edx, %r8d /* r8d is offset flag for exit tail */
312 xchg %ecx, %eax
313 xchg %rsi, %rdi
314 LABEL(bigger_sse4_2):
315 lea 15(%rax), %r9
316 sub %rcx, %r9
317 lea LABEL(unaligned_table_sse4_2)(%rip), %r10
318 movslq (%r10, %r9,4), %r9
319 lea (%r10, %r9), %r10
320 jmp *%r10 /* jump to corresponding case */
322 /*
323 * The following cases will be handled by ashr_0
324 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
325 * n(0~15) n(0~15) 15(15+ n-n) ashr_0
326 */
327 .p2align 4
328 LABEL(ashr_0_sse4_2):
330 movdqa (%rsi), %xmm1
331 pxor %xmm0, %xmm0 /* clear %xmm0 for null char check */
332 pcmpeqb %xmm1, %xmm0 /* Any null chars? */
333 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
334 pcmpeqb (%rdi), %xmm1 /* compare 16 bytes for equality */
335 # else
336 movdqa (%rdi), %xmm2
337 TOLOWER (%xmm1, %xmm2)
338 pcmpeqb %xmm2, %xmm1 /* compare 16 bytes for equality */
339 # endif
340 psubb %xmm0, %xmm1 /* packed sub of comparison results*/
341 pmovmskb %xmm1, %r9d
342 shr %cl, %edx /* adjust 0xffff for offset */
343 shr %cl, %r9d /* adjust for 16-byte offset */
344 sub %r9d, %edx
345 /*
346 * edx must be the same with r9d if in left byte (16-rcx) is equal to
347 * the start from (16-rax) and no null char was seen.
348 */
349 jne LABEL(less32bytes_sse4_2) /* mismatch or null char */
350 UPDATE_STRNCMP_COUNTER
351 mov $16, %rcx
352 mov $16, %r9
353 pxor %xmm0, %xmm0 /* clear xmm0, may have changed above */
355 /*
356 * Now both strings are aligned at 16-byte boundary. Loop over strings
357 * checking 32-bytes per iteration.
358 */
359 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
360 .p2align 4
361 LABEL(ashr_0_use_sse4_2):
362 movdqa (%rdi,%rdx), %xmm0
363 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
364 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
365 # else
366 movdqa (%rsi,%rdx), %xmm1
367 TOLOWER (%xmm0, %xmm1)
368 pcmpistri $0x1a, %xmm1, %xmm0
369 # endif
370 lea 16(%rdx), %rdx
371 jbe LABEL(ashr_0_use_sse4_2_exit)
372 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
373 sub $16, %r11
374 jbe LABEL(strcmp_exitz_sse4_2)
375 # endif
377 movdqa (%rdi,%rdx), %xmm0
378 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
379 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
380 # else
381 movdqa (%rsi,%rdx), %xmm1
382 TOLOWER (%xmm0, %xmm1)
383 pcmpistri $0x1a, %xmm1, %xmm0
384 # endif
385 lea 16(%rdx), %rdx
386 jbe LABEL(ashr_0_use_sse4_2_exit)
387 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
388 sub $16, %r11
389 jbe LABEL(strcmp_exitz_sse4_2)
390 # endif
391 jmp LABEL(ashr_0_use_sse4_2)
394 .p2align 4
395 LABEL(ashr_0_use_sse4_2_exit):
396 jnc LABEL(strcmp_exitz_sse4_2)
397 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
398 sub %rcx, %r11
399 jbe LABEL(strcmp_exitz_sse4_2)
400 # endif
401 lea -16(%rdx, %rcx), %rcx
402 movzbl (%rdi, %rcx), %eax
403 movzbl (%rsi, %rcx), %edx
404 # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
405 leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rcx
406 movl (%rcx,%rax,4), %eax
407 movl (%rcx,%rdx,4), %edx
408 # endif
409 sub %edx, %eax
410 ret
414 /*
415 * The following cases will be handled by ashr_1
416 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
417 * n(15) n -15 0(15 +(n-15) - n) ashr_1
418 */
419 .p2align 4
420 LABEL(ashr_1_sse4_2):
421 pxor %xmm0, %xmm0
422 movdqa (%rdi), %xmm2
423 movdqa (%rsi), %xmm1
424 pcmpeqb %xmm1, %xmm0 /* Any null chars? */
425 pslldq $15, %xmm2 /* shift first string to align with second */
426 TOLOWER (%xmm1, %xmm2)
427 pcmpeqb %xmm1, %xmm2 /* compare 16 bytes for equality */
428 psubb %xmm0, %xmm2 /* packed sub of comparison results*/
429 pmovmskb %xmm2, %r9d
430 shr %cl, %edx /* adjust 0xffff for offset */
431 shr %cl, %r9d /* adjust for 16-byte offset */
432 sub %r9d, %edx
433 jnz LABEL(less32bytes_sse4_2)/* mismatch or null char seen */
434 movdqa (%rdi), %xmm3
435 UPDATE_STRNCMP_COUNTER
437 pxor %xmm0, %xmm0
438 mov $16, %rcx /* index for loads*/
439 mov $1, %r9d /* byte position left over from less32bytes case */
440 /*
441 * Setup %r10 value allows us to detect crossing a page boundary.
442 * When %r10 goes positive we have crossed a page boundary and
443 * need to do a nibble.
444 */
445 lea 1(%rdi), %r10
446 and $0xfff, %r10 /* offset into 4K page */
447 sub $0x1000, %r10 /* subtract 4K pagesize */
448 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
450 .p2align 4
451 LABEL(loop_ashr_1_use_sse4_2):
452 add $16, %r10
453 jg LABEL(nibble_ashr_1_use_sse4_2)
455 LABEL(nibble_ashr_1_use_sse4_2_restart):
456 movdqa (%rdi, %rdx), %xmm0
457 palignr $1, -16(%rdi, %rdx), %xmm0
458 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
459 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
460 # else
461 movdqa (%rsi,%rdx), %xmm1
462 TOLOWER (%xmm0, %xmm1)
463 pcmpistri $0x1a, %xmm1, %xmm0
464 # endif
465 jbe LABEL(use_sse4_2_exit)
466 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
467 sub $16, %r11
468 jbe LABEL(strcmp_exitz_sse4_2)
469 # endif
471 add $16, %rdx
472 add $16, %r10
473 jg LABEL(nibble_ashr_1_use_sse4_2)
475 movdqa (%rdi, %rdx), %xmm0
476 palignr $1, -16(%rdi, %rdx), %xmm0
477 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
478 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
479 # else
480 movdqa (%rsi,%rdx), %xmm1
481 TOLOWER (%xmm0, %xmm1)
482 pcmpistri $0x1a, %xmm1, %xmm0
483 # endif
484 jbe LABEL(use_sse4_2_exit)
485 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
486 sub $16, %r11
487 jbe LABEL(strcmp_exitz_sse4_2)
488 # endif
489 add $16, %rdx
490 jmp LABEL(loop_ashr_1_use_sse4_2)
492 .p2align 4
493 LABEL(nibble_ashr_1_use_sse4_2):
494 sub $0x1000, %r10
495 movdqa -16(%rdi, %rdx), %xmm0
496 psrldq $1, %xmm0
497 pcmpistri $0x3a,%xmm0, %xmm0
498 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
499 cmp %r11, %rcx
500 jae LABEL(nibble_ashr_use_sse4_2_exit)
501 # endif
502 cmp $14, %ecx
503 ja LABEL(nibble_ashr_1_use_sse4_2_restart)
505 jmp LABEL(nibble_ashr_use_sse4_2_exit)
507 /*
508 * The following cases will be handled by ashr_2
509 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
510 * n(14~15) n -14 1(15 +(n-14) - n) ashr_2
511 */
512 .p2align 4
513 LABEL(ashr_2_sse4_2):
514 pxor %xmm0, %xmm0
515 movdqa (%rdi), %xmm2
516 movdqa (%rsi), %xmm1
517 pcmpeqb %xmm1, %xmm0
518 pslldq $14, %xmm2
519 TOLOWER (%xmm1, %xmm2)
520 pcmpeqb %xmm1, %xmm2
521 psubb %xmm0, %xmm2
522 pmovmskb %xmm2, %r9d
523 shr %cl, %edx
524 shr %cl, %r9d
525 sub %r9d, %edx
526 jnz LABEL(less32bytes_sse4_2)
527 movdqa (%rdi), %xmm3
528 UPDATE_STRNCMP_COUNTER
530 pxor %xmm0, %xmm0
531 mov $16, %rcx /* index for loads */
532 mov $2, %r9d /* byte position left over from less32bytes case */
533 /*
534 * Setup %r10 value allows us to detect crossing a page boundary.
535 * When %r10 goes positive we have crossed a page boundary and
536 * need to do a nibble.
537 */
538 lea 2(%rdi), %r10
539 and $0xfff, %r10 /* offset into 4K page */
540 sub $0x1000, %r10 /* subtract 4K pagesize */
541 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
543 .p2align 4
544 LABEL(loop_ashr_2_use_sse4_2):
545 add $16, %r10
546 jg LABEL(nibble_ashr_2_use_sse4_2)
548 LABEL(nibble_ashr_2_use_sse4_2_restart):
549 movdqa (%rdi, %rdx), %xmm0
550 palignr $2, -16(%rdi, %rdx), %xmm0
551 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
552 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
553 # else
554 movdqa (%rsi,%rdx), %xmm1
555 TOLOWER (%xmm0, %xmm1)
556 pcmpistri $0x1a, %xmm1, %xmm0
557 # endif
558 jbe LABEL(use_sse4_2_exit)
559 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
560 sub $16, %r11
561 jbe LABEL(strcmp_exitz_sse4_2)
562 # endif
564 add $16, %rdx
565 add $16, %r10
566 jg LABEL(nibble_ashr_2_use_sse4_2)
568 movdqa (%rdi, %rdx), %xmm0
569 palignr $2, -16(%rdi, %rdx), %xmm0
570 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
571 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
572 # else
573 movdqa (%rsi,%rdx), %xmm1
574 TOLOWER (%xmm0, %xmm1)
575 pcmpistri $0x1a, %xmm1, %xmm0
576 # endif
577 jbe LABEL(use_sse4_2_exit)
578 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
579 sub $16, %r11
580 jbe LABEL(strcmp_exitz_sse4_2)
581 # endif
582 add $16, %rdx
583 jmp LABEL(loop_ashr_2_use_sse4_2)
585 .p2align 4
586 LABEL(nibble_ashr_2_use_sse4_2):
587 sub $0x1000, %r10
588 movdqa -16(%rdi, %rdx), %xmm0
589 psrldq $2, %xmm0
590 pcmpistri $0x3a,%xmm0, %xmm0
591 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
592 cmp %r11, %rcx
593 jae LABEL(nibble_ashr_use_sse4_2_exit)
594 # endif
595 cmp $13, %ecx
596 ja LABEL(nibble_ashr_2_use_sse4_2_restart)
598 jmp LABEL(nibble_ashr_use_sse4_2_exit)
600 /*
601 * The following cases will be handled by ashr_3
602 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
603 * n(13~15) n -13 2(15 +(n-13) - n) ashr_3
604 */
605 .p2align 4
606 LABEL(ashr_3_sse4_2):
607 pxor %xmm0, %xmm0
608 movdqa (%rdi), %xmm2
609 movdqa (%rsi), %xmm1
610 pcmpeqb %xmm1, %xmm0
611 pslldq $13, %xmm2
612 TOLOWER (%xmm1, %xmm2)
613 pcmpeqb %xmm1, %xmm2
614 psubb %xmm0, %xmm2
615 pmovmskb %xmm2, %r9d
616 shr %cl, %edx
617 shr %cl, %r9d
618 sub %r9d, %edx
619 jnz LABEL(less32bytes_sse4_2)
620 movdqa (%rdi), %xmm3
622 UPDATE_STRNCMP_COUNTER
624 pxor %xmm0, %xmm0
625 mov $16, %rcx /* index for loads */
626 mov $3, %r9d /* byte position left over from less32bytes case */
627 /*
628 * Setup %r10 value allows us to detect crossing a page boundary.
629 * When %r10 goes positive we have crossed a page boundary and
630 * need to do a nibble.
631 */
632 lea 3(%rdi), %r10
633 and $0xfff, %r10 /* offset into 4K page */
634 sub $0x1000, %r10 /* subtract 4K pagesize */
635 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
637 LABEL(loop_ashr_3_use_sse4_2):
638 add $16, %r10
639 jg LABEL(nibble_ashr_3_use_sse4_2)
641 LABEL(nibble_ashr_3_use_sse4_2_restart):
642 movdqa (%rdi, %rdx), %xmm0
643 palignr $3, -16(%rdi, %rdx), %xmm0
644 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
645 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
646 # else
647 movdqa (%rsi,%rdx), %xmm1
648 TOLOWER (%xmm0, %xmm1)
649 pcmpistri $0x1a, %xmm1, %xmm0
650 # endif
651 jbe LABEL(use_sse4_2_exit)
652 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
653 sub $16, %r11
654 jbe LABEL(strcmp_exitz_sse4_2)
655 # endif
657 add $16, %rdx
658 add $16, %r10
659 jg LABEL(nibble_ashr_3_use_sse4_2)
661 movdqa (%rdi, %rdx), %xmm0
662 palignr $3, -16(%rdi, %rdx), %xmm0
663 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
664 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
665 # else
666 movdqa (%rsi,%rdx), %xmm1
667 TOLOWER (%xmm0, %xmm1)
668 pcmpistri $0x1a, %xmm1, %xmm0
669 # endif
670 jbe LABEL(use_sse4_2_exit)
671 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
672 sub $16, %r11
673 jbe LABEL(strcmp_exitz_sse4_2)
674 # endif
675 add $16, %rdx
676 jmp LABEL(loop_ashr_3_use_sse4_2)
678 .p2align 4
679 LABEL(nibble_ashr_3_use_sse4_2):
680 sub $0x1000, %r10
681 movdqa -16(%rdi, %rdx), %xmm0
682 psrldq $3, %xmm0
683 pcmpistri $0x3a,%xmm0, %xmm0
684 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
685 cmp %r11, %rcx
686 jae LABEL(nibble_ashr_use_sse4_2_exit)
687 # endif
688 cmp $12, %ecx
689 ja LABEL(nibble_ashr_3_use_sse4_2_restart)
691 jmp LABEL(nibble_ashr_use_sse4_2_exit)
693 /*
694 * The following cases will be handled by ashr_4
695 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
696 * n(12~15) n -12 3(15 +(n-12) - n) ashr_4
697 */
698 .p2align 4
699 LABEL(ashr_4_sse4_2):
700 pxor %xmm0, %xmm0
701 movdqa (%rdi), %xmm2
702 movdqa (%rsi), %xmm1
703 pcmpeqb %xmm1, %xmm0
704 pslldq $12, %xmm2
705 TOLOWER (%xmm1, %xmm2)
706 pcmpeqb %xmm1, %xmm2
707 psubb %xmm0, %xmm2
708 pmovmskb %xmm2, %r9d
709 shr %cl, %edx
710 shr %cl, %r9d
711 sub %r9d, %edx
712 jnz LABEL(less32bytes_sse4_2)
713 movdqa (%rdi), %xmm3
715 UPDATE_STRNCMP_COUNTER
717 pxor %xmm0, %xmm0
718 mov $16, %rcx /* index for loads */
719 mov $4, %r9d /* byte position left over from less32bytes case */
720 /*
721 * Setup %r10 value allows us to detect crossing a page boundary.
722 * When %r10 goes positive we have crossed a page boundary and
723 * need to do a nibble.
724 */
725 lea 4(%rdi), %r10
726 and $0xfff, %r10 /* offset into 4K page */
727 sub $0x1000, %r10 /* subtract 4K pagesize */
728 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
730 .p2align 4
731 LABEL(loop_ashr_4_use_sse4_2):
732 add $16, %r10
733 jg LABEL(nibble_ashr_4_use_sse4_2)
735 LABEL(nibble_ashr_4_use_sse4_2_restart):
736 movdqa (%rdi, %rdx), %xmm0
737 palignr $4, -16(%rdi, %rdx), %xmm0
738 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
739 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
740 # else
741 movdqa (%rsi,%rdx), %xmm1
742 TOLOWER (%xmm0, %xmm1)
743 pcmpistri $0x1a, %xmm1, %xmm0
744 # endif
745 jbe LABEL(use_sse4_2_exit)
746 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
747 sub $16, %r11
748 jbe LABEL(strcmp_exitz_sse4_2)
749 # endif
751 add $16, %rdx
752 add $16, %r10
753 jg LABEL(nibble_ashr_4_use_sse4_2)
755 movdqa (%rdi, %rdx), %xmm0
756 palignr $4, -16(%rdi, %rdx), %xmm0
757 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
758 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
759 # else
760 movdqa (%rsi,%rdx), %xmm1
761 TOLOWER (%xmm0, %xmm1)
762 pcmpistri $0x1a, %xmm1, %xmm0
763 # endif
764 jbe LABEL(use_sse4_2_exit)
765 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
766 sub $16, %r11
767 jbe LABEL(strcmp_exitz_sse4_2)
768 # endif
769 add $16, %rdx
770 jmp LABEL(loop_ashr_4_use_sse4_2)
772 .p2align 4
773 LABEL(nibble_ashr_4_use_sse4_2):
774 sub $0x1000, %r10
775 movdqa -16(%rdi, %rdx), %xmm0
776 psrldq $4, %xmm0
777 pcmpistri $0x3a,%xmm0, %xmm0
778 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
779 cmp %r11, %rcx
780 jae LABEL(nibble_ashr_use_sse4_2_exit)
781 # endif
782 cmp $11, %ecx
783 ja LABEL(nibble_ashr_4_use_sse4_2_restart)
785 jmp LABEL(nibble_ashr_use_sse4_2_exit)
787 /*
788 * The following cases will be handled by ashr_5
789 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
790 * n(11~15) n - 11 4(15 +(n-11) - n) ashr_5
791 */
792 .p2align 4
793 LABEL(ashr_5_sse4_2):
794 pxor %xmm0, %xmm0
795 movdqa (%rdi), %xmm2
796 movdqa (%rsi), %xmm1
797 pcmpeqb %xmm1, %xmm0
798 pslldq $11, %xmm2
799 TOLOWER (%xmm1, %xmm2)
800 pcmpeqb %xmm1, %xmm2
801 psubb %xmm0, %xmm2
802 pmovmskb %xmm2, %r9d
803 shr %cl, %edx
804 shr %cl, %r9d
805 sub %r9d, %edx
806 jnz LABEL(less32bytes_sse4_2)
807 movdqa (%rdi), %xmm3
809 UPDATE_STRNCMP_COUNTER
811 pxor %xmm0, %xmm0
812 mov $16, %rcx /* index for loads */
813 mov $5, %r9d /* byte position left over from less32bytes case */
814 /*
815 * Setup %r10 value allows us to detect crossing a page boundary.
816 * When %r10 goes positive we have crossed a page boundary and
817 * need to do a nibble.
818 */
819 lea 5(%rdi), %r10
820 and $0xfff, %r10 /* offset into 4K page */
821 sub $0x1000, %r10 /* subtract 4K pagesize */
822 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
824 .p2align 4
825 LABEL(loop_ashr_5_use_sse4_2):
826 add $16, %r10
827 jg LABEL(nibble_ashr_5_use_sse4_2)
829 LABEL(nibble_ashr_5_use_sse4_2_restart):
830 movdqa (%rdi, %rdx), %xmm0
831 palignr $5, -16(%rdi, %rdx), %xmm0
832 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
833 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
834 # else
835 movdqa (%rsi,%rdx), %xmm1
836 TOLOWER (%xmm0, %xmm1)
837 pcmpistri $0x1a, %xmm1, %xmm0
838 # endif
839 jbe LABEL(use_sse4_2_exit)
840 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
841 sub $16, %r11
842 jbe LABEL(strcmp_exitz_sse4_2)
843 # endif
845 add $16, %rdx
846 add $16, %r10
847 jg LABEL(nibble_ashr_5_use_sse4_2)
849 movdqa (%rdi, %rdx), %xmm0
851 palignr $5, -16(%rdi, %rdx), %xmm0
852 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
853 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
854 # else
855 movdqa (%rsi,%rdx), %xmm1
856 TOLOWER (%xmm0, %xmm1)
857 pcmpistri $0x1a, %xmm1, %xmm0
858 # endif
859 jbe LABEL(use_sse4_2_exit)
860 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
861 sub $16, %r11
862 jbe LABEL(strcmp_exitz_sse4_2)
863 # endif
864 add $16, %rdx
865 jmp LABEL(loop_ashr_5_use_sse4_2)
867 .p2align 4
868 LABEL(nibble_ashr_5_use_sse4_2):
869 sub $0x1000, %r10
870 movdqa -16(%rdi, %rdx), %xmm0
871 psrldq $5, %xmm0
872 pcmpistri $0x3a,%xmm0, %xmm0
873 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
874 cmp %r11, %rcx
875 jae LABEL(nibble_ashr_use_sse4_2_exit)
876 # endif
877 cmp $10, %ecx
878 ja LABEL(nibble_ashr_5_use_sse4_2_restart)
880 jmp LABEL(nibble_ashr_use_sse4_2_exit)
882 /*
883 * The following cases will be handled by ashr_6
884 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
885 * n(10~15) n - 10 5(15 +(n-10) - n) ashr_6
886 */
887 .p2align 4
888 LABEL(ashr_6_sse4_2):
889 pxor %xmm0, %xmm0
890 movdqa (%rdi), %xmm2
891 movdqa (%rsi), %xmm1
892 pcmpeqb %xmm1, %xmm0
893 pslldq $10, %xmm2
894 TOLOWER (%xmm1, %xmm2)
895 pcmpeqb %xmm1, %xmm2
896 psubb %xmm0, %xmm2
897 pmovmskb %xmm2, %r9d
898 shr %cl, %edx
899 shr %cl, %r9d
900 sub %r9d, %edx
901 jnz LABEL(less32bytes_sse4_2)
902 movdqa (%rdi), %xmm3
904 UPDATE_STRNCMP_COUNTER
906 pxor %xmm0, %xmm0
907 mov $16, %rcx /* index for loads */
908 mov $6, %r9d /* byte position left over from less32bytes case */
909 /*
910 * Setup %r10 value allows us to detect crossing a page boundary.
911 * When %r10 goes positive we have crossed a page boundary and
912 * need to do a nibble.
913 */
914 lea 6(%rdi), %r10
915 and $0xfff, %r10 /* offset into 4K page */
916 sub $0x1000, %r10 /* subtract 4K pagesize */
917 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
919 .p2align 4
920 LABEL(loop_ashr_6_use_sse4_2):
921 add $16, %r10
922 jg LABEL(nibble_ashr_6_use_sse4_2)
924 LABEL(nibble_ashr_6_use_sse4_2_restart):
925 movdqa (%rdi, %rdx), %xmm0
926 palignr $6, -16(%rdi, %rdx), %xmm0
927 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
928 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
929 # else
930 movdqa (%rsi,%rdx), %xmm1
931 TOLOWER (%xmm0, %xmm1)
932 pcmpistri $0x1a, %xmm1, %xmm0
933 # endif
934 jbe LABEL(use_sse4_2_exit)
935 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
936 sub $16, %r11
937 jbe LABEL(strcmp_exitz_sse4_2)
938 # endif
940 add $16, %rdx
941 add $16, %r10
942 jg LABEL(nibble_ashr_6_use_sse4_2)
944 movdqa (%rdi, %rdx), %xmm0
945 palignr $6, -16(%rdi, %rdx), %xmm0
946 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
947 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
948 # else
949 movdqa (%rsi,%rdx), %xmm1
950 TOLOWER (%xmm0, %xmm1)
951 pcmpistri $0x1a, %xmm1, %xmm0
952 # endif
953 jbe LABEL(use_sse4_2_exit)
954 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
955 sub $16, %r11
956 jbe LABEL(strcmp_exitz_sse4_2)
957 # endif
958 add $16, %rdx
959 jmp LABEL(loop_ashr_6_use_sse4_2)
961 .p2align 4
962 LABEL(nibble_ashr_6_use_sse4_2):
963 sub $0x1000, %r10
964 movdqa -16(%rdi, %rdx), %xmm0
965 psrldq $6, %xmm0
966 pcmpistri $0x3a,%xmm0, %xmm0
967 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
968 cmp %r11, %rcx
969 jae LABEL(nibble_ashr_use_sse4_2_exit)
970 # endif
971 cmp $9, %ecx
972 ja LABEL(nibble_ashr_6_use_sse4_2_restart)
974 jmp LABEL(nibble_ashr_use_sse4_2_exit)
976 /*
977 * The following cases will be handled by ashr_7
978 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
979 * n(9~15) n - 9 6(15 +(n - 9) - n) ashr_7
980 */
981 .p2align 4
982 LABEL(ashr_7_sse4_2):
983 pxor %xmm0, %xmm0
984 movdqa (%rdi), %xmm2
985 movdqa (%rsi), %xmm1
986 pcmpeqb %xmm1, %xmm0
987 pslldq $9, %xmm2
988 TOLOWER (%xmm1, %xmm2)
989 pcmpeqb %xmm1, %xmm2
990 psubb %xmm0, %xmm2
991 pmovmskb %xmm2, %r9d
992 shr %cl, %edx
993 shr %cl, %r9d
994 sub %r9d, %edx
995 jnz LABEL(less32bytes_sse4_2)
996 movdqa (%rdi), %xmm3
998 UPDATE_STRNCMP_COUNTER
1000 pxor %xmm0, %xmm0
1001 mov $16, %rcx /* index for loads */
1002 mov $7, %r9d /* byte position left over from less32bytes case */
1003 /*
1004 * Setup %r10 value allows us to detect crossing a page boundary.
1005 * When %r10 goes positive we have crossed a page boundary and
1006 * need to do a nibble.
1007 */
1008 lea 7(%rdi), %r10
1009 and $0xfff, %r10 /* offset into 4K page */
1010 sub $0x1000, %r10 /* subtract 4K pagesize */
1011 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1013 .p2align 4
1014 LABEL(loop_ashr_7_use_sse4_2):
1015 add $16, %r10
1016 jg LABEL(nibble_ashr_7_use_sse4_2)
1018 LABEL(nibble_ashr_7_use_sse4_2_restart):
1019 movdqa (%rdi, %rdx), %xmm0
1020 palignr $7, -16(%rdi, %rdx), %xmm0
1021 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1022 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1023 # else
1024 movdqa (%rsi,%rdx), %xmm1
1025 TOLOWER (%xmm0, %xmm1)
1026 pcmpistri $0x1a, %xmm1, %xmm0
1027 # endif
1028 jbe LABEL(use_sse4_2_exit)
1029 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1030 sub $16, %r11
1031 jbe LABEL(strcmp_exitz_sse4_2)
1032 # endif
1034 add $16, %rdx
1035 add $16, %r10
1036 jg LABEL(nibble_ashr_7_use_sse4_2)
1038 movdqa (%rdi, %rdx), %xmm0
1039 palignr $7, -16(%rdi, %rdx), %xmm0
1040 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1041 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1042 # else
1043 movdqa (%rsi,%rdx), %xmm1
1044 TOLOWER (%xmm0, %xmm1)
1045 pcmpistri $0x1a, %xmm1, %xmm0
1046 # endif
1047 jbe LABEL(use_sse4_2_exit)
1048 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1049 sub $16, %r11
1050 jbe LABEL(strcmp_exitz_sse4_2)
1051 # endif
1052 add $16, %rdx
1053 jmp LABEL(loop_ashr_7_use_sse4_2)
1055 .p2align 4
1056 LABEL(nibble_ashr_7_use_sse4_2):
1057 sub $0x1000, %r10
1058 movdqa -16(%rdi, %rdx), %xmm0
1059 psrldq $7, %xmm0
1060 pcmpistri $0x3a,%xmm0, %xmm0
1061 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1062 cmp %r11, %rcx
1063 jae LABEL(nibble_ashr_use_sse4_2_exit)
1064 # endif
1065 cmp $8, %ecx
1066 ja LABEL(nibble_ashr_7_use_sse4_2_restart)
1068 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1070 /*
1071 * The following cases will be handled by ashr_8
1072 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1073 * n(8~15) n - 8 7(15 +(n - 8) - n) ashr_8
1074 */
1075 .p2align 4
1076 LABEL(ashr_8_sse4_2):
1077 pxor %xmm0, %xmm0
1078 movdqa (%rdi), %xmm2
1079 movdqa (%rsi), %xmm1
1080 pcmpeqb %xmm1, %xmm0
1081 pslldq $8, %xmm2
1082 TOLOWER (%xmm1, %xmm2)
1083 pcmpeqb %xmm1, %xmm2
1084 psubb %xmm0, %xmm2
1085 pmovmskb %xmm2, %r9d
1086 shr %cl, %edx
1087 shr %cl, %r9d
1088 sub %r9d, %edx
1089 jnz LABEL(less32bytes_sse4_2)
1090 movdqa (%rdi), %xmm3
1092 UPDATE_STRNCMP_COUNTER
1094 pxor %xmm0, %xmm0
1095 mov $16, %rcx /* index for loads */
1096 mov $8, %r9d /* byte position left over from less32bytes case */
1097 /*
1098 * Setup %r10 value allows us to detect crossing a page boundary.
1099 * When %r10 goes positive we have crossed a page boundary and
1100 * need to do a nibble.
1101 */
1102 lea 8(%rdi), %r10
1103 and $0xfff, %r10 /* offset into 4K page */
1104 sub $0x1000, %r10 /* subtract 4K pagesize */
1105 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1107 .p2align 4
1108 LABEL(loop_ashr_8_use_sse4_2):
1109 add $16, %r10
1110 jg LABEL(nibble_ashr_8_use_sse4_2)
1112 LABEL(nibble_ashr_8_use_sse4_2_restart):
1113 movdqa (%rdi, %rdx), %xmm0
1114 palignr $8, -16(%rdi, %rdx), %xmm0
1115 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1116 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1117 # else
1118 movdqa (%rsi,%rdx), %xmm1
1119 TOLOWER (%xmm0, %xmm1)
1120 pcmpistri $0x1a, %xmm1, %xmm0
1121 # endif
1122 jbe LABEL(use_sse4_2_exit)
1123 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1124 sub $16, %r11
1125 jbe LABEL(strcmp_exitz_sse4_2)
1126 # endif
1128 add $16, %rdx
1129 add $16, %r10
1130 jg LABEL(nibble_ashr_8_use_sse4_2)
1132 movdqa (%rdi, %rdx), %xmm0
1133 palignr $8, -16(%rdi, %rdx), %xmm0
1134 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1135 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1136 # else
1137 movdqa (%rsi,%rdx), %xmm1
1138 TOLOWER (%xmm0, %xmm1)
1139 pcmpistri $0x1a, %xmm1, %xmm0
1140 # endif
1141 jbe LABEL(use_sse4_2_exit)
1142 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1143 sub $16, %r11
1144 jbe LABEL(strcmp_exitz_sse4_2)
1145 # endif
1146 add $16, %rdx
1147 jmp LABEL(loop_ashr_8_use_sse4_2)
1149 .p2align 4
1150 LABEL(nibble_ashr_8_use_sse4_2):
1151 sub $0x1000, %r10
1152 movdqa -16(%rdi, %rdx), %xmm0
1153 psrldq $8, %xmm0
1154 pcmpistri $0x3a,%xmm0, %xmm0
1155 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1156 cmp %r11, %rcx
1157 jae LABEL(nibble_ashr_use_sse4_2_exit)
1158 # endif
1159 cmp $7, %ecx
1160 ja LABEL(nibble_ashr_8_use_sse4_2_restart)
1162 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1164 /*
1165 * The following cases will be handled by ashr_9
1166 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1167 * n(7~15) n - 7 8(15 +(n - 7) - n) ashr_9
1168 */
1169 .p2align 4
1170 LABEL(ashr_9_sse4_2):
1171 pxor %xmm0, %xmm0
1172 movdqa (%rdi), %xmm2
1173 movdqa (%rsi), %xmm1
1174 pcmpeqb %xmm1, %xmm0
1175 pslldq $7, %xmm2
1176 TOLOWER (%xmm1, %xmm2)
1177 pcmpeqb %xmm1, %xmm2
1178 psubb %xmm0, %xmm2
1179 pmovmskb %xmm2, %r9d
1180 shr %cl, %edx
1181 shr %cl, %r9d
1182 sub %r9d, %edx
1183 jnz LABEL(less32bytes_sse4_2)
1184 movdqa (%rdi), %xmm3
1186 UPDATE_STRNCMP_COUNTER
1188 pxor %xmm0, %xmm0
1189 mov $16, %rcx /* index for loads */
1190 mov $9, %r9d /* byte position left over from less32bytes case */
1191 /*
1192 * Setup %r10 value allows us to detect crossing a page boundary.
1193 * When %r10 goes positive we have crossed a page boundary and
1194 * need to do a nibble.
1195 */
1196 lea 9(%rdi), %r10
1197 and $0xfff, %r10 /* offset into 4K page */
1198 sub $0x1000, %r10 /* subtract 4K pagesize */
1199 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1201 .p2align 4
1202 LABEL(loop_ashr_9_use_sse4_2):
1203 add $16, %r10
1204 jg LABEL(nibble_ashr_9_use_sse4_2)
1206 LABEL(nibble_ashr_9_use_sse4_2_restart):
1207 movdqa (%rdi, %rdx), %xmm0
1209 palignr $9, -16(%rdi, %rdx), %xmm0
1210 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1211 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1212 # else
1213 movdqa (%rsi,%rdx), %xmm1
1214 TOLOWER (%xmm0, %xmm1)
1215 pcmpistri $0x1a, %xmm1, %xmm0
1216 # endif
1217 jbe LABEL(use_sse4_2_exit)
1218 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1219 sub $16, %r11
1220 jbe LABEL(strcmp_exitz_sse4_2)
1221 # endif
1223 add $16, %rdx
1224 add $16, %r10
1225 jg LABEL(nibble_ashr_9_use_sse4_2)
1227 movdqa (%rdi, %rdx), %xmm0
1228 palignr $9, -16(%rdi, %rdx), %xmm0
1229 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1230 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1231 # else
1232 movdqa (%rsi,%rdx), %xmm1
1233 TOLOWER (%xmm0, %xmm1)
1234 pcmpistri $0x1a, %xmm1, %xmm0
1235 # endif
1236 jbe LABEL(use_sse4_2_exit)
1237 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1238 sub $16, %r11
1239 jbe LABEL(strcmp_exitz_sse4_2)
1240 # endif
1241 add $16, %rdx
1242 jmp LABEL(loop_ashr_9_use_sse4_2)
1244 .p2align 4
1245 LABEL(nibble_ashr_9_use_sse4_2):
1246 sub $0x1000, %r10
1247 movdqa -16(%rdi, %rdx), %xmm0
1248 psrldq $9, %xmm0
1249 pcmpistri $0x3a,%xmm0, %xmm0
1250 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1251 cmp %r11, %rcx
1252 jae LABEL(nibble_ashr_use_sse4_2_exit)
1253 # endif
1254 cmp $6, %ecx
1255 ja LABEL(nibble_ashr_9_use_sse4_2_restart)
1257 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1259 /*
1260 * The following cases will be handled by ashr_10
1261 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1262 * n(6~15) n - 6 9(15 +(n - 6) - n) ashr_10
1263 */
1264 .p2align 4
1265 LABEL(ashr_10_sse4_2):
1266 pxor %xmm0, %xmm0
1267 movdqa (%rdi), %xmm2
1268 movdqa (%rsi), %xmm1
1269 pcmpeqb %xmm1, %xmm0
1270 pslldq $6, %xmm2
1271 TOLOWER (%xmm1, %xmm2)
1272 pcmpeqb %xmm1, %xmm2
1273 psubb %xmm0, %xmm2
1274 pmovmskb %xmm2, %r9d
1275 shr %cl, %edx
1276 shr %cl, %r9d
1277 sub %r9d, %edx
1278 jnz LABEL(less32bytes_sse4_2)
1279 movdqa (%rdi), %xmm3
1281 UPDATE_STRNCMP_COUNTER
1283 pxor %xmm0, %xmm0
1284 mov $16, %rcx /* index for loads */
1285 mov $10, %r9d /* byte position left over from less32bytes case */
1286 /*
1287 * Setup %r10 value allows us to detect crossing a page boundary.
1288 * When %r10 goes positive we have crossed a page boundary and
1289 * need to do a nibble.
1290 */
1291 lea 10(%rdi), %r10
1292 and $0xfff, %r10 /* offset into 4K page */
1293 sub $0x1000, %r10 /* subtract 4K pagesize */
1294 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1296 .p2align 4
1297 LABEL(loop_ashr_10_use_sse4_2):
1298 add $16, %r10
1299 jg LABEL(nibble_ashr_10_use_sse4_2)
1301 LABEL(nibble_ashr_10_use_sse4_2_restart):
1302 movdqa (%rdi, %rdx), %xmm0
1303 palignr $10, -16(%rdi, %rdx), %xmm0
1304 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1305 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1306 # else
1307 movdqa (%rsi,%rdx), %xmm1
1308 TOLOWER (%xmm0, %xmm1)
1309 pcmpistri $0x1a, %xmm1, %xmm0
1310 # endif
1311 jbe LABEL(use_sse4_2_exit)
1312 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1313 sub $16, %r11
1314 jbe LABEL(strcmp_exitz_sse4_2)
1315 # endif
1317 add $16, %rdx
1318 add $16, %r10
1319 jg LABEL(nibble_ashr_10_use_sse4_2)
1321 movdqa (%rdi, %rdx), %xmm0
1322 palignr $10, -16(%rdi, %rdx), %xmm0
1323 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1324 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1325 # else
1326 movdqa (%rsi,%rdx), %xmm1
1327 TOLOWER (%xmm0, %xmm1)
1328 pcmpistri $0x1a, %xmm1, %xmm0
1329 # endif
1330 jbe LABEL(use_sse4_2_exit)
1331 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1332 sub $16, %r11
1333 jbe LABEL(strcmp_exitz_sse4_2)
1334 # endif
1335 add $16, %rdx
1336 jmp LABEL(loop_ashr_10_use_sse4_2)
1338 .p2align 4
1339 LABEL(nibble_ashr_10_use_sse4_2):
1340 sub $0x1000, %r10
1341 movdqa -16(%rdi, %rdx), %xmm0
1342 psrldq $10, %xmm0
1343 pcmpistri $0x3a,%xmm0, %xmm0
1344 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1345 cmp %r11, %rcx
1346 jae LABEL(nibble_ashr_use_sse4_2_exit)
1347 # endif
1348 cmp $5, %ecx
1349 ja LABEL(nibble_ashr_10_use_sse4_2_restart)
1351 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1353 /*
1354 * The following cases will be handled by ashr_11
1355 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1356 * n(5~15) n - 5 10(15 +(n - 5) - n) ashr_11
1357 */
1358 .p2align 4
1359 LABEL(ashr_11_sse4_2):
1360 pxor %xmm0, %xmm0
1361 movdqa (%rdi), %xmm2
1362 movdqa (%rsi), %xmm1
1363 pcmpeqb %xmm1, %xmm0
1364 pslldq $5, %xmm2
1365 TOLOWER (%xmm1, %xmm2)
1366 pcmpeqb %xmm1, %xmm2
1367 psubb %xmm0, %xmm2
1368 pmovmskb %xmm2, %r9d
1369 shr %cl, %edx
1370 shr %cl, %r9d
1371 sub %r9d, %edx
1372 jnz LABEL(less32bytes_sse4_2)
1373 movdqa (%rdi), %xmm3
1375 UPDATE_STRNCMP_COUNTER
1377 pxor %xmm0, %xmm0
1378 mov $16, %rcx /* index for loads */
1379 mov $11, %r9d /* byte position left over from less32bytes case */
1380 /*
1381 * Setup %r10 value allows us to detect crossing a page boundary.
1382 * When %r10 goes positive we have crossed a page boundary and
1383 * need to do a nibble.
1384 */
1385 lea 11(%rdi), %r10
1386 and $0xfff, %r10 /* offset into 4K page */
1387 sub $0x1000, %r10 /* subtract 4K pagesize */
1388 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1390 .p2align 4
1391 LABEL(loop_ashr_11_use_sse4_2):
1392 add $16, %r10
1393 jg LABEL(nibble_ashr_11_use_sse4_2)
1395 LABEL(nibble_ashr_11_use_sse4_2_restart):
1396 movdqa (%rdi, %rdx), %xmm0
1397 palignr $11, -16(%rdi, %rdx), %xmm0
1398 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1399 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1400 # else
1401 movdqa (%rsi,%rdx), %xmm1
1402 TOLOWER (%xmm0, %xmm1)
1403 pcmpistri $0x1a, %xmm1, %xmm0
1404 # endif
1405 jbe LABEL(use_sse4_2_exit)
1406 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1407 sub $16, %r11
1408 jbe LABEL(strcmp_exitz_sse4_2)
1409 # endif
1411 add $16, %rdx
1412 add $16, %r10
1413 jg LABEL(nibble_ashr_11_use_sse4_2)
1415 movdqa (%rdi, %rdx), %xmm0
1416 palignr $11, -16(%rdi, %rdx), %xmm0
1417 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1418 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1419 # else
1420 movdqa (%rsi,%rdx), %xmm1
1421 TOLOWER (%xmm0, %xmm1)
1422 pcmpistri $0x1a, %xmm1, %xmm0
1423 # endif
1424 jbe LABEL(use_sse4_2_exit)
1425 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1426 sub $16, %r11
1427 jbe LABEL(strcmp_exitz_sse4_2)
1428 # endif
1429 add $16, %rdx
1430 jmp LABEL(loop_ashr_11_use_sse4_2)
1432 .p2align 4
1433 LABEL(nibble_ashr_11_use_sse4_2):
1434 sub $0x1000, %r10
1435 movdqa -16(%rdi, %rdx), %xmm0
1436 psrldq $11, %xmm0
1437 pcmpistri $0x3a,%xmm0, %xmm0
1438 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1439 cmp %r11, %rcx
1440 jae LABEL(nibble_ashr_use_sse4_2_exit)
1441 # endif
1442 cmp $4, %ecx
1443 ja LABEL(nibble_ashr_11_use_sse4_2_restart)
1445 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1447 /*
1448 * The following cases will be handled by ashr_12
1449 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1450 * n(4~15) n - 4 11(15 +(n - 4) - n) ashr_12
1451 */
1452 .p2align 4
1453 LABEL(ashr_12_sse4_2):
1454 pxor %xmm0, %xmm0
1455 movdqa (%rdi), %xmm2
1456 movdqa (%rsi), %xmm1
1457 pcmpeqb %xmm1, %xmm0
1458 pslldq $4, %xmm2
1459 TOLOWER (%xmm1, %xmm2)
1460 pcmpeqb %xmm1, %xmm2
1461 psubb %xmm0, %xmm2
1462 pmovmskb %xmm2, %r9d
1463 shr %cl, %edx
1464 shr %cl, %r9d
1465 sub %r9d, %edx
1466 jnz LABEL(less32bytes_sse4_2)
1467 movdqa (%rdi), %xmm3
1469 UPDATE_STRNCMP_COUNTER
1471 pxor %xmm0, %xmm0
1472 mov $16, %rcx /* index for loads */
1473 mov $12, %r9d /* byte position left over from less32bytes case */
1474 /*
1475 * Setup %r10 value allows us to detect crossing a page boundary.
1476 * When %r10 goes positive we have crossed a page boundary and
1477 * need to do a nibble.
1478 */
1479 lea 12(%rdi), %r10
1480 and $0xfff, %r10 /* offset into 4K page */
1481 sub $0x1000, %r10 /* subtract 4K pagesize */
1482 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1484 .p2align 4
1485 LABEL(loop_ashr_12_use_sse4_2):
1486 add $16, %r10
1487 jg LABEL(nibble_ashr_12_use_sse4_2)
1489 LABEL(nibble_ashr_12_use_sse4_2_restart):
1490 movdqa (%rdi, %rdx), %xmm0
1491 palignr $12, -16(%rdi, %rdx), %xmm0
1492 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1493 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1494 # else
1495 movdqa (%rsi,%rdx), %xmm1
1496 TOLOWER (%xmm0, %xmm1)
1497 pcmpistri $0x1a, %xmm1, %xmm0
1498 # endif
1499 jbe LABEL(use_sse4_2_exit)
1500 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1501 sub $16, %r11
1502 jbe LABEL(strcmp_exitz_sse4_2)
1503 # endif
1505 add $16, %rdx
1506 add $16, %r10
1507 jg LABEL(nibble_ashr_12_use_sse4_2)
1509 movdqa (%rdi, %rdx), %xmm0
1510 palignr $12, -16(%rdi, %rdx), %xmm0
1511 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1512 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1513 # else
1514 movdqa (%rsi,%rdx), %xmm1
1515 TOLOWER (%xmm0, %xmm1)
1516 pcmpistri $0x1a, %xmm1, %xmm0
1517 # endif
1518 jbe LABEL(use_sse4_2_exit)
1519 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1520 sub $16, %r11
1521 jbe LABEL(strcmp_exitz_sse4_2)
1522 # endif
1523 add $16, %rdx
1524 jmp LABEL(loop_ashr_12_use_sse4_2)
1526 .p2align 4
1527 LABEL(nibble_ashr_12_use_sse4_2):
1528 sub $0x1000, %r10
1529 movdqa -16(%rdi, %rdx), %xmm0
1530 psrldq $12, %xmm0
1531 pcmpistri $0x3a,%xmm0, %xmm0
1532 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1533 cmp %r11, %rcx
1534 jae LABEL(nibble_ashr_use_sse4_2_exit)
1535 # endif
1536 cmp $3, %ecx
1537 ja LABEL(nibble_ashr_12_use_sse4_2_restart)
1539 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1541 /*
1542 * The following cases will be handled by ashr_13
1543 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1544 * n(3~15) n - 3 12(15 +(n - 3) - n) ashr_13
1545 */
1546 .p2align 4
1547 LABEL(ashr_13_sse4_2):
1548 pxor %xmm0, %xmm0
1549 movdqa (%rdi), %xmm2
1550 movdqa (%rsi), %xmm1
1551 pcmpeqb %xmm1, %xmm0
1552 pslldq $3, %xmm2
1553 TOLOWER (%xmm1, %xmm2)
1554 pcmpeqb %xmm1, %xmm2
1555 psubb %xmm0, %xmm2
1556 pmovmskb %xmm2, %r9d
1557 shr %cl, %edx
1558 shr %cl, %r9d
1559 sub %r9d, %edx
1560 jnz LABEL(less32bytes_sse4_2)
1561 movdqa (%rdi), %xmm3
1563 UPDATE_STRNCMP_COUNTER
1565 pxor %xmm0, %xmm0
1566 mov $16, %rcx /* index for loads */
1567 mov $13, %r9d /* byte position left over from less32bytes case */
1568 /*
1569 * Setup %r10 value allows us to detect crossing a page boundary.
1570 * When %r10 goes positive we have crossed a page boundary and
1571 * need to do a nibble.
1572 */
1573 lea 13(%rdi), %r10
1574 and $0xfff, %r10 /* offset into 4K page */
1575 sub $0x1000, %r10 /* subtract 4K pagesize */
1577 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1579 .p2align 4
1580 LABEL(loop_ashr_13_use_sse4_2):
1581 add $16, %r10
1582 jg LABEL(nibble_ashr_13_use_sse4_2)
1584 LABEL(nibble_ashr_13_use_sse4_2_restart):
1585 movdqa (%rdi, %rdx), %xmm0
1586 palignr $13, -16(%rdi, %rdx), %xmm0
1587 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1588 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1589 # else
1590 movdqa (%rsi,%rdx), %xmm1
1591 TOLOWER (%xmm0, %xmm1)
1592 pcmpistri $0x1a, %xmm1, %xmm0
1593 # endif
1594 jbe LABEL(use_sse4_2_exit)
1595 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1596 sub $16, %r11
1597 jbe LABEL(strcmp_exitz_sse4_2)
1598 # endif
1600 add $16, %rdx
1601 add $16, %r10
1602 jg LABEL(nibble_ashr_13_use_sse4_2)
1604 movdqa (%rdi, %rdx), %xmm0
1605 palignr $13, -16(%rdi, %rdx), %xmm0
1606 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1607 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1608 # else
1609 movdqa (%rsi,%rdx), %xmm1
1610 TOLOWER (%xmm0, %xmm1)
1611 pcmpistri $0x1a, %xmm1, %xmm0
1612 # endif
1613 jbe LABEL(use_sse4_2_exit)
1614 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1615 sub $16, %r11
1616 jbe LABEL(strcmp_exitz_sse4_2)
1617 # endif
1618 add $16, %rdx
1619 jmp LABEL(loop_ashr_13_use_sse4_2)
1621 .p2align 4
1622 LABEL(nibble_ashr_13_use_sse4_2):
1623 sub $0x1000, %r10
1624 movdqa -16(%rdi, %rdx), %xmm0
1625 psrldq $13, %xmm0
1626 pcmpistri $0x3a,%xmm0, %xmm0
1627 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1628 cmp %r11, %rcx
1629 jae LABEL(nibble_ashr_use_sse4_2_exit)
1630 # endif
1631 cmp $2, %ecx
1632 ja LABEL(nibble_ashr_13_use_sse4_2_restart)
1634 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1636 /*
1637 * The following cases will be handled by ashr_14
1638 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1639 * n(2~15) n - 2 13(15 +(n - 2) - n) ashr_14
1640 */
1641 .p2align 4
1642 LABEL(ashr_14_sse4_2):
1643 pxor %xmm0, %xmm0
1644 movdqa (%rdi), %xmm2
1645 movdqa (%rsi), %xmm1
1646 pcmpeqb %xmm1, %xmm0
1647 pslldq $2, %xmm2
1648 TOLOWER (%xmm1, %xmm2)
1649 pcmpeqb %xmm1, %xmm2
1650 psubb %xmm0, %xmm2
1651 pmovmskb %xmm2, %r9d
1652 shr %cl, %edx
1653 shr %cl, %r9d
1654 sub %r9d, %edx
1655 jnz LABEL(less32bytes_sse4_2)
1656 movdqa (%rdi), %xmm3
1658 UPDATE_STRNCMP_COUNTER
1660 pxor %xmm0, %xmm0
1661 mov $16, %rcx /* index for loads */
1662 mov $14, %r9d /* byte position left over from less32bytes case */
1663 /*
1664 * Setup %r10 value allows us to detect crossing a page boundary.
1665 * When %r10 goes positive we have crossed a page boundary and
1666 * need to do a nibble.
1667 */
1668 lea 14(%rdi), %r10
1669 and $0xfff, %r10 /* offset into 4K page */
1670 sub $0x1000, %r10 /* subtract 4K pagesize */
1672 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1674 .p2align 4
1675 LABEL(loop_ashr_14_use_sse4_2):
1676 add $16, %r10
1677 jg LABEL(nibble_ashr_14_use_sse4_2)
1679 LABEL(nibble_ashr_14_use_sse4_2_restart):
1680 movdqa (%rdi, %rdx), %xmm0
1681 palignr $14, -16(%rdi, %rdx), %xmm0
1682 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1683 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1684 # else
1685 movdqa (%rsi,%rdx), %xmm1
1686 TOLOWER (%xmm0, %xmm1)
1687 pcmpistri $0x1a, %xmm1, %xmm0
1688 # endif
1689 jbe LABEL(use_sse4_2_exit)
1690 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1691 sub $16, %r11
1692 jbe LABEL(strcmp_exitz_sse4_2)
1693 # endif
1695 add $16, %rdx
1696 add $16, %r10
1697 jg LABEL(nibble_ashr_14_use_sse4_2)
1699 movdqa (%rdi, %rdx), %xmm0
1700 palignr $14, -16(%rdi, %rdx), %xmm0
1701 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1702 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1703 # else
1704 movdqa (%rsi,%rdx), %xmm1
1705 TOLOWER (%xmm0, %xmm1)
1706 pcmpistri $0x1a, %xmm1, %xmm0
1707 # endif
1708 jbe LABEL(use_sse4_2_exit)
1709 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1710 sub $16, %r11
1711 jbe LABEL(strcmp_exitz_sse4_2)
1712 # endif
1713 add $16, %rdx
1714 jmp LABEL(loop_ashr_14_use_sse4_2)
1716 .p2align 4
1717 LABEL(nibble_ashr_14_use_sse4_2):
1718 sub $0x1000, %r10
1719 movdqa -16(%rdi, %rdx), %xmm0
1720 psrldq $14, %xmm0
1721 pcmpistri $0x3a,%xmm0, %xmm0
1722 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1723 cmp %r11, %rcx
1724 jae LABEL(nibble_ashr_use_sse4_2_exit)
1725 # endif
1726 cmp $1, %ecx
1727 ja LABEL(nibble_ashr_14_use_sse4_2_restart)
1729 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1731 /*
1732 * The following cases will be handled by ashr_15
1733 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1734 * n(1~15) n - 1 14(15 +(n - 1) - n) ashr_15
1735 */
1736 .p2align 4
1737 LABEL(ashr_15_sse4_2):
1738 pxor %xmm0, %xmm0
1739 movdqa (%rdi), %xmm2
1740 movdqa (%rsi), %xmm1
1741 pcmpeqb %xmm1, %xmm0
1742 pslldq $1, %xmm2
1743 TOLOWER (%xmm1, %xmm2)
1744 pcmpeqb %xmm1, %xmm2
1745 psubb %xmm0, %xmm2
1746 pmovmskb %xmm2, %r9d
1747 shr %cl, %edx
1748 shr %cl, %r9d
1749 sub %r9d, %edx
1750 jnz LABEL(less32bytes_sse4_2)
1752 movdqa (%rdi), %xmm3
1754 UPDATE_STRNCMP_COUNTER
1756 pxor %xmm0, %xmm0
1757 mov $16, %rcx /* index for loads */
1758 mov $15, %r9d /* byte position left over from less32bytes case */
1759 /*
1760 * Setup %r10 value allows us to detect crossing a page boundary.
1761 * When %r10 goes positive we have crossed a page boundary and
1762 * need to do a nibble.
1763 */
1764 lea 15(%rdi), %r10
1765 and $0xfff, %r10 /* offset into 4K page */
1767 sub $0x1000, %r10 /* subtract 4K pagesize */
1769 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1771 .p2align 4
1772 LABEL(loop_ashr_15_use_sse4_2):
1773 add $16, %r10
1774 jg LABEL(nibble_ashr_15_use_sse4_2)
1776 LABEL(nibble_ashr_15_use_sse4_2_restart):
1777 movdqa (%rdi, %rdx), %xmm0
1778 palignr $15, -16(%rdi, %rdx), %xmm0
1779 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1780 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1781 # else
1782 movdqa (%rsi,%rdx), %xmm1
1783 TOLOWER (%xmm0, %xmm1)
1784 pcmpistri $0x1a, %xmm1, %xmm0
1785 # endif
1786 jbe LABEL(use_sse4_2_exit)
1787 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1788 sub $16, %r11
1789 jbe LABEL(strcmp_exitz_sse4_2)
1790 # endif
1792 add $16, %rdx
1793 add $16, %r10
1794 jg LABEL(nibble_ashr_15_use_sse4_2)
1796 movdqa (%rdi, %rdx), %xmm0
1797 palignr $15, -16(%rdi, %rdx), %xmm0
1798 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1799 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1800 # else
1801 movdqa (%rsi,%rdx), %xmm1
1802 TOLOWER (%xmm0, %xmm1)
1803 pcmpistri $0x1a, %xmm1, %xmm0
1804 # endif
1805 jbe LABEL(use_sse4_2_exit)
1806 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1807 sub $16, %r11
1808 jbe LABEL(strcmp_exitz_sse4_2)
1809 # endif
1810 add $16, %rdx
1811 jmp LABEL(loop_ashr_15_use_sse4_2)
1813 .p2align 4
1814 LABEL(nibble_ashr_15_use_sse4_2):
1815 sub $0x1000, %r10
1816 movdqa -16(%rdi, %rdx), %xmm0
1817 psrldq $15, %xmm0
1818 pcmpistri $0x3a,%xmm0, %xmm0
1819 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1820 cmp %r11, %rcx
1821 jae LABEL(nibble_ashr_use_sse4_2_exit)
1822 # endif
1823 cmp $0, %ecx
1824 ja LABEL(nibble_ashr_15_use_sse4_2_restart)
1826 LABEL(nibble_ashr_use_sse4_2_exit):
1827 # if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1828 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1829 # else
1830 movdqa (%rsi,%rdx), %xmm1
1831 TOLOWER (%xmm0, %xmm1)
1832 pcmpistri $0x1a, %xmm1, %xmm0
1833 # endif
1834 .p2align 4
1835 LABEL(use_sse4_2_exit):
1836 jnc LABEL(strcmp_exitz_sse4_2)
1837 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1838 sub %rcx, %r11
1839 jbe LABEL(strcmp_exitz_sse4_2)
1840 # endif
1841 add %rcx, %rdx
1842 lea -16(%rdi, %r9), %rdi
1843 movzbl (%rdi, %rdx), %eax
1844 movzbl (%rsi, %rdx), %edx
1845 test %r8d, %r8d
1846 jz LABEL(use_sse4_2_ret_sse4_2)
1847 xchg %eax, %edx
1848 LABEL(use_sse4_2_ret_sse4_2):
1849 # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
1850 leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rcx
1851 movl (%rcx,%rdx,4), %edx
1852 movl (%rcx,%rax,4), %eax
1853 # endif
1855 sub %edx, %eax
1856 ret
1858 LABEL(less32bytes_sse4_2):
1859 lea (%rdi, %rax), %rdi /* locate the exact address for first operand(rdi) */
1860 lea (%rsi, %rcx), %rsi /* locate the exact address for second operand(rsi) */
1861 test %r8d, %r8d
1862 jz LABEL(ret_sse4_2)
1863 xchg %rsi, %rdi /* recover original order according to flag(%r8d) */
1865 .p2align 4
1866 LABEL(ret_sse4_2):
1867 LABEL(less16bytes_sse4_2):
1868 bsf %rdx, %rdx /* find and store bit index in %rdx */
1870 # if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1871 sub %rdx, %r11
1872 jbe LABEL(strcmp_exitz_sse4_2)
1873 # endif
1874 movzbl (%rsi, %rdx), %ecx
1875 movzbl (%rdi, %rdx), %eax
1877 # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
1878 leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx
1879 movl (%rdx,%rcx,4), %ecx
1880 movl (%rdx,%rax,4), %eax
1881 # endif
1883 sub %ecx, %eax
1884 ret
1886 LABEL(strcmp_exitz_sse4_2):
1887 xor %eax, %eax
1888 ret
1890 .p2align 4
1891 // XXX Same as code above
1892 LABEL(Byte0_sse4_2):
1893 movzx (%rsi), %ecx
1894 movzx (%rdi), %eax
1896 # if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
1897 leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx
1898 movl (%rdx,%rcx,4), %ecx
1899 movl (%rdx,%rax,4), %eax
1900 # endif
1902 sub %ecx, %eax
1903 ret
1904 cfi_endproc
1905 .size STRCMP_SSE42, .-STRCMP_SSE42
1907 # undef UCLOW_reg
1908 # undef UCHIGH_reg
1909 # undef LCQWORD_reg
1910 # undef TOLOWER
1912 /* Put all SSE 4.2 functions together. */
1913 .section .rodata.sse4.2,"a",@progbits
1914 .p2align 3
1915 LABEL(unaligned_table_sse4_2):
1916 .int LABEL(ashr_1_sse4_2) - LABEL(unaligned_table_sse4_2)
1917 .int LABEL(ashr_2_sse4_2) - LABEL(unaligned_table_sse4_2)
1918 .int LABEL(ashr_3_sse4_2) - LABEL(unaligned_table_sse4_2)
1919 .int LABEL(ashr_4_sse4_2) - LABEL(unaligned_table_sse4_2)
1920 .int LABEL(ashr_5_sse4_2) - LABEL(unaligned_table_sse4_2)
1921 .int LABEL(ashr_6_sse4_2) - LABEL(unaligned_table_sse4_2)
1922 .int LABEL(ashr_7_sse4_2) - LABEL(unaligned_table_sse4_2)
1923 .int LABEL(ashr_8_sse4_2) - LABEL(unaligned_table_sse4_2)
1924 .int LABEL(ashr_9_sse4_2) - LABEL(unaligned_table_sse4_2)
1925 .int LABEL(ashr_10_sse4_2) - LABEL(unaligned_table_sse4_2)
1926 .int LABEL(ashr_11_sse4_2) - LABEL(unaligned_table_sse4_2)
1927 .int LABEL(ashr_12_sse4_2) - LABEL(unaligned_table_sse4_2)
1928 .int LABEL(ashr_13_sse4_2) - LABEL(unaligned_table_sse4_2)
1929 .int LABEL(ashr_14_sse4_2) - LABEL(unaligned_table_sse4_2)
1930 .int LABEL(ashr_15_sse4_2) - LABEL(unaligned_table_sse4_2)
1931 .int LABEL(ashr_0_sse4_2) - LABEL(unaligned_table_sse4_2)
1934 # undef ENTRY
1935 # define ENTRY(name) \
1936 .type STRCMP_SSE2, @function; \
1937 .align 16; \
1938 STRCMP_SSE2: cfi_startproc; \
1939 CALL_MCOUNT
1940 # undef END
1941 # define END(name) \
1942 cfi_endproc; .size STRCMP_SSE2, .-STRCMP_SSE2
1944 # ifdef USE_AS_STRCASECMP_L
1945 # define ENTRY2(name) \
1946 .type __strcasecmp_sse2, @function; \
1947 .align 16; \
1948 __strcasecmp_sse2: cfi_startproc; \
1949 CALL_MCOUNT
1950 # define END2(name) \
1951 cfi_endproc; .size __strcasecmp_sse2, .-__strcasecmp_sse2
1952 # endif
1954 # ifdef USE_AS_STRNCASECMP_L
1955 # define ENTRY2(name) \
1956 .type __strncasecmp_sse2, @function; \
1957 .align 16; \
1958 __strncasecmp_sse2: cfi_startproc; \
1959 CALL_MCOUNT
1960 # define END2(name) \
1961 cfi_endproc; .size __strncasecmp_sse2, .-__strncasecmp_sse2
1962 # endif
1964 # undef libc_hidden_builtin_def
1965 /* It doesn't make sense to send libc-internal strcmp calls through a PLT.
1966 The speedup we get from using SSE4.2 instruction is likely eaten away
1967 by the indirect call in the PLT. */
1968 # define libc_hidden_builtin_def(name) \
1969 .globl __GI_STRCMP; __GI_STRCMP = STRCMP_SSE2
1970 #endif
1972 #include "../strcmp.S"