2 Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
3 Contributed by Intel Corporation.
4 This file is part of the GNU C Library.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <http://www.gnu.org/licenses/>. */
23 | _SIDD_CMP_EQUAL_EACH
24 | _SIDD_NEGATIVE_POLARITY
25 | _SIDD_LEAST_SIGNIFICANT
26 on pcmpistri to find out if two 16byte data elements are the same
27 and the offset of the first different byte. There are 4 cases:
29 1. Both 16byte data elements are valid and identical.
30 2. Both 16byte data elements have EOS and identical.
31 3. Both 16byte data elements are valid and they differ at offset X.
32 4. At least one 16byte data element has EOS at offset X. Two 16byte
33 data elements must differ at or before offset X.
35 Here is the table of ECX, CFlag, ZFlag and SFlag for 4 cases:
37 case ECX CFlag ZFlag SFlag
43 We exit from the loop for cases 2, 3 and 4 with jbe which branches
44 when either CFlag or ZFlag is 1. If CFlag == 0, we return 0 for
47 /* Put all SSE 4.2 functions together. */
48 .section .text.SECTION,"ax",@progbits
50 .type STRCMP_SSE42, @function
51 #ifdef USE_AS_STRCASECMP_L
52 ENTRY (GLABEL(__strcasecmp))
53 movq __libc_tsd_LOCALE@gottpoff(%rip),%rax
56 // XXX 5 byte should be before the function
58 .byte 0x0f,0x1f,0x44,0x00,0x00
59 END (GLABEL(__strcasecmp))
60 /* FALLTHROUGH to strcasecmp_l. */
62 #ifdef USE_AS_STRNCASECMP_L
63 ENTRY (GLABEL(__strncasecmp))
64 movq __libc_tsd_LOCALE@gottpoff(%rip),%rax
67 // XXX 5 byte should be before the function
69 .byte 0x0f,0x1f,0x44,0x00,0x00
70 END (GLABEL(__strncasecmp))
71 /* FALLTHROUGH to strncasecmp_l. */
76 # define movdqa vmovdqa
77 # define movdqu vmovdqu
78 # define pmovmskb vpmovmskb
79 # define pcmpistri vpcmpistri
81 # define pcmpeqb vpcmpeqb
82 # define psrldq vpsrldq
83 # define pslldq vpslldq
84 # define palignr vpalignr
86 # define D(arg) arg, arg
96 * This implementation uses SSE to compare up to 16 bytes at a time.
98 #ifdef USE_AS_STRCASECMP_L
99 /* We have to fall back on the C implementation for locales
100 with encodings not matching ASCII for single bytes. */
101 # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0
102 movq LOCALE_T___LOCALES+LC_CTYPE*8(%rdx), %rax
106 testl $1, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax)
107 jne __strcasecmp_l_nonascii
109 #ifdef USE_AS_STRNCASECMP_L
110 /* We have to fall back on the C implementation for locales
111 with encodings not matching ASCII for single bytes. */
112 # if LOCALE_T___LOCALES != 0 || LC_CTYPE != 0
113 movq LOCALE_T___LOCALES+LC_CTYPE*8(%rcx), %rax
117 testl $1, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES(%rax)
118 jne __strncasecmp_l_nonascii
121 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
123 je LABEL(strcmp_exitz)
130 /* Use 64bit AND here to avoid long NOP padding. */
131 and $0x3f, %rcx /* rsi alignment in cache line */
132 and $0x3f, %rax /* rdi alignment in cache line */
133 #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
134 .section .rodata.cst16,"aM",@progbits,16
137 .quad 0x4040404040404040
138 .quad 0x4040404040404040
141 .quad 0x5a5a5a5a5a5a5a5a
142 .quad 0x5a5a5a5a5a5a5a5a
144 .quad 0x5b5b5b5b5b5b5b5b
145 .quad 0x5b5b5b5b5b5b5b5b
148 .quad 0x2020202020202020
149 .quad 0x2020202020202020
151 movdqa LABEL(belowupper)(%rip), %xmm4
152 # define UCLOW_reg %xmm4
153 movdqa LABEL(topupper)(%rip), %xmm5
154 # define UCHIGH_reg %xmm5
155 movdqa LABEL(touppermask)(%rip), %xmm6
156 # define LCQWORD_reg %xmm6
159 ja LABEL(crosscache)/* rsi: 16-byte load will cross cache line */
161 ja LABEL(crosscache)/* rdi: 16-byte load will cross cache line */
164 #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
166 # define TOLOWER(reg1, reg2) \
167 vpcmpgtb UCLOW_reg, reg1, %xmm7; \
168 vpcmpgtb UCHIGH_reg, reg1, %xmm8; \
169 vpcmpgtb UCLOW_reg, reg2, %xmm9; \
170 vpcmpgtb UCHIGH_reg, reg2, %xmm10; \
171 vpandn %xmm7, %xmm8, %xmm8; \
172 vpandn %xmm9, %xmm10, %xmm10; \
173 vpand LCQWORD_reg, %xmm8, %xmm8; \
174 vpand LCQWORD_reg, %xmm10, %xmm10; \
175 vpor reg1, %xmm8, reg1; \
176 vpor reg2, %xmm10, reg2
178 # define TOLOWER(reg1, reg2) \
179 movdqa reg1, %xmm7; \
180 movdqa UCHIGH_reg, %xmm8; \
181 movdqa reg2, %xmm9; \
182 movdqa UCHIGH_reg, %xmm10; \
183 pcmpgtb UCLOW_reg, %xmm7; \
184 pcmpgtb reg1, %xmm8; \
185 pcmpgtb UCLOW_reg, %xmm9; \
186 pcmpgtb reg2, %xmm10; \
188 pand %xmm10, %xmm9; \
189 pand LCQWORD_reg, %xmm7; \
190 pand LCQWORD_reg, %xmm9; \
194 TOLOWER (%xmm1, %xmm2)
196 # define TOLOWER(reg1, reg2)
198 pxor %xmm0, D(%xmm0) /* clear %xmm0 for null char checks */
199 pcmpeqb %xmm1, D(%xmm0) /* Any null chars? */
200 pcmpeqb %xmm2, D(%xmm1) /* compare first 16 bytes for equality */
201 psubb %xmm0, D(%xmm1) /* packed sub of comparison results*/
203 sub $0xffff, %edx /* if first 16 bytes are same, edx == 0xffff */
204 jnz LABEL(less16bytes)/* If not, find different value or null char */
205 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
207 jbe LABEL(strcmp_exitz)/* finish comparision */
209 add $16, %rsi /* prepare to search next 16 bytes */
210 add $16, %rdi /* prepare to search next 16 bytes */
213 * Determine source and destination string offsets from 16-byte
214 * alignment. Use relative offset difference between the two to
215 * determine which case below to use.
219 and $0xfffffffffffffff0, %rsi /* force %rsi is 16 byte aligned */
220 and $0xfffffffffffffff0, %rdi /* force %rdi is 16 byte aligned */
221 mov $0xffff, %edx /* for equivalent offset */
223 and $0xf, %ecx /* offset of rsi */
224 and $0xf, %eax /* offset of rdi */
225 pxor %xmm0, D(%xmm0) /* clear %xmm0 for null char check */
227 je LABEL(ashr_0) /* rsi and rdi relative offset same */
229 mov %edx, %r8d /* r8d is offset flag for exit tail */
237 lea LABEL(unaligned_table)(%rip), %r10
238 movslq (%r10, %r9,4), %r9
239 pcmpeqb %xmm1, D(%xmm0) /* Any null chars? */
240 lea (%r10, %r9), %r10
241 jmp *%r10 /* jump to corresponding case */
244 * The following cases will be handled by ashr_0
245 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
246 * n(0~15) n(0~15) 15(15+ n-n) ashr_0
252 pcmpeqb %xmm1, D(%xmm0) /* Any null chars? */
253 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
254 pcmpeqb (%rdi), D(%xmm1) /* compare 16 bytes for equality */
257 TOLOWER (%xmm1, %xmm2)
258 pcmpeqb %xmm2, D(%xmm1) /* compare 16 bytes for equality */
260 psubb %xmm0, D(%xmm1) /* packed sub of comparison results*/
262 shr %cl, %edx /* adjust 0xffff for offset */
263 shr %cl, %r9d /* adjust for 16-byte offset */
266 * edx must be the same with r9d if in left byte (16-rcx) is equal to
267 * the start from (16-rax) and no null char was seen.
269 jne LABEL(less32bytes) /* mismatch or null char */
270 UPDATE_STRNCMP_COUNTER
275 * Now both strings are aligned at 16-byte boundary. Loop over strings
276 * checking 32-bytes per iteration.
278 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
281 movdqa (%rdi,%rdx), %xmm0
282 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
283 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
285 movdqa (%rsi,%rdx), %xmm1
286 TOLOWER (%xmm0, %xmm1)
287 pcmpistri $0x1a, %xmm1, %xmm0
290 jbe LABEL(ashr_0_exit_use)
291 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
293 jbe LABEL(strcmp_exitz)
296 movdqa (%rdi,%rdx), %xmm0
297 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
298 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
300 movdqa (%rsi,%rdx), %xmm1
301 TOLOWER (%xmm0, %xmm1)
302 pcmpistri $0x1a, %xmm1, %xmm0
305 jbe LABEL(ashr_0_exit_use)
306 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
308 jbe LABEL(strcmp_exitz)
310 jmp LABEL(ashr_0_use)
314 LABEL(ashr_0_exit_use):
315 jnc LABEL(strcmp_exitz)
316 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
318 jbe LABEL(strcmp_exitz)
320 lea -16(%rdx, %rcx), %rcx
321 movzbl (%rdi, %rcx), %eax
322 movzbl (%rsi, %rcx), %edx
323 #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
324 leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rcx
325 movl (%rcx,%rax,4), %eax
326 movl (%rcx,%rdx,4), %edx
334 * The following cases will be handled by ashr_1
335 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
336 * n(15) n -15 0(15 +(n-15) - n) ashr_1
340 pslldq $15, D(%xmm2) /* shift first string to align with second */
341 TOLOWER (%xmm1, %xmm2)
342 pcmpeqb %xmm1, D(%xmm2) /* compare 16 bytes for equality */
343 psubb %xmm0, D(%xmm2) /* packed sub of comparison results*/
345 shr %cl, %edx /* adjust 0xffff for offset */
346 shr %cl, %r9d /* adjust for 16-byte offset */
348 jnz LABEL(less32bytes) /* mismatch or null char seen */
350 UPDATE_STRNCMP_COUNTER
352 mov $16, %rcx /* index for loads*/
353 mov $1, %r9d /* byte position left over from less32bytes case */
355 * Setup %r10 value allows us to detect crossing a page boundary.
356 * When %r10 goes positive we have crossed a page boundary and
357 * need to do a nibble.
360 and $0xfff, %r10 /* offset into 4K page */
361 sub $0x1000, %r10 /* subtract 4K pagesize */
362 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
365 LABEL(loop_ashr_1_use):
367 jg LABEL(nibble_ashr_1_use)
369 LABEL(nibble_ashr_1_restart_use):
370 movdqa (%rdi, %rdx), %xmm0
371 palignr $1, -16(%rdi, %rdx), D(%xmm0)
372 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
373 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
375 movdqa (%rsi,%rdx), %xmm1
376 TOLOWER (%xmm0, %xmm1)
377 pcmpistri $0x1a, %xmm1, %xmm0
380 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
382 jbe LABEL(strcmp_exitz)
387 jg LABEL(nibble_ashr_1_use)
389 movdqa (%rdi, %rdx), %xmm0
390 palignr $1, -16(%rdi, %rdx), D(%xmm0)
391 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
392 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
394 movdqa (%rsi,%rdx), %xmm1
395 TOLOWER (%xmm0, %xmm1)
396 pcmpistri $0x1a, %xmm1, %xmm0
399 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
401 jbe LABEL(strcmp_exitz)
404 jmp LABEL(loop_ashr_1_use)
407 LABEL(nibble_ashr_1_use):
409 movdqa -16(%rdi, %rdx), %xmm0
411 pcmpistri $0x3a,%xmm0, %xmm0
412 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
414 jae LABEL(nibble_ashr_exit_use)
417 ja LABEL(nibble_ashr_1_restart_use)
419 jmp LABEL(nibble_ashr_exit_use)
422 * The following cases will be handled by ashr_2
423 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
424 * n(14~15) n -14 1(15 +(n-14) - n) ashr_2
429 TOLOWER (%xmm1, %xmm2)
430 pcmpeqb %xmm1, D(%xmm2)
431 psubb %xmm0, D(%xmm2)
436 jnz LABEL(less32bytes)
438 UPDATE_STRNCMP_COUNTER
440 mov $16, %rcx /* index for loads */
441 mov $2, %r9d /* byte position left over from less32bytes case */
443 * Setup %r10 value allows us to detect crossing a page boundary.
444 * When %r10 goes positive we have crossed a page boundary and
445 * need to do a nibble.
448 and $0xfff, %r10 /* offset into 4K page */
449 sub $0x1000, %r10 /* subtract 4K pagesize */
450 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
453 LABEL(loop_ashr_2_use):
455 jg LABEL(nibble_ashr_2_use)
457 LABEL(nibble_ashr_2_restart_use):
458 movdqa (%rdi, %rdx), %xmm0
459 palignr $2, -16(%rdi, %rdx), D(%xmm0)
460 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
461 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
463 movdqa (%rsi,%rdx), %xmm1
464 TOLOWER (%xmm0, %xmm1)
465 pcmpistri $0x1a, %xmm1, %xmm0
468 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
470 jbe LABEL(strcmp_exitz)
475 jg LABEL(nibble_ashr_2_use)
477 movdqa (%rdi, %rdx), %xmm0
478 palignr $2, -16(%rdi, %rdx), D(%xmm0)
479 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
480 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
482 movdqa (%rsi,%rdx), %xmm1
483 TOLOWER (%xmm0, %xmm1)
484 pcmpistri $0x1a, %xmm1, %xmm0
487 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
489 jbe LABEL(strcmp_exitz)
492 jmp LABEL(loop_ashr_2_use)
495 LABEL(nibble_ashr_2_use):
497 movdqa -16(%rdi, %rdx), %xmm0
499 pcmpistri $0x3a,%xmm0, %xmm0
500 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
502 jae LABEL(nibble_ashr_exit_use)
505 ja LABEL(nibble_ashr_2_restart_use)
507 jmp LABEL(nibble_ashr_exit_use)
510 * The following cases will be handled by ashr_3
511 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
512 * n(13~15) n -13 2(15 +(n-13) - n) ashr_3
517 TOLOWER (%xmm1, %xmm2)
518 pcmpeqb %xmm1, D(%xmm2)
519 psubb %xmm0, D(%xmm2)
524 jnz LABEL(less32bytes)
527 UPDATE_STRNCMP_COUNTER
529 mov $16, %rcx /* index for loads */
530 mov $3, %r9d /* byte position left over from less32bytes case */
532 * Setup %r10 value allows us to detect crossing a page boundary.
533 * When %r10 goes positive we have crossed a page boundary and
534 * need to do a nibble.
537 and $0xfff, %r10 /* offset into 4K page */
538 sub $0x1000, %r10 /* subtract 4K pagesize */
539 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
541 LABEL(loop_ashr_3_use):
543 jg LABEL(nibble_ashr_3_use)
545 LABEL(nibble_ashr_3_restart_use):
546 movdqa (%rdi, %rdx), %xmm0
547 palignr $3, -16(%rdi, %rdx), D(%xmm0)
548 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
549 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
551 movdqa (%rsi,%rdx), %xmm1
552 TOLOWER (%xmm0, %xmm1)
553 pcmpistri $0x1a, %xmm1, %xmm0
556 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
558 jbe LABEL(strcmp_exitz)
563 jg LABEL(nibble_ashr_3_use)
565 movdqa (%rdi, %rdx), %xmm0
566 palignr $3, -16(%rdi, %rdx), D(%xmm0)
567 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
568 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
570 movdqa (%rsi,%rdx), %xmm1
571 TOLOWER (%xmm0, %xmm1)
572 pcmpistri $0x1a, %xmm1, %xmm0
575 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
577 jbe LABEL(strcmp_exitz)
580 jmp LABEL(loop_ashr_3_use)
583 LABEL(nibble_ashr_3_use):
585 movdqa -16(%rdi, %rdx), %xmm0
587 pcmpistri $0x3a,%xmm0, %xmm0
588 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
590 jae LABEL(nibble_ashr_exit_use)
593 ja LABEL(nibble_ashr_3_restart_use)
595 jmp LABEL(nibble_ashr_exit_use)
598 * The following cases will be handled by ashr_4
599 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
600 * n(12~15) n -12 3(15 +(n-12) - n) ashr_4
605 TOLOWER (%xmm1, %xmm2)
606 pcmpeqb %xmm1, D(%xmm2)
607 psubb %xmm0, D(%xmm2)
612 jnz LABEL(less32bytes)
615 UPDATE_STRNCMP_COUNTER
617 mov $16, %rcx /* index for loads */
618 mov $4, %r9d /* byte position left over from less32bytes case */
620 * Setup %r10 value allows us to detect crossing a page boundary.
621 * When %r10 goes positive we have crossed a page boundary and
622 * need to do a nibble.
625 and $0xfff, %r10 /* offset into 4K page */
626 sub $0x1000, %r10 /* subtract 4K pagesize */
627 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
630 LABEL(loop_ashr_4_use):
632 jg LABEL(nibble_ashr_4_use)
634 LABEL(nibble_ashr_4_restart_use):
635 movdqa (%rdi, %rdx), %xmm0
636 palignr $4, -16(%rdi, %rdx), D(%xmm0)
637 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
638 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
640 movdqa (%rsi,%rdx), %xmm1
641 TOLOWER (%xmm0, %xmm1)
642 pcmpistri $0x1a, %xmm1, %xmm0
645 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
647 jbe LABEL(strcmp_exitz)
652 jg LABEL(nibble_ashr_4_use)
654 movdqa (%rdi, %rdx), %xmm0
655 palignr $4, -16(%rdi, %rdx), D(%xmm0)
656 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
657 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
659 movdqa (%rsi,%rdx), %xmm1
660 TOLOWER (%xmm0, %xmm1)
661 pcmpistri $0x1a, %xmm1, %xmm0
664 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
666 jbe LABEL(strcmp_exitz)
669 jmp LABEL(loop_ashr_4_use)
672 LABEL(nibble_ashr_4_use):
674 movdqa -16(%rdi, %rdx), %xmm0
676 pcmpistri $0x3a,%xmm0, %xmm0
677 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
679 jae LABEL(nibble_ashr_exit_use)
682 ja LABEL(nibble_ashr_4_restart_use)
684 jmp LABEL(nibble_ashr_exit_use)
687 * The following cases will be handled by ashr_5
688 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
689 * n(11~15) n - 11 4(15 +(n-11) - n) ashr_5
694 TOLOWER (%xmm1, %xmm2)
695 pcmpeqb %xmm1, D(%xmm2)
696 psubb %xmm0, D(%xmm2)
701 jnz LABEL(less32bytes)
704 UPDATE_STRNCMP_COUNTER
706 mov $16, %rcx /* index for loads */
707 mov $5, %r9d /* byte position left over from less32bytes case */
709 * Setup %r10 value allows us to detect crossing a page boundary.
710 * When %r10 goes positive we have crossed a page boundary and
711 * need to do a nibble.
714 and $0xfff, %r10 /* offset into 4K page */
715 sub $0x1000, %r10 /* subtract 4K pagesize */
716 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
719 LABEL(loop_ashr_5_use):
721 jg LABEL(nibble_ashr_5_use)
723 LABEL(nibble_ashr_5_restart_use):
724 movdqa (%rdi, %rdx), %xmm0
725 palignr $5, -16(%rdi, %rdx), D(%xmm0)
726 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
727 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
729 movdqa (%rsi,%rdx), %xmm1
730 TOLOWER (%xmm0, %xmm1)
731 pcmpistri $0x1a, %xmm1, %xmm0
734 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
736 jbe LABEL(strcmp_exitz)
741 jg LABEL(nibble_ashr_5_use)
743 movdqa (%rdi, %rdx), %xmm0
745 palignr $5, -16(%rdi, %rdx), D(%xmm0)
746 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
747 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
749 movdqa (%rsi,%rdx), %xmm1
750 TOLOWER (%xmm0, %xmm1)
751 pcmpistri $0x1a, %xmm1, %xmm0
754 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
756 jbe LABEL(strcmp_exitz)
759 jmp LABEL(loop_ashr_5_use)
762 LABEL(nibble_ashr_5_use):
764 movdqa -16(%rdi, %rdx), %xmm0
766 pcmpistri $0x3a,%xmm0, %xmm0
767 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
769 jae LABEL(nibble_ashr_exit_use)
772 ja LABEL(nibble_ashr_5_restart_use)
774 jmp LABEL(nibble_ashr_exit_use)
777 * The following cases will be handled by ashr_6
778 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
779 * n(10~15) n - 10 5(15 +(n-10) - n) ashr_6
784 TOLOWER (%xmm1, %xmm2)
785 pcmpeqb %xmm1, D(%xmm2)
786 psubb %xmm0, D(%xmm2)
791 jnz LABEL(less32bytes)
794 UPDATE_STRNCMP_COUNTER
796 mov $16, %rcx /* index for loads */
797 mov $6, %r9d /* byte position left over from less32bytes case */
799 * Setup %r10 value allows us to detect crossing a page boundary.
800 * When %r10 goes positive we have crossed a page boundary and
801 * need to do a nibble.
804 and $0xfff, %r10 /* offset into 4K page */
805 sub $0x1000, %r10 /* subtract 4K pagesize */
806 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
809 LABEL(loop_ashr_6_use):
811 jg LABEL(nibble_ashr_6_use)
813 LABEL(nibble_ashr_6_restart_use):
814 movdqa (%rdi, %rdx), %xmm0
815 palignr $6, -16(%rdi, %rdx), D(%xmm0)
816 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
817 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
819 movdqa (%rsi,%rdx), %xmm1
820 TOLOWER (%xmm0, %xmm1)
821 pcmpistri $0x1a, %xmm1, %xmm0
824 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
826 jbe LABEL(strcmp_exitz)
831 jg LABEL(nibble_ashr_6_use)
833 movdqa (%rdi, %rdx), %xmm0
834 palignr $6, -16(%rdi, %rdx), D(%xmm0)
835 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
836 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
838 movdqa (%rsi,%rdx), %xmm1
839 TOLOWER (%xmm0, %xmm1)
840 pcmpistri $0x1a, %xmm1, %xmm0
843 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
845 jbe LABEL(strcmp_exitz)
848 jmp LABEL(loop_ashr_6_use)
851 LABEL(nibble_ashr_6_use):
853 movdqa -16(%rdi, %rdx), %xmm0
855 pcmpistri $0x3a,%xmm0, %xmm0
856 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
858 jae LABEL(nibble_ashr_exit_use)
861 ja LABEL(nibble_ashr_6_restart_use)
863 jmp LABEL(nibble_ashr_exit_use)
866 * The following cases will be handled by ashr_7
867 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
868 * n(9~15) n - 9 6(15 +(n - 9) - n) ashr_7
873 TOLOWER (%xmm1, %xmm2)
874 pcmpeqb %xmm1, D(%xmm2)
875 psubb %xmm0, D(%xmm2)
880 jnz LABEL(less32bytes)
883 UPDATE_STRNCMP_COUNTER
885 mov $16, %rcx /* index for loads */
886 mov $7, %r9d /* byte position left over from less32bytes case */
888 * Setup %r10 value allows us to detect crossing a page boundary.
889 * When %r10 goes positive we have crossed a page boundary and
890 * need to do a nibble.
893 and $0xfff, %r10 /* offset into 4K page */
894 sub $0x1000, %r10 /* subtract 4K pagesize */
895 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
898 LABEL(loop_ashr_7_use):
900 jg LABEL(nibble_ashr_7_use)
902 LABEL(nibble_ashr_7_restart_use):
903 movdqa (%rdi, %rdx), %xmm0
904 palignr $7, -16(%rdi, %rdx), D(%xmm0)
905 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
906 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
908 movdqa (%rsi,%rdx), %xmm1
909 TOLOWER (%xmm0, %xmm1)
910 pcmpistri $0x1a, %xmm1, %xmm0
913 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
915 jbe LABEL(strcmp_exitz)
920 jg LABEL(nibble_ashr_7_use)
922 movdqa (%rdi, %rdx), %xmm0
923 palignr $7, -16(%rdi, %rdx), D(%xmm0)
924 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
925 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
927 movdqa (%rsi,%rdx), %xmm1
928 TOLOWER (%xmm0, %xmm1)
929 pcmpistri $0x1a, %xmm1, %xmm0
932 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
934 jbe LABEL(strcmp_exitz)
937 jmp LABEL(loop_ashr_7_use)
940 LABEL(nibble_ashr_7_use):
942 movdqa -16(%rdi, %rdx), %xmm0
944 pcmpistri $0x3a,%xmm0, %xmm0
945 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
947 jae LABEL(nibble_ashr_exit_use)
950 ja LABEL(nibble_ashr_7_restart_use)
952 jmp LABEL(nibble_ashr_exit_use)
955 * The following cases will be handled by ashr_8
956 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
957 * n(8~15) n - 8 7(15 +(n - 8) - n) ashr_8
962 TOLOWER (%xmm1, %xmm2)
963 pcmpeqb %xmm1, D(%xmm2)
964 psubb %xmm0, D(%xmm2)
969 jnz LABEL(less32bytes)
972 UPDATE_STRNCMP_COUNTER
974 mov $16, %rcx /* index for loads */
975 mov $8, %r9d /* byte position left over from less32bytes case */
977 * Setup %r10 value allows us to detect crossing a page boundary.
978 * When %r10 goes positive we have crossed a page boundary and
979 * need to do a nibble.
982 and $0xfff, %r10 /* offset into 4K page */
983 sub $0x1000, %r10 /* subtract 4K pagesize */
984 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
987 LABEL(loop_ashr_8_use):
989 jg LABEL(nibble_ashr_8_use)
991 LABEL(nibble_ashr_8_restart_use):
992 movdqa (%rdi, %rdx), %xmm0
993 palignr $8, -16(%rdi, %rdx), D(%xmm0)
994 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
995 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
997 movdqa (%rsi,%rdx), %xmm1
998 TOLOWER (%xmm0, %xmm1)
999 pcmpistri $0x1a, %xmm1, %xmm0
1002 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1004 jbe LABEL(strcmp_exitz)
1009 jg LABEL(nibble_ashr_8_use)
1011 movdqa (%rdi, %rdx), %xmm0
1012 palignr $8, -16(%rdi, %rdx), D(%xmm0)
1013 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1014 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1016 movdqa (%rsi,%rdx), %xmm1
1017 TOLOWER (%xmm0, %xmm1)
1018 pcmpistri $0x1a, %xmm1, %xmm0
1021 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1023 jbe LABEL(strcmp_exitz)
1026 jmp LABEL(loop_ashr_8_use)
1029 LABEL(nibble_ashr_8_use):
1031 movdqa -16(%rdi, %rdx), %xmm0
1033 pcmpistri $0x3a,%xmm0, %xmm0
1034 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1036 jae LABEL(nibble_ashr_exit_use)
1039 ja LABEL(nibble_ashr_8_restart_use)
1041 jmp LABEL(nibble_ashr_exit_use)
1044 * The following cases will be handled by ashr_9
1045 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1046 * n(7~15) n - 7 8(15 +(n - 7) - n) ashr_9
1051 TOLOWER (%xmm1, %xmm2)
1052 pcmpeqb %xmm1, D(%xmm2)
1053 psubb %xmm0, D(%xmm2)
1054 pmovmskb %xmm2, %r9d
1058 jnz LABEL(less32bytes)
1059 movdqa (%rdi), %xmm3
1061 UPDATE_STRNCMP_COUNTER
1063 mov $16, %rcx /* index for loads */
1064 mov $9, %r9d /* byte position left over from less32bytes case */
1066 * Setup %r10 value allows us to detect crossing a page boundary.
1067 * When %r10 goes positive we have crossed a page boundary and
1068 * need to do a nibble.
1071 and $0xfff, %r10 /* offset into 4K page */
1072 sub $0x1000, %r10 /* subtract 4K pagesize */
1073 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1076 LABEL(loop_ashr_9_use):
1078 jg LABEL(nibble_ashr_9_use)
1080 LABEL(nibble_ashr_9_restart_use):
1081 movdqa (%rdi, %rdx), %xmm0
1083 palignr $9, -16(%rdi, %rdx), D(%xmm0)
1084 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1085 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1087 movdqa (%rsi,%rdx), %xmm1
1088 TOLOWER (%xmm0, %xmm1)
1089 pcmpistri $0x1a, %xmm1, %xmm0
1092 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1094 jbe LABEL(strcmp_exitz)
1099 jg LABEL(nibble_ashr_9_use)
1101 movdqa (%rdi, %rdx), %xmm0
1102 palignr $9, -16(%rdi, %rdx), D(%xmm0)
1103 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1104 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1106 movdqa (%rsi,%rdx), %xmm1
1107 TOLOWER (%xmm0, %xmm1)
1108 pcmpistri $0x1a, %xmm1, %xmm0
1111 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1113 jbe LABEL(strcmp_exitz)
1116 jmp LABEL(loop_ashr_9_use)
1119 LABEL(nibble_ashr_9_use):
1121 movdqa -16(%rdi, %rdx), %xmm0
1123 pcmpistri $0x3a,%xmm0, %xmm0
1124 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1126 jae LABEL(nibble_ashr_exit_use)
1129 ja LABEL(nibble_ashr_9_restart_use)
1131 jmp LABEL(nibble_ashr_exit_use)
1134 * The following cases will be handled by ashr_10
1135 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1136 * n(6~15) n - 6 9(15 +(n - 6) - n) ashr_10
1141 TOLOWER (%xmm1, %xmm2)
1142 pcmpeqb %xmm1, D(%xmm2)
1143 psubb %xmm0, D(%xmm2)
1144 pmovmskb %xmm2, %r9d
1148 jnz LABEL(less32bytes)
1149 movdqa (%rdi), %xmm3
1151 UPDATE_STRNCMP_COUNTER
1153 mov $16, %rcx /* index for loads */
1154 mov $10, %r9d /* byte position left over from less32bytes case */
1156 * Setup %r10 value allows us to detect crossing a page boundary.
1157 * When %r10 goes positive we have crossed a page boundary and
1158 * need to do a nibble.
1161 and $0xfff, %r10 /* offset into 4K page */
1162 sub $0x1000, %r10 /* subtract 4K pagesize */
1163 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1166 LABEL(loop_ashr_10_use):
1168 jg LABEL(nibble_ashr_10_use)
1170 LABEL(nibble_ashr_10_restart_use):
1171 movdqa (%rdi, %rdx), %xmm0
1172 palignr $10, -16(%rdi, %rdx), D(%xmm0)
1173 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1174 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1176 movdqa (%rsi,%rdx), %xmm1
1177 TOLOWER (%xmm0, %xmm1)
1178 pcmpistri $0x1a, %xmm1, %xmm0
1181 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1183 jbe LABEL(strcmp_exitz)
1188 jg LABEL(nibble_ashr_10_use)
1190 movdqa (%rdi, %rdx), %xmm0
1191 palignr $10, -16(%rdi, %rdx), D(%xmm0)
1192 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1193 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1195 movdqa (%rsi,%rdx), %xmm1
1196 TOLOWER (%xmm0, %xmm1)
1197 pcmpistri $0x1a, %xmm1, %xmm0
1200 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1202 jbe LABEL(strcmp_exitz)
1205 jmp LABEL(loop_ashr_10_use)
1208 LABEL(nibble_ashr_10_use):
1210 movdqa -16(%rdi, %rdx), %xmm0
1211 psrldq $10, D(%xmm0)
1212 pcmpistri $0x3a,%xmm0, %xmm0
1213 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1215 jae LABEL(nibble_ashr_exit_use)
1218 ja LABEL(nibble_ashr_10_restart_use)
1220 jmp LABEL(nibble_ashr_exit_use)
1223 * The following cases will be handled by ashr_11
1224 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1225 * n(5~15) n - 5 10(15 +(n - 5) - n) ashr_11
1230 TOLOWER (%xmm1, %xmm2)
1231 pcmpeqb %xmm1, D(%xmm2)
1232 psubb %xmm0, D(%xmm2)
1233 pmovmskb %xmm2, %r9d
1237 jnz LABEL(less32bytes)
1238 movdqa (%rdi), %xmm3
1240 UPDATE_STRNCMP_COUNTER
1242 mov $16, %rcx /* index for loads */
1243 mov $11, %r9d /* byte position left over from less32bytes case */
1245 * Setup %r10 value allows us to detect crossing a page boundary.
1246 * When %r10 goes positive we have crossed a page boundary and
1247 * need to do a nibble.
1250 and $0xfff, %r10 /* offset into 4K page */
1251 sub $0x1000, %r10 /* subtract 4K pagesize */
1252 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1255 LABEL(loop_ashr_11_use):
1257 jg LABEL(nibble_ashr_11_use)
1259 LABEL(nibble_ashr_11_restart_use):
1260 movdqa (%rdi, %rdx), %xmm0
1261 palignr $11, -16(%rdi, %rdx), D(%xmm0)
1262 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1263 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1265 movdqa (%rsi,%rdx), %xmm1
1266 TOLOWER (%xmm0, %xmm1)
1267 pcmpistri $0x1a, %xmm1, %xmm0
1270 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1272 jbe LABEL(strcmp_exitz)
1277 jg LABEL(nibble_ashr_11_use)
1279 movdqa (%rdi, %rdx), %xmm0
1280 palignr $11, -16(%rdi, %rdx), D(%xmm0)
1281 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1282 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1284 movdqa (%rsi,%rdx), %xmm1
1285 TOLOWER (%xmm0, %xmm1)
1286 pcmpistri $0x1a, %xmm1, %xmm0
1289 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1291 jbe LABEL(strcmp_exitz)
1294 jmp LABEL(loop_ashr_11_use)
1297 LABEL(nibble_ashr_11_use):
1299 movdqa -16(%rdi, %rdx), %xmm0
1300 psrldq $11, D(%xmm0)
1301 pcmpistri $0x3a,%xmm0, %xmm0
1302 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1304 jae LABEL(nibble_ashr_exit_use)
1307 ja LABEL(nibble_ashr_11_restart_use)
1309 jmp LABEL(nibble_ashr_exit_use)
1312 * The following cases will be handled by ashr_12
1313 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1314 * n(4~15) n - 4 11(15 +(n - 4) - n) ashr_12
1319 TOLOWER (%xmm1, %xmm2)
1320 pcmpeqb %xmm1, D(%xmm2)
1321 psubb %xmm0, D(%xmm2)
1322 pmovmskb %xmm2, %r9d
1326 jnz LABEL(less32bytes)
1327 movdqa (%rdi), %xmm3
1329 UPDATE_STRNCMP_COUNTER
1331 mov $16, %rcx /* index for loads */
1332 mov $12, %r9d /* byte position left over from less32bytes case */
1334 * Setup %r10 value allows us to detect crossing a page boundary.
1335 * When %r10 goes positive we have crossed a page boundary and
1336 * need to do a nibble.
1339 and $0xfff, %r10 /* offset into 4K page */
1340 sub $0x1000, %r10 /* subtract 4K pagesize */
1341 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1344 LABEL(loop_ashr_12_use):
1346 jg LABEL(nibble_ashr_12_use)
1348 LABEL(nibble_ashr_12_restart_use):
1349 movdqa (%rdi, %rdx), %xmm0
1350 palignr $12, -16(%rdi, %rdx), D(%xmm0)
1351 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1352 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1354 movdqa (%rsi,%rdx), %xmm1
1355 TOLOWER (%xmm0, %xmm1)
1356 pcmpistri $0x1a, %xmm1, %xmm0
1359 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1361 jbe LABEL(strcmp_exitz)
1366 jg LABEL(nibble_ashr_12_use)
1368 movdqa (%rdi, %rdx), %xmm0
1369 palignr $12, -16(%rdi, %rdx), D(%xmm0)
1370 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1371 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1373 movdqa (%rsi,%rdx), %xmm1
1374 TOLOWER (%xmm0, %xmm1)
1375 pcmpistri $0x1a, %xmm1, %xmm0
1378 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1380 jbe LABEL(strcmp_exitz)
1383 jmp LABEL(loop_ashr_12_use)
1386 LABEL(nibble_ashr_12_use):
1388 movdqa -16(%rdi, %rdx), %xmm0
1389 psrldq $12, D(%xmm0)
1390 pcmpistri $0x3a,%xmm0, %xmm0
1391 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1393 jae LABEL(nibble_ashr_exit_use)
1396 ja LABEL(nibble_ashr_12_restart_use)
1398 jmp LABEL(nibble_ashr_exit_use)
1401 * The following cases will be handled by ashr_13
1402 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1403 * n(3~15) n - 3 12(15 +(n - 3) - n) ashr_13
1408 TOLOWER (%xmm1, %xmm2)
1409 pcmpeqb %xmm1, D(%xmm2)
1410 psubb %xmm0, D(%xmm2)
1411 pmovmskb %xmm2, %r9d
1415 jnz LABEL(less32bytes)
1416 movdqa (%rdi), %xmm3
1418 UPDATE_STRNCMP_COUNTER
1420 mov $16, %rcx /* index for loads */
1421 mov $13, %r9d /* byte position left over from less32bytes case */
1423 * Setup %r10 value allows us to detect crossing a page boundary.
1424 * When %r10 goes positive we have crossed a page boundary and
1425 * need to do a nibble.
1428 and $0xfff, %r10 /* offset into 4K page */
1429 sub $0x1000, %r10 /* subtract 4K pagesize */
1431 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1434 LABEL(loop_ashr_13_use):
1436 jg LABEL(nibble_ashr_13_use)
1438 LABEL(nibble_ashr_13_restart_use):
1439 movdqa (%rdi, %rdx), %xmm0
1440 palignr $13, -16(%rdi, %rdx), D(%xmm0)
1441 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1442 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1444 movdqa (%rsi,%rdx), %xmm1
1445 TOLOWER (%xmm0, %xmm1)
1446 pcmpistri $0x1a, %xmm1, %xmm0
1449 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1451 jbe LABEL(strcmp_exitz)
1456 jg LABEL(nibble_ashr_13_use)
1458 movdqa (%rdi, %rdx), %xmm0
1459 palignr $13, -16(%rdi, %rdx), D(%xmm0)
1460 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1461 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1463 movdqa (%rsi,%rdx), %xmm1
1464 TOLOWER (%xmm0, %xmm1)
1465 pcmpistri $0x1a, %xmm1, %xmm0
1468 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1470 jbe LABEL(strcmp_exitz)
1473 jmp LABEL(loop_ashr_13_use)
1476 LABEL(nibble_ashr_13_use):
1478 movdqa -16(%rdi, %rdx), %xmm0
1479 psrldq $13, D(%xmm0)
1480 pcmpistri $0x3a,%xmm0, %xmm0
1481 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1483 jae LABEL(nibble_ashr_exit_use)
1486 ja LABEL(nibble_ashr_13_restart_use)
1488 jmp LABEL(nibble_ashr_exit_use)
1491 * The following cases will be handled by ashr_14
1492 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1493 * n(2~15) n - 2 13(15 +(n - 2) - n) ashr_14
1498 TOLOWER (%xmm1, %xmm2)
1499 pcmpeqb %xmm1, D(%xmm2)
1500 psubb %xmm0, D(%xmm2)
1501 pmovmskb %xmm2, %r9d
1505 jnz LABEL(less32bytes)
1506 movdqa (%rdi), %xmm3
1508 UPDATE_STRNCMP_COUNTER
1510 mov $16, %rcx /* index for loads */
1511 mov $14, %r9d /* byte position left over from less32bytes case */
1513 * Setup %r10 value allows us to detect crossing a page boundary.
1514 * When %r10 goes positive we have crossed a page boundary and
1515 * need to do a nibble.
1518 and $0xfff, %r10 /* offset into 4K page */
1519 sub $0x1000, %r10 /* subtract 4K pagesize */
1521 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1524 LABEL(loop_ashr_14_use):
1526 jg LABEL(nibble_ashr_14_use)
1528 LABEL(nibble_ashr_14_restart_use):
1529 movdqa (%rdi, %rdx), %xmm0
1530 palignr $14, -16(%rdi, %rdx), D(%xmm0)
1531 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1532 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1534 movdqa (%rsi,%rdx), %xmm1
1535 TOLOWER (%xmm0, %xmm1)
1536 pcmpistri $0x1a, %xmm1, %xmm0
1539 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1541 jbe LABEL(strcmp_exitz)
1546 jg LABEL(nibble_ashr_14_use)
1548 movdqa (%rdi, %rdx), %xmm0
1549 palignr $14, -16(%rdi, %rdx), D(%xmm0)
1550 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1551 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1553 movdqa (%rsi,%rdx), %xmm1
1554 TOLOWER (%xmm0, %xmm1)
1555 pcmpistri $0x1a, %xmm1, %xmm0
1558 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1560 jbe LABEL(strcmp_exitz)
1563 jmp LABEL(loop_ashr_14_use)
1566 LABEL(nibble_ashr_14_use):
1568 movdqa -16(%rdi, %rdx), %xmm0
1569 psrldq $14, D(%xmm0)
1570 pcmpistri $0x3a,%xmm0, %xmm0
1571 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1573 jae LABEL(nibble_ashr_exit_use)
1576 ja LABEL(nibble_ashr_14_restart_use)
1578 jmp LABEL(nibble_ashr_exit_use)
1581 * The following cases will be handled by ashr_15
1582 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1583 * n(1~15) n - 1 14(15 +(n - 1) - n) ashr_15
1588 TOLOWER (%xmm1, %xmm2)
1589 pcmpeqb %xmm1, D(%xmm2)
1590 psubb %xmm0, D(%xmm2)
1591 pmovmskb %xmm2, %r9d
1595 jnz LABEL(less32bytes)
1597 movdqa (%rdi), %xmm3
1599 UPDATE_STRNCMP_COUNTER
1601 mov $16, %rcx /* index for loads */
1602 mov $15, %r9d /* byte position left over from less32bytes case */
1604 * Setup %r10 value allows us to detect crossing a page boundary.
1605 * When %r10 goes positive we have crossed a page boundary and
1606 * need to do a nibble.
1609 and $0xfff, %r10 /* offset into 4K page */
1611 sub $0x1000, %r10 /* subtract 4K pagesize */
1613 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1616 LABEL(loop_ashr_15_use):
1618 jg LABEL(nibble_ashr_15_use)
1620 LABEL(nibble_ashr_15_restart_use):
1621 movdqa (%rdi, %rdx), %xmm0
1622 palignr $15, -16(%rdi, %rdx), D(%xmm0)
1623 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1624 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1626 movdqa (%rsi,%rdx), %xmm1
1627 TOLOWER (%xmm0, %xmm1)
1628 pcmpistri $0x1a, %xmm1, %xmm0
1631 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1633 jbe LABEL(strcmp_exitz)
1638 jg LABEL(nibble_ashr_15_use)
1640 movdqa (%rdi, %rdx), %xmm0
1641 palignr $15, -16(%rdi, %rdx), D(%xmm0)
1642 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1643 pcmpistri $0x1a, (%rsi,%rdx), %xmm0
1645 movdqa (%rsi,%rdx), %xmm1
1646 TOLOWER (%xmm0, %xmm1)
1647 pcmpistri $0x1a, %xmm1, %xmm0
1650 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1652 jbe LABEL(strcmp_exitz)
1655 jmp LABEL(loop_ashr_15_use)
1658 LABEL(nibble_ashr_15_use):
1660 movdqa -16(%rdi, %rdx), %xmm0
1661 psrldq $15, D(%xmm0)
1662 pcmpistri $0x3a,%xmm0, %xmm0
1663 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1665 jae LABEL(nibble_ashr_exit_use)
1668 ja LABEL(nibble_ashr_15_restart_use)
1670 LABEL(nibble_ashr_exit_use):
1671 #if !defined USE_AS_STRCASECMP_L && !defined USE_AS_STRNCASECMP_L
1672 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1674 movdqa (%rsi,%rdx), %xmm1
1675 TOLOWER (%xmm0, %xmm1)
1676 pcmpistri $0x1a, %xmm1, %xmm0
1680 jnc LABEL(strcmp_exitz)
1681 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1683 jbe LABEL(strcmp_exitz)
1686 lea -16(%rdi, %r9), %rdi
1687 movzbl (%rdi, %rdx), %eax
1688 movzbl (%rsi, %rdx), %edx
1693 #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
1694 leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rcx
1695 movl (%rcx,%rdx,4), %edx
1696 movl (%rcx,%rax,4), %eax
1703 lea (%rdi, %rax), %rdi /* locate the exact address for first operand(rdi) */
1704 lea (%rsi, %rcx), %rsi /* locate the exact address for second operand(rsi) */
1707 xchg %rsi, %rdi /* recover original order according to flag(%r8d) */
1712 bsf %rdx, %rdx /* find and store bit index in %rdx */
1714 #if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L
1716 jbe LABEL(strcmp_exitz)
1718 movzbl (%rsi, %rdx), %ecx
1719 movzbl (%rdi, %rdx), %eax
1721 #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
1722 leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx
1723 movl (%rdx,%rcx,4), %ecx
1724 movl (%rdx,%rax,4), %eax
1730 LABEL(strcmp_exitz):
1735 // XXX Same as code above
1740 #if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L
1741 leaq _nl_C_LC_CTYPE_tolower+128*4(%rip), %rdx
1742 movl (%rdx,%rcx,4), %ecx
1743 movl (%rdx,%rax,4), %eax
1749 .size STRCMP_SSE42, .-STRCMP_SSE42
1756 /* Put all SSE 4.2 functions together. */
1757 .section .rodata.SECTION,"a",@progbits
1759 LABEL(unaligned_table):
1760 .int LABEL(ashr_1) - LABEL(unaligned_table)
1761 .int LABEL(ashr_2) - LABEL(unaligned_table)
1762 .int LABEL(ashr_3) - LABEL(unaligned_table)
1763 .int LABEL(ashr_4) - LABEL(unaligned_table)
1764 .int LABEL(ashr_5) - LABEL(unaligned_table)
1765 .int LABEL(ashr_6) - LABEL(unaligned_table)
1766 .int LABEL(ashr_7) - LABEL(unaligned_table)
1767 .int LABEL(ashr_8) - LABEL(unaligned_table)
1768 .int LABEL(ashr_9) - LABEL(unaligned_table)
1769 .int LABEL(ashr_10) - LABEL(unaligned_table)
1770 .int LABEL(ashr_11) - LABEL(unaligned_table)
1771 .int LABEL(ashr_12) - LABEL(unaligned_table)
1772 .int LABEL(ashr_13) - LABEL(unaligned_table)
1773 .int LABEL(ashr_14) - LABEL(unaligned_table)
1774 .int LABEL(ashr_15) - LABEL(unaligned_table)
1775 .int LABEL(ashr_0) - LABEL(unaligned_table)