| blob | 05adf1e2e66b28f71fc526a5d96810cfde531646 |
1 /* strcmp with SSE4.2
2 Copyright (C) 2009 Free Software Foundation, Inc.
3 Contributed by Intel Corporation.
4 This file is part of the GNU C Library.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, write to the Free
18 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
19 02111-1307 USA. */
21 #include <sysdep.h>
22 #include <ifunc-defines.h>
24 #ifdef USE_AS_STRNCMP
25 /* Since the counter, %r11, is unsigned, we branch to strcmp_exitz
26 if the new counter > the old one or is 0. */
27 #define UPDATE_STRNCMP_COUNTER \
28 /* calculate left number to compare */ \
29 lea -16(%rcx, %r11), %r9; \
30 cmp %r9, %r11; \
31 jb LABEL(strcmp_exitz_sse4_2); \
32 test %r9, %r9; \
33 je LABEL(strcmp_exitz_sse4_2); \
34 mov %r9, %r11
36 #define STRCMP_SSE42 __strncmp_sse42
37 #define STRCMP_SSSE3 __strncmp_ssse3
38 #define STRCMP_SSE2 __strncmp_sse2
39 #define __GI_STRCMP __GI_strncmp
40 #else
41 #define UPDATE_STRNCMP_COUNTER
42 #ifndef STRCMP
43 #define STRCMP strcmp
44 #define STRCMP_SSE42 __strcmp_sse42
45 #define STRCMP_SSSE3 __strcmp_ssse3
46 #define STRCMP_SSE2 __strcmp_sse2
47 #define __GI_STRCMP __GI_strcmp
48 #endif
49 #endif
51 #ifndef LABEL
52 #define LABEL(l) L(l)
53 #endif
55 /* Define multiple versions only for the definition in libc. Don't
56 define multiple versions for strncmp in static library since we
57 need strncmp before the initialization happened. */
58 #if (defined SHARED || !defined USE_AS_STRNCMP) && !defined NOT_IN_libc
59 .text
60 ENTRY(STRCMP)
61 .type STRCMP, @gnu_indirect_function
62 cmpl $0, __cpu_features+KIND_OFFSET(%rip)
63 jne 1f
64 call __init_cpu_features
65 1:
66 leaq STRCMP_SSE42(%rip), %rax
67 testl $(1<<20), __cpu_features+CPUID_OFFSET+COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_ECX_OFFSET(%rip)
68 jnz 2f
69 leaq STRCMP_SSSE3(%rip), %rax
70 testl $(1<<9), __cpu_features+CPUID_OFFSET+COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_ECX_OFFSET(%rip)
71 jnz 2f
72 leaq STRCMP_SSE2(%rip), %rax
73 2: ret
74 END(STRCMP)
76 /* We use 0x1a:
77 _SIDD_SBYTE_OPS
78 | _SIDD_CMP_EQUAL_EACH
79 | _SIDD_NEGATIVE_POLARITY
80 | _SIDD_LEAST_SIGNIFICANT
81 on pcmpistri to find out if two 16byte data elements are the same
82 and the offset of the first different byte. There are 4 cases:
84 1. Both 16byte data elements are valid and identical.
85 2. Both 16byte data elements have EOS and identical.
86 3. Both 16byte data elements are valid and they differ at offset X.
87 4. At least one 16byte data element has EOS at offset X. Two 16byte
88 data elements must differ at or before offset X.
90 Here is the table of ECX, CFlag, ZFlag and SFlag for 4 cases:
92 case ECX CFlag ZFlag SFlag
93 1 16 0 0 0
94 2 16 0 1 1
95 3 X 1 0 0
96 4 0 <= X 1 0/1 0/1
98 We exit from the loop for cases 2, 3 and 4 with jbe which branches
99 when either CFlag or ZFlag is 1. If CFlag == 0, we return 0 for
100 case 2. */
102 /* Put all SSE 4.2 functions together. */
103 .section .text.sse4.2,"ax",@progbits
104 .align 16
105 .type STRCMP_SSE42, @function
106 STRCMP_SSE42:
107 cfi_startproc
108 CALL_MCOUNT
110 /*
111 * This implementation uses SSE to compare up to 16 bytes at a time.
112 */
113 #ifdef USE_AS_STRNCMP
114 test %rdx, %rdx
115 je LABEL(strcmp_exitz_sse4_2)
116 cmp $1, %rdx
117 je LABEL(Byte0_sse4_2)
118 mov %rdx, %r11
119 #endif
120 mov %esi, %ecx
121 mov %edi, %eax
122 /* Use 64bit AND here to avoid long NOP padding. */
123 and $0x3f, %rcx /* rsi alignment in cache line */
124 and $0x3f, %rax /* rdi alignment in cache line */
125 cmp $0x30, %ecx
126 ja LABEL(crosscache_sse4_2)/* rsi: 16-byte load will cross cache line */
127 cmp $0x30, %eax
128 ja LABEL(crosscache_sse4_2)/* rdi: 16-byte load will cross cache line */
129 movdqu (%rdi), %xmm1
130 movdqu (%rsi), %xmm2
131 pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */
132 pcmpeqb %xmm1, %xmm0 /* Any null chars? */
133 pcmpeqb %xmm2, %xmm1 /* compare first 16 bytes for equality */
134 psubb %xmm0, %xmm1 /* packed sub of comparison results*/
135 pmovmskb %xmm1, %edx
136 sub $0xffff, %edx /* if first 16 bytes are same, edx == 0xffff */
137 jnz LABEL(less16bytes_sse4_2)/* If not, find different value or null char */
138 #ifdef USE_AS_STRNCMP
139 sub $16, %r11
140 jbe LABEL(strcmp_exitz_sse4_2)/* finish comparision */
141 #endif
142 add $16, %rsi /* prepare to search next 16 bytes */
143 add $16, %rdi /* prepare to search next 16 bytes */
145 /*
146 * Determine source and destination string offsets from 16-byte alignment.
147 * Use relative offset difference between the two to determine which case
148 * below to use.
149 */
150 .p2align 4
151 LABEL(crosscache_sse4_2):
152 and $0xfffffffffffffff0, %rsi /* force %rsi is 16 byte aligned */
153 and $0xfffffffffffffff0, %rdi /* force %rdi is 16 byte aligned */
154 mov $0xffff, %edx /* for equivalent offset */
155 xor %r8d, %r8d
156 and $0xf, %ecx /* offset of rsi */
157 and $0xf, %eax /* offset of rdi */
158 cmp %eax, %ecx
159 je LABEL(ashr_0_sse4_2) /* rsi and rdi relative offset same */
160 ja LABEL(bigger_sse4_2)
161 mov %edx, %r8d /* r8d is offset flag for exit tail */
162 xchg %ecx, %eax
163 xchg %rsi, %rdi
164 LABEL(bigger_sse4_2):
165 lea 15(%rax), %r9
166 sub %rcx, %r9
167 lea LABEL(unaligned_table_sse4_2)(%rip), %r10
168 movslq (%r10, %r9,4), %r9
169 lea (%r10, %r9), %r10
170 jmp *%r10 /* jump to corresponding case */
172 /*
173 * The following cases will be handled by ashr_0
174 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
175 * n(0~15) n(0~15) 15(15+ n-n) ashr_0
176 */
177 .p2align 4
178 LABEL(ashr_0_sse4_2):
180 movdqa (%rsi), %xmm1
181 pxor %xmm0, %xmm0 /* clear %xmm0 for null char check */
182 pcmpeqb %xmm1, %xmm0 /* Any null chars? */
183 pcmpeqb (%rdi), %xmm1 /* compare 16 bytes for equality */
184 psubb %xmm0, %xmm1 /* packed sub of comparison results*/
185 pmovmskb %xmm1, %r9d
186 shr %cl, %edx /* adjust 0xffff for offset */
187 shr %cl, %r9d /* adjust for 16-byte offset */
188 sub %r9d, %edx
189 /*
190 * edx must be the same with r9d if in left byte (16-rcx) is equal to
191 * the start from (16-rax) and no null char was seen.
192 */
193 jne LABEL(less32bytes_sse4_2) /* mismatch or null char */
194 UPDATE_STRNCMP_COUNTER
195 mov $16, %rcx
196 mov $16, %r9
197 pxor %xmm0, %xmm0 /* clear xmm0, may have changed above */
199 /*
200 * Now both strings are aligned at 16-byte boundary. Loop over strings
201 * checking 32-bytes per iteration.
202 */
203 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
204 .p2align 4
205 LABEL(ashr_0_use_sse4_2):
206 movdqa (%rdi,%rdx), %xmm0
207 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
208 lea 16(%rdx), %rdx
209 jbe LABEL(ashr_0_use_sse4_2_exit)
210 #ifdef USE_AS_STRNCMP
211 sub $16, %r11
212 jbe LABEL(strcmp_exitz_sse4_2)
213 #endif
215 movdqa (%rdi,%rdx), %xmm0
216 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
217 lea 16(%rdx), %rdx
218 jbe LABEL(ashr_0_use_sse4_2_exit)
219 #ifdef USE_AS_STRNCMP
220 sub $16, %r11
221 jbe LABEL(strcmp_exitz_sse4_2)
222 #endif
223 jmp LABEL(ashr_0_use_sse4_2)
226 .p2align 4
227 LABEL(ashr_0_use_sse4_2_exit):
228 jnc LABEL(strcmp_exitz_sse4_2)
229 #ifdef USE_AS_STRNCMP
230 sub %rcx, %r11
231 jbe LABEL(strcmp_exitz_sse4_2)
232 #endif
233 lea -16(%rdx, %rcx), %rcx
234 movzbl (%rdi, %rcx), %eax
235 movzbl (%rsi, %rcx), %edx
236 sub %edx, %eax
237 ret
242 /*
243 * The following cases will be handled by ashr_1
244 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
245 * n(15) n -15 0(15 +(n-15) - n) ashr_1
246 */
247 .p2align 4
248 LABEL(ashr_1_sse4_2):
249 pxor %xmm0, %xmm0
250 movdqa (%rdi), %xmm2
251 movdqa (%rsi), %xmm1
252 pcmpeqb %xmm1, %xmm0 /* Any null chars? */
253 pslldq $15, %xmm2 /* shift first string to align with second */
254 pcmpeqb %xmm1, %xmm2 /* compare 16 bytes for equality */
255 psubb %xmm0, %xmm2 /* packed sub of comparison results*/
256 pmovmskb %xmm2, %r9d
257 shr %cl, %edx /* adjust 0xffff for offset */
258 shr %cl, %r9d /* adjust for 16-byte offset */
259 sub %r9d, %edx
260 jnz LABEL(less32bytes_sse4_2)/* mismatch or null char seen */
261 movdqa (%rdi), %xmm3
262 UPDATE_STRNCMP_COUNTER
264 pxor %xmm0, %xmm0
265 mov $16, %rcx /* index for loads*/
266 mov $1, %r9d /* byte position left over from less32bytes case */
267 /*
268 * Setup %r10 value allows us to detect crossing a page boundary.
269 * When %r10 goes positive we have crossed a page boundary and
270 * need to do a nibble.
271 */
272 lea 1(%rdi), %r10
273 and $0xfff, %r10 /* offset into 4K page */
274 sub $0x1000, %r10 /* subtract 4K pagesize */
275 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
277 .p2align 4
278 LABEL(loop_ashr_1_use_sse4_2):
279 add $16, %r10
280 jg LABEL(nibble_ashr_1_use_sse4_2)
282 movdqa (%rdi, %rdx), %xmm0
283 palignr $1, -16(%rdi, %rdx), %xmm0
284 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
285 jbe LABEL(use_sse4_2_exit)
286 #ifdef USE_AS_STRNCMP
287 sub $16, %r11
288 jbe LABEL(strcmp_exitz_sse4_2)
289 #endif
291 add $16, %rdx
292 add $16, %r10
293 jg LABEL(nibble_ashr_1_use_sse4_2)
295 movdqa (%rdi, %rdx), %xmm0
296 palignr $1, -16(%rdi, %rdx), %xmm0
297 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
298 jbe LABEL(use_sse4_2_exit)
299 #ifdef USE_AS_STRNCMP
300 sub $16, %r11
301 jbe LABEL(strcmp_exitz_sse4_2)
302 #endif
303 add $16, %rdx
304 jmp LABEL(loop_ashr_1_use_sse4_2)
306 .p2align 4
307 LABEL(nibble_ashr_1_use_sse4_2):
308 sub $0x1000, %r10
309 movdqa -16(%rdi, %rdx), %xmm0
310 psrldq $1, %xmm0
311 pcmpistri $0x3a,%xmm0, %xmm0
312 #ifdef USE_AS_STRNCMP
313 cmp %r11, %rcx
314 jae LABEL(nibble_ashr_use_sse4_2_exit)
315 #endif
316 cmp $14, %ecx
317 ja LABEL(loop_ashr_1_use_sse4_2)
319 jmp LABEL(nibble_ashr_use_sse4_2_exit)
321 /*
322 * The following cases will be handled by ashr_2
323 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
324 * n(14~15) n -14 1(15 +(n-14) - n) ashr_2
325 */
326 .p2align 4
327 LABEL(ashr_2_sse4_2):
328 pxor %xmm0, %xmm0
329 movdqa (%rdi), %xmm2
330 movdqa (%rsi), %xmm1
331 pcmpeqb %xmm1, %xmm0
332 pslldq $14, %xmm2
333 pcmpeqb %xmm1, %xmm2
334 psubb %xmm0, %xmm2
335 pmovmskb %xmm2, %r9d
336 shr %cl, %edx
337 shr %cl, %r9d
338 sub %r9d, %edx
339 jnz LABEL(less32bytes_sse4_2)
340 movdqa (%rdi), %xmm3
341 UPDATE_STRNCMP_COUNTER
343 pxor %xmm0, %xmm0
344 mov $16, %rcx /* index for loads */
345 mov $2, %r9d /* byte position left over from less32bytes case */
346 /*
347 * Setup %r10 value allows us to detect crossing a page boundary.
348 * When %r10 goes positive we have crossed a page boundary and
349 * need to do a nibble.
350 */
351 lea 2(%rdi), %r10
352 and $0xfff, %r10 /* offset into 4K page */
353 sub $0x1000, %r10 /* subtract 4K pagesize */
354 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
356 .p2align 4
357 LABEL(loop_ashr_2_use_sse4_2):
358 add $16, %r10
359 jg LABEL(nibble_ashr_2_use_sse4_2)
361 movdqa (%rdi, %rdx), %xmm0
362 palignr $2, -16(%rdi, %rdx), %xmm0
363 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
364 jbe LABEL(use_sse4_2_exit)
365 #ifdef USE_AS_STRNCMP
366 sub $16, %r11
367 jbe LABEL(strcmp_exitz_sse4_2)
368 #endif
370 add $16, %rdx
371 add $16, %r10
372 jg LABEL(nibble_ashr_2_use_sse4_2)
374 movdqa (%rdi, %rdx), %xmm0
375 palignr $2, -16(%rdi, %rdx), %xmm0
376 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
377 jbe LABEL(use_sse4_2_exit)
378 #ifdef USE_AS_STRNCMP
379 sub $16, %r11
380 jbe LABEL(strcmp_exitz_sse4_2)
381 #endif
382 add $16, %rdx
383 jmp LABEL(loop_ashr_2_use_sse4_2)
385 .p2align 4
386 LABEL(nibble_ashr_2_use_sse4_2):
387 sub $0x1000, %r10
388 movdqa -16(%rdi, %rdx), %xmm0
389 psrldq $2, %xmm0
390 pcmpistri $0x3a,%xmm0, %xmm0
391 #ifdef USE_AS_STRNCMP
392 cmp %r11, %rcx
393 jae LABEL(nibble_ashr_use_sse4_2_exit)
394 #endif
395 cmp $13, %ecx
396 ja LABEL(loop_ashr_2_use_sse4_2)
398 jmp LABEL(nibble_ashr_use_sse4_2_exit)
400 /*
401 * The following cases will be handled by ashr_3
402 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
403 * n(13~15) n -13 2(15 +(n-13) - n) ashr_3
404 */
405 .p2align 4
406 LABEL(ashr_3_sse4_2):
407 pxor %xmm0, %xmm0
408 movdqa (%rdi), %xmm2
409 movdqa (%rsi), %xmm1
410 pcmpeqb %xmm1, %xmm0
411 pslldq $13, %xmm2
412 pcmpeqb %xmm1, %xmm2
413 psubb %xmm0, %xmm2
414 pmovmskb %xmm2, %r9d
415 shr %cl, %edx
416 shr %cl, %r9d
417 sub %r9d, %edx
418 jnz LABEL(less32bytes_sse4_2)
419 movdqa (%rdi), %xmm3
421 UPDATE_STRNCMP_COUNTER
423 pxor %xmm0, %xmm0
424 mov $16, %rcx /* index for loads */
425 mov $3, %r9d /* byte position left over from less32bytes case */
426 /*
427 * Setup %r10 value allows us to detect crossing a page boundary.
428 * When %r10 goes positive we have crossed a page boundary and
429 * need to do a nibble.
430 */
431 lea 3(%rdi), %r10
432 and $0xfff, %r10 /* offset into 4K page */
433 sub $0x1000, %r10 /* subtract 4K pagesize */
434 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
436 LABEL(loop_ashr_3_use_sse4_2):
437 add $16, %r10
438 jg LABEL(nibble_ashr_3_use_sse4_2)
440 movdqa (%rdi, %rdx), %xmm0
441 palignr $3, -16(%rdi, %rdx), %xmm0
442 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
443 jbe LABEL(use_sse4_2_exit)
444 #ifdef USE_AS_STRNCMP
445 sub $16, %r11
446 jbe LABEL(strcmp_exitz_sse4_2)
447 #endif
449 add $16, %rdx
450 add $16, %r10
451 jg LABEL(nibble_ashr_3_use_sse4_2)
453 movdqa (%rdi, %rdx), %xmm0
454 palignr $3, -16(%rdi, %rdx), %xmm0
455 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
456 jbe LABEL(use_sse4_2_exit)
457 #ifdef USE_AS_STRNCMP
458 sub $16, %r11
459 jbe LABEL(strcmp_exitz_sse4_2)
460 #endif
461 add $16, %rdx
462 jmp LABEL(loop_ashr_3_use_sse4_2)
464 .p2align 4
465 LABEL(nibble_ashr_3_use_sse4_2):
466 sub $0x1000, %r10
467 movdqa -16(%rdi, %rdx), %xmm0
468 psrldq $3, %xmm0
469 pcmpistri $0x3a,%xmm0, %xmm0
470 #ifdef USE_AS_STRNCMP
471 cmp %r11, %rcx
472 jae LABEL(nibble_ashr_use_sse4_2_exit)
473 #endif
474 cmp $12, %ecx
475 ja LABEL(loop_ashr_3_use_sse4_2)
477 jmp LABEL(nibble_ashr_use_sse4_2_exit)
479 /*
480 * The following cases will be handled by ashr_4
481 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
482 * n(12~15) n -12 3(15 +(n-12) - n) ashr_4
483 */
484 .p2align 4
485 LABEL(ashr_4_sse4_2):
486 pxor %xmm0, %xmm0
487 movdqa (%rdi), %xmm2
488 movdqa (%rsi), %xmm1
489 pcmpeqb %xmm1, %xmm0
490 pslldq $12, %xmm2
491 pcmpeqb %xmm1, %xmm2
492 psubb %xmm0, %xmm2
493 pmovmskb %xmm2, %r9d
494 shr %cl, %edx
495 shr %cl, %r9d
496 sub %r9d, %edx
497 jnz LABEL(less32bytes_sse4_2)
498 movdqa (%rdi), %xmm3
500 UPDATE_STRNCMP_COUNTER
502 pxor %xmm0, %xmm0
503 mov $16, %rcx /* index for loads */
504 mov $4, %r9d /* byte position left over from less32bytes case */
505 /*
506 * Setup %r10 value allows us to detect crossing a page boundary.
507 * When %r10 goes positive we have crossed a page boundary and
508 * need to do a nibble.
509 */
510 lea 4(%rdi), %r10
511 and $0xfff, %r10 /* offset into 4K page */
512 sub $0x1000, %r10 /* subtract 4K pagesize */
513 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
515 .p2align 4
516 LABEL(loop_ashr_4_use_sse4_2):
517 add $16, %r10
518 jg LABEL(nibble_ashr_4_use_sse4_2)
520 movdqa (%rdi, %rdx), %xmm0
521 palignr $4, -16(%rdi, %rdx), %xmm0
522 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
523 jbe LABEL(use_sse4_2_exit)
524 #ifdef USE_AS_STRNCMP
525 sub $16, %r11
526 jbe LABEL(strcmp_exitz_sse4_2)
527 #endif
529 add $16, %rdx
530 add $16, %r10
531 jg LABEL(nibble_ashr_4_use_sse4_2)
533 movdqa (%rdi, %rdx), %xmm0
534 palignr $4, -16(%rdi, %rdx), %xmm0
535 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
536 jbe LABEL(use_sse4_2_exit)
537 #ifdef USE_AS_STRNCMP
538 sub $16, %r11
539 jbe LABEL(strcmp_exitz_sse4_2)
540 #endif
541 add $16, %rdx
542 jmp LABEL(loop_ashr_4_use_sse4_2)
544 .p2align 4
545 LABEL(nibble_ashr_4_use_sse4_2):
546 sub $0x1000, %r10
547 movdqa -16(%rdi, %rdx), %xmm0
548 psrldq $4, %xmm0
549 pcmpistri $0x3a,%xmm0, %xmm0
550 #ifdef USE_AS_STRNCMP
551 cmp %r11, %rcx
552 jae LABEL(nibble_ashr_use_sse4_2_exit)
553 #endif
554 cmp $11, %ecx
555 ja LABEL(loop_ashr_4_use_sse4_2)
557 jmp LABEL(nibble_ashr_use_sse4_2_exit)
559 /*
560 * The following cases will be handled by ashr_5
561 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
562 * n(11~15) n - 11 4(15 +(n-11) - n) ashr_5
563 */
564 .p2align 4
565 LABEL(ashr_5_sse4_2):
566 pxor %xmm0, %xmm0
567 movdqa (%rdi), %xmm2
568 movdqa (%rsi), %xmm1
569 pcmpeqb %xmm1, %xmm0
570 pslldq $11, %xmm2
571 pcmpeqb %xmm1, %xmm2
572 psubb %xmm0, %xmm2
573 pmovmskb %xmm2, %r9d
574 shr %cl, %edx
575 shr %cl, %r9d
576 sub %r9d, %edx
577 jnz LABEL(less32bytes_sse4_2)
578 movdqa (%rdi), %xmm3
580 UPDATE_STRNCMP_COUNTER
582 pxor %xmm0, %xmm0
583 mov $16, %rcx /* index for loads */
584 mov $5, %r9d /* byte position left over from less32bytes case */
585 /*
586 * Setup %r10 value allows us to detect crossing a page boundary.
587 * When %r10 goes positive we have crossed a page boundary and
588 * need to do a nibble.
589 */
590 lea 5(%rdi), %r10
591 and $0xfff, %r10 /* offset into 4K page */
592 sub $0x1000, %r10 /* subtract 4K pagesize */
593 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
595 .p2align 4
596 LABEL(loop_ashr_5_use_sse4_2):
597 add $16, %r10
598 jg LABEL(nibble_ashr_5_use_sse4_2)
600 movdqa (%rdi, %rdx), %xmm0
601 palignr $5, -16(%rdi, %rdx), %xmm0
602 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
603 jbe LABEL(use_sse4_2_exit)
604 #ifdef USE_AS_STRNCMP
605 sub $16, %r11
606 jbe LABEL(strcmp_exitz_sse4_2)
607 #endif
609 add $16, %rdx
610 add $16, %r10
611 jg LABEL(nibble_ashr_5_use_sse4_2)
613 movdqa (%rdi, %rdx), %xmm0
615 palignr $5, -16(%rdi, %rdx), %xmm0
616 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
617 jbe LABEL(use_sse4_2_exit)
618 #ifdef USE_AS_STRNCMP
619 sub $16, %r11
620 jbe LABEL(strcmp_exitz_sse4_2)
621 #endif
622 add $16, %rdx
623 jmp LABEL(loop_ashr_5_use_sse4_2)
625 .p2align 4
626 LABEL(nibble_ashr_5_use_sse4_2):
627 sub $0x1000, %r10
628 movdqa -16(%rdi, %rdx), %xmm0
629 psrldq $5, %xmm0
630 pcmpistri $0x3a,%xmm0, %xmm0
631 #ifdef USE_AS_STRNCMP
632 cmp %r11, %rcx
633 jae LABEL(nibble_ashr_use_sse4_2_exit)
634 #endif
635 cmp $10, %ecx
636 ja LABEL(loop_ashr_5_use_sse4_2)
638 jmp LABEL(nibble_ashr_use_sse4_2_exit)
640 /*
641 * The following cases will be handled by ashr_6
642 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
643 * n(10~15) n - 10 5(15 +(n-10) - n) ashr_6
644 */
645 .p2align 4
646 LABEL(ashr_6_sse4_2):
647 pxor %xmm0, %xmm0
648 movdqa (%rdi), %xmm2
649 movdqa (%rsi), %xmm1
650 pcmpeqb %xmm1, %xmm0
651 pslldq $10, %xmm2
652 pcmpeqb %xmm1, %xmm2
653 psubb %xmm0, %xmm2
654 pmovmskb %xmm2, %r9d
655 shr %cl, %edx
656 shr %cl, %r9d
657 sub %r9d, %edx
658 jnz LABEL(less32bytes_sse4_2)
659 movdqa (%rdi), %xmm3
661 UPDATE_STRNCMP_COUNTER
663 pxor %xmm0, %xmm0
664 mov $16, %rcx /* index for loads */
665 mov $6, %r9d /* byte position left over from less32bytes case */
666 /*
667 * Setup %r10 value allows us to detect crossing a page boundary.
668 * When %r10 goes positive we have crossed a page boundary and
669 * need to do a nibble.
670 */
671 lea 6(%rdi), %r10
672 and $0xfff, %r10 /* offset into 4K page */
673 sub $0x1000, %r10 /* subtract 4K pagesize */
674 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
676 .p2align 4
677 LABEL(loop_ashr_6_use_sse4_2):
678 add $16, %r10
679 jg LABEL(nibble_ashr_6_use_sse4_2)
681 movdqa (%rdi, %rdx), %xmm0
682 palignr $6, -16(%rdi, %rdx), %xmm0
683 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
684 jbe LABEL(use_sse4_2_exit)
685 #ifdef USE_AS_STRNCMP
686 sub $16, %r11
687 jbe LABEL(strcmp_exitz_sse4_2)
688 #endif
690 add $16, %rdx
691 add $16, %r10
692 jg LABEL(nibble_ashr_6_use_sse4_2)
694 movdqa (%rdi, %rdx), %xmm0
695 palignr $6, -16(%rdi, %rdx), %xmm0
696 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
697 jbe LABEL(use_sse4_2_exit)
698 #ifdef USE_AS_STRNCMP
699 sub $16, %r11
700 jbe LABEL(strcmp_exitz_sse4_2)
701 #endif
702 add $16, %rdx
703 jmp LABEL(loop_ashr_6_use_sse4_2)
705 .p2align 4
706 LABEL(nibble_ashr_6_use_sse4_2):
707 sub $0x1000, %r10
708 movdqa -16(%rdi, %rdx), %xmm0
709 psrldq $6, %xmm0
710 pcmpistri $0x3a,%xmm0, %xmm0
711 #ifdef USE_AS_STRNCMP
712 cmp %r11, %rcx
713 jae LABEL(nibble_ashr_use_sse4_2_exit)
714 #endif
715 cmp $9, %ecx
716 ja LABEL(loop_ashr_6_use_sse4_2)
718 jmp LABEL(nibble_ashr_use_sse4_2_exit)
720 /*
721 * The following cases will be handled by ashr_7
722 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
723 * n(9~15) n - 9 6(15 +(n - 9) - n) ashr_7
724 */
725 .p2align 4
726 LABEL(ashr_7_sse4_2):
727 pxor %xmm0, %xmm0
728 movdqa (%rdi), %xmm2
729 movdqa (%rsi), %xmm1
730 pcmpeqb %xmm1, %xmm0
731 pslldq $9, %xmm2
732 pcmpeqb %xmm1, %xmm2
733 psubb %xmm0, %xmm2
734 pmovmskb %xmm2, %r9d
735 shr %cl, %edx
736 shr %cl, %r9d
737 sub %r9d, %edx
738 jnz LABEL(less32bytes_sse4_2)
739 movdqa (%rdi), %xmm3
741 UPDATE_STRNCMP_COUNTER
743 pxor %xmm0, %xmm0
744 mov $16, %rcx /* index for loads */
745 mov $7, %r9d /* byte position left over from less32bytes case */
746 /*
747 * Setup %r10 value allows us to detect crossing a page boundary.
748 * When %r10 goes positive we have crossed a page boundary and
749 * need to do a nibble.
750 */
751 lea 7(%rdi), %r10
752 and $0xfff, %r10 /* offset into 4K page */
753 sub $0x1000, %r10 /* subtract 4K pagesize */
754 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
756 .p2align 4
757 LABEL(loop_ashr_7_use_sse4_2):
758 add $16, %r10
759 jg LABEL(nibble_ashr_7_use_sse4_2)
761 movdqa (%rdi, %rdx), %xmm0
762 palignr $7, -16(%rdi, %rdx), %xmm0
763 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
764 jbe LABEL(use_sse4_2_exit)
765 #ifdef USE_AS_STRNCMP
766 sub $16, %r11
767 jbe LABEL(strcmp_exitz_sse4_2)
768 #endif
770 add $16, %rdx
771 add $16, %r10
772 jg LABEL(nibble_ashr_7_use_sse4_2)
774 movdqa (%rdi, %rdx), %xmm0
775 palignr $7, -16(%rdi, %rdx), %xmm0
776 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
777 jbe LABEL(use_sse4_2_exit)
778 #ifdef USE_AS_STRNCMP
779 sub $16, %r11
780 jbe LABEL(strcmp_exitz_sse4_2)
781 #endif
782 add $16, %rdx
783 jmp LABEL(loop_ashr_7_use_sse4_2)
785 .p2align 4
786 LABEL(nibble_ashr_7_use_sse4_2):
787 sub $0x1000, %r10
788 movdqa -16(%rdi, %rdx), %xmm0
789 psrldq $7, %xmm0
790 pcmpistri $0x3a,%xmm0, %xmm0
791 #ifdef USE_AS_STRNCMP
792 cmp %r11, %rcx
793 jae LABEL(nibble_ashr_use_sse4_2_exit)
794 #endif
795 cmp $8, %ecx
796 ja LABEL(loop_ashr_7_use_sse4_2)
798 jmp LABEL(nibble_ashr_use_sse4_2_exit)
800 /*
801 * The following cases will be handled by ashr_8
802 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
803 * n(8~15) n - 8 7(15 +(n - 8) - n) ashr_8
804 */
805 .p2align 4
806 LABEL(ashr_8_sse4_2):
807 pxor %xmm0, %xmm0
808 movdqa (%rdi), %xmm2
809 movdqa (%rsi), %xmm1
810 pcmpeqb %xmm1, %xmm0
811 pslldq $8, %xmm2
812 pcmpeqb %xmm1, %xmm2
813 psubb %xmm0, %xmm2
814 pmovmskb %xmm2, %r9d
815 shr %cl, %edx
816 shr %cl, %r9d
817 sub %r9d, %edx
818 jnz LABEL(less32bytes_sse4_2)
819 movdqa (%rdi), %xmm3
821 UPDATE_STRNCMP_COUNTER
823 pxor %xmm0, %xmm0
824 mov $16, %rcx /* index for loads */
825 mov $8, %r9d /* byte position left over from less32bytes case */
826 /*
827 * Setup %r10 value allows us to detect crossing a page boundary.
828 * When %r10 goes positive we have crossed a page boundary and
829 * need to do a nibble.
830 */
831 lea 8(%rdi), %r10
832 and $0xfff, %r10 /* offset into 4K page */
833 sub $0x1000, %r10 /* subtract 4K pagesize */
834 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
836 .p2align 4
837 LABEL(loop_ashr_8_use_sse4_2):
838 add $16, %r10
839 jg LABEL(nibble_ashr_8_use_sse4_2)
841 movdqa (%rdi, %rdx), %xmm0
842 palignr $8, -16(%rdi, %rdx), %xmm0
843 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
844 jbe LABEL(use_sse4_2_exit)
845 #ifdef USE_AS_STRNCMP
846 sub $16, %r11
847 jbe LABEL(strcmp_exitz_sse4_2)
848 #endif
850 add $16, %rdx
851 add $16, %r10
852 jg LABEL(nibble_ashr_8_use_sse4_2)
854 movdqa (%rdi, %rdx), %xmm0
855 palignr $8, -16(%rdi, %rdx), %xmm0
856 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
857 jbe LABEL(use_sse4_2_exit)
858 #ifdef USE_AS_STRNCMP
859 sub $16, %r11
860 jbe LABEL(strcmp_exitz_sse4_2)
861 #endif
862 add $16, %rdx
863 jmp LABEL(loop_ashr_8_use_sse4_2)
865 .p2align 4
866 LABEL(nibble_ashr_8_use_sse4_2):
867 sub $0x1000, %r10
868 movdqa -16(%rdi, %rdx), %xmm0
869 psrldq $8, %xmm0
870 pcmpistri $0x3a,%xmm0, %xmm0
871 #ifdef USE_AS_STRNCMP
872 cmp %r11, %rcx
873 jae LABEL(nibble_ashr_use_sse4_2_exit)
874 #endif
875 cmp $7, %ecx
876 ja LABEL(loop_ashr_8_use_sse4_2)
878 jmp LABEL(nibble_ashr_use_sse4_2_exit)
880 /*
881 * The following cases will be handled by ashr_9
882 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
883 * n(7~15) n - 7 8(15 +(n - 7) - n) ashr_9
884 */
885 .p2align 4
886 LABEL(ashr_9_sse4_2):
887 pxor %xmm0, %xmm0
888 movdqa (%rdi), %xmm2
889 movdqa (%rsi), %xmm1
890 pcmpeqb %xmm1, %xmm0
891 pslldq $7, %xmm2
892 pcmpeqb %xmm1, %xmm2
893 psubb %xmm0, %xmm2
894 pmovmskb %xmm2, %r9d
895 shr %cl, %edx
896 shr %cl, %r9d
897 sub %r9d, %edx
898 jnz LABEL(less32bytes_sse4_2)
899 movdqa (%rdi), %xmm3
901 UPDATE_STRNCMP_COUNTER
903 pxor %xmm0, %xmm0
904 mov $16, %rcx /* index for loads */
905 mov $9, %r9d /* byte position left over from less32bytes case */
906 /*
907 * Setup %r10 value allows us to detect crossing a page boundary.
908 * When %r10 goes positive we have crossed a page boundary and
909 * need to do a nibble.
910 */
911 lea 9(%rdi), %r10
912 and $0xfff, %r10 /* offset into 4K page */
913 sub $0x1000, %r10 /* subtract 4K pagesize */
914 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
916 .p2align 4
917 LABEL(loop_ashr_9_use_sse4_2):
918 add $16, %r10
919 jg LABEL(nibble_ashr_9_use_sse4_2)
921 movdqa (%rdi, %rdx), %xmm0
923 palignr $9, -16(%rdi, %rdx), %xmm0
924 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
925 jbe LABEL(use_sse4_2_exit)
926 #ifdef USE_AS_STRNCMP
927 sub $16, %r11
928 jbe LABEL(strcmp_exitz_sse4_2)
929 #endif
931 add $16, %rdx
932 add $16, %r10
933 jg LABEL(nibble_ashr_9_use_sse4_2)
935 movdqa (%rdi, %rdx), %xmm0
936 palignr $9, -16(%rdi, %rdx), %xmm0
937 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
938 jbe LABEL(use_sse4_2_exit)
939 #ifdef USE_AS_STRNCMP
940 sub $16, %r11
941 jbe LABEL(strcmp_exitz_sse4_2)
942 #endif
943 add $16, %rdx
944 jmp LABEL(loop_ashr_9_use_sse4_2)
946 .p2align 4
947 LABEL(nibble_ashr_9_use_sse4_2):
948 sub $0x1000, %r10
949 movdqa -16(%rdi, %rdx), %xmm0
950 psrldq $9, %xmm0
951 pcmpistri $0x3a,%xmm0, %xmm0
952 #ifdef USE_AS_STRNCMP
953 cmp %r11, %rcx
954 jae LABEL(nibble_ashr_use_sse4_2_exit)
955 #endif
956 cmp $6, %ecx
957 ja LABEL(loop_ashr_9_use_sse4_2)
959 jmp LABEL(nibble_ashr_use_sse4_2_exit)
961 /*
962 * The following cases will be handled by ashr_10
963 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
964 * n(6~15) n - 6 9(15 +(n - 6) - n) ashr_10
965 */
966 .p2align 4
967 LABEL(ashr_10_sse4_2):
968 pxor %xmm0, %xmm0
969 movdqa (%rdi), %xmm2
970 movdqa (%rsi), %xmm1
971 pcmpeqb %xmm1, %xmm0
972 pslldq $6, %xmm2
973 pcmpeqb %xmm1, %xmm2
974 psubb %xmm0, %xmm2
975 pmovmskb %xmm2, %r9d
976 shr %cl, %edx
977 shr %cl, %r9d
978 sub %r9d, %edx
979 jnz LABEL(less32bytes_sse4_2)
980 movdqa (%rdi), %xmm3
982 UPDATE_STRNCMP_COUNTER
984 pxor %xmm0, %xmm0
985 mov $16, %rcx /* index for loads */
986 mov $10, %r9d /* byte position left over from less32bytes case */
987 /*
988 * Setup %r10 value allows us to detect crossing a page boundary.
989 * When %r10 goes positive we have crossed a page boundary and
990 * need to do a nibble.
991 */
992 lea 10(%rdi), %r10
993 and $0xfff, %r10 /* offset into 4K page */
994 sub $0x1000, %r10 /* subtract 4K pagesize */
995 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
997 .p2align 4
998 LABEL(loop_ashr_10_use_sse4_2):
999 add $16, %r10
1000 jg LABEL(nibble_ashr_10_use_sse4_2)
1002 movdqa (%rdi, %rdx), %xmm0
1003 palignr $10, -16(%rdi, %rdx), %xmm0
1004 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1005 jbe LABEL(use_sse4_2_exit)
1006 #ifdef USE_AS_STRNCMP
1007 sub $16, %r11
1008 jbe LABEL(strcmp_exitz_sse4_2)
1009 #endif
1011 add $16, %rdx
1012 add $16, %r10
1013 jg LABEL(nibble_ashr_10_use_sse4_2)
1015 movdqa (%rdi, %rdx), %xmm0
1016 palignr $10, -16(%rdi, %rdx), %xmm0
1017 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1018 jbe LABEL(use_sse4_2_exit)
1019 #ifdef USE_AS_STRNCMP
1020 sub $16, %r11
1021 jbe LABEL(strcmp_exitz_sse4_2)
1022 #endif
1023 add $16, %rdx
1024 jmp LABEL(loop_ashr_10_use_sse4_2)
1026 .p2align 4
1027 LABEL(nibble_ashr_10_use_sse4_2):
1028 sub $0x1000, %r10
1029 movdqa -16(%rdi, %rdx), %xmm0
1030 psrldq $10, %xmm0
1031 pcmpistri $0x3a,%xmm0, %xmm0
1032 #ifdef USE_AS_STRNCMP
1033 cmp %r11, %rcx
1034 jae LABEL(nibble_ashr_use_sse4_2_exit)
1035 #endif
1036 cmp $5, %ecx
1037 ja LABEL(loop_ashr_10_use_sse4_2)
1039 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1041 /*
1042 * The following cases will be handled by ashr_11
1043 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1044 * n(5~15) n - 5 10(15 +(n - 5) - n) ashr_11
1045 */
1046 .p2align 4
1047 LABEL(ashr_11_sse4_2):
1048 pxor %xmm0, %xmm0
1049 movdqa (%rdi), %xmm2
1050 movdqa (%rsi), %xmm1
1051 pcmpeqb %xmm1, %xmm0
1052 pslldq $5, %xmm2
1053 pcmpeqb %xmm1, %xmm2
1054 psubb %xmm0, %xmm2
1055 pmovmskb %xmm2, %r9d
1056 shr %cl, %edx
1057 shr %cl, %r9d
1058 sub %r9d, %edx
1059 jnz LABEL(less32bytes_sse4_2)
1060 movdqa (%rdi), %xmm3
1062 UPDATE_STRNCMP_COUNTER
1064 pxor %xmm0, %xmm0
1065 mov $16, %rcx /* index for loads */
1066 mov $11, %r9d /* byte position left over from less32bytes case */
1067 /*
1068 * Setup %r10 value allows us to detect crossing a page boundary.
1069 * When %r10 goes positive we have crossed a page boundary and
1070 * need to do a nibble.
1071 */
1072 lea 11(%rdi), %r10
1073 and $0xfff, %r10 /* offset into 4K page */
1074 sub $0x1000, %r10 /* subtract 4K pagesize */
1075 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1077 .p2align 4
1078 LABEL(loop_ashr_11_use_sse4_2):
1079 add $16, %r10
1080 jg LABEL(nibble_ashr_11_use_sse4_2)
1082 movdqa (%rdi, %rdx), %xmm0
1083 palignr $11, -16(%rdi, %rdx), %xmm0
1084 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1085 jbe LABEL(use_sse4_2_exit)
1086 #ifdef USE_AS_STRNCMP
1087 sub $16, %r11
1088 jbe LABEL(strcmp_exitz_sse4_2)
1089 #endif
1091 add $16, %rdx
1092 add $16, %r10
1093 jg LABEL(nibble_ashr_11_use_sse4_2)
1095 movdqa (%rdi, %rdx), %xmm0
1096 palignr $11, -16(%rdi, %rdx), %xmm0
1097 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1098 jbe LABEL(use_sse4_2_exit)
1099 #ifdef USE_AS_STRNCMP
1100 sub $16, %r11
1101 jbe LABEL(strcmp_exitz_sse4_2)
1102 #endif
1103 add $16, %rdx
1104 jmp LABEL(loop_ashr_11_use_sse4_2)
1106 .p2align 4
1107 LABEL(nibble_ashr_11_use_sse4_2):
1108 sub $0x1000, %r10
1109 movdqa -16(%rdi, %rdx), %xmm0
1110 psrldq $11, %xmm0
1111 pcmpistri $0x3a,%xmm0, %xmm0
1112 #ifdef USE_AS_STRNCMP
1113 cmp %r11, %rcx
1114 jae LABEL(nibble_ashr_use_sse4_2_exit)
1115 #endif
1116 cmp $4, %ecx
1117 ja LABEL(loop_ashr_11_use_sse4_2)
1119 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1121 /*
1122 * The following cases will be handled by ashr_12
1123 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1124 * n(4~15) n - 4 11(15 +(n - 4) - n) ashr_12
1125 */
1126 .p2align 4
1127 LABEL(ashr_12_sse4_2):
1128 pxor %xmm0, %xmm0
1129 movdqa (%rdi), %xmm2
1130 movdqa (%rsi), %xmm1
1131 pcmpeqb %xmm1, %xmm0
1132 pslldq $4, %xmm2
1133 pcmpeqb %xmm1, %xmm2
1134 psubb %xmm0, %xmm2
1135 pmovmskb %xmm2, %r9d
1136 shr %cl, %edx
1137 shr %cl, %r9d
1138 sub %r9d, %edx
1139 jnz LABEL(less32bytes_sse4_2)
1140 movdqa (%rdi), %xmm3
1142 UPDATE_STRNCMP_COUNTER
1144 pxor %xmm0, %xmm0
1145 mov $16, %rcx /* index for loads */
1146 mov $12, %r9d /* byte position left over from less32bytes case */
1147 /*
1148 * Setup %r10 value allows us to detect crossing a page boundary.
1149 * When %r10 goes positive we have crossed a page boundary and
1150 * need to do a nibble.
1151 */
1152 lea 12(%rdi), %r10
1153 and $0xfff, %r10 /* offset into 4K page */
1154 sub $0x1000, %r10 /* subtract 4K pagesize */
1155 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1157 .p2align 4
1158 LABEL(loop_ashr_12_use_sse4_2):
1159 add $16, %r10
1160 jg LABEL(nibble_ashr_12_use_sse4_2)
1162 movdqa (%rdi, %rdx), %xmm0
1163 palignr $12, -16(%rdi, %rdx), %xmm0
1164 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1165 jbe LABEL(use_sse4_2_exit)
1166 #ifdef USE_AS_STRNCMP
1167 sub $16, %r11
1168 jbe LABEL(strcmp_exitz_sse4_2)
1169 #endif
1171 add $16, %rdx
1172 add $16, %r10
1173 jg LABEL(nibble_ashr_12_use_sse4_2)
1175 movdqa (%rdi, %rdx), %xmm0
1176 palignr $12, -16(%rdi, %rdx), %xmm0
1177 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1178 jbe LABEL(use_sse4_2_exit)
1179 #ifdef USE_AS_STRNCMP
1180 sub $16, %r11
1181 jbe LABEL(strcmp_exitz_sse4_2)
1182 #endif
1183 add $16, %rdx
1184 jmp LABEL(loop_ashr_12_use_sse4_2)
1186 .p2align 4
1187 LABEL(nibble_ashr_12_use_sse4_2):
1188 sub $0x1000, %r10
1189 movdqa -16(%rdi, %rdx), %xmm0
1190 psrldq $12, %xmm0
1191 pcmpistri $0x3a,%xmm0, %xmm0
1192 #ifdef USE_AS_STRNCMP
1193 cmp %r11, %rcx
1194 jae LABEL(nibble_ashr_use_sse4_2_exit)
1195 #endif
1196 cmp $3, %ecx
1197 ja LABEL(loop_ashr_12_use_sse4_2)
1199 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1201 /*
1202 * The following cases will be handled by ashr_13
1203 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1204 * n(3~15) n - 3 12(15 +(n - 3) - n) ashr_13
1205 */
1206 .p2align 4
1207 LABEL(ashr_13_sse4_2):
1208 pxor %xmm0, %xmm0
1209 movdqa (%rdi), %xmm2
1210 movdqa (%rsi), %xmm1
1211 pcmpeqb %xmm1, %xmm0
1212 pslldq $3, %xmm2
1213 pcmpeqb %xmm1, %xmm2
1214 psubb %xmm0, %xmm2
1215 pmovmskb %xmm2, %r9d
1216 shr %cl, %edx
1217 shr %cl, %r9d
1218 sub %r9d, %edx
1219 jnz LABEL(less32bytes_sse4_2)
1220 movdqa (%rdi), %xmm3
1222 UPDATE_STRNCMP_COUNTER
1224 pxor %xmm0, %xmm0
1225 mov $16, %rcx /* index for loads */
1226 mov $13, %r9d /* byte position left over from less32bytes case */
1227 /*
1228 * Setup %r10 value allows us to detect crossing a page boundary.
1229 * When %r10 goes positive we have crossed a page boundary and
1230 * need to do a nibble.
1231 */
1232 lea 13(%rdi), %r10
1233 and $0xfff, %r10 /* offset into 4K page */
1234 sub $0x1000, %r10 /* subtract 4K pagesize */
1236 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1238 .p2align 4
1239 LABEL(loop_ashr_13_use_sse4_2):
1240 add $16, %r10
1241 jg LABEL(nibble_ashr_13_use_sse4_2)
1243 movdqa (%rdi, %rdx), %xmm0
1244 palignr $13, -16(%rdi, %rdx), %xmm0
1245 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1246 jbe LABEL(use_sse4_2_exit)
1247 #ifdef USE_AS_STRNCMP
1248 sub $16, %r11
1249 jbe LABEL(strcmp_exitz_sse4_2)
1250 #endif
1252 add $16, %rdx
1253 add $16, %r10
1254 jg LABEL(nibble_ashr_13_use_sse4_2)
1256 movdqa (%rdi, %rdx), %xmm0
1257 palignr $13, -16(%rdi, %rdx), %xmm0
1258 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1259 jbe LABEL(use_sse4_2_exit)
1260 #ifdef USE_AS_STRNCMP
1261 sub $16, %r11
1262 jbe LABEL(strcmp_exitz_sse4_2)
1263 #endif
1264 add $16, %rdx
1265 jmp LABEL(loop_ashr_13_use_sse4_2)
1267 .p2align 4
1268 LABEL(nibble_ashr_13_use_sse4_2):
1269 sub $0x1000, %r10
1270 movdqa -16(%rdi, %rdx), %xmm0
1271 psrldq $13, %xmm0
1272 pcmpistri $0x3a,%xmm0, %xmm0
1273 #ifdef USE_AS_STRNCMP
1274 cmp %r11, %rcx
1275 jae LABEL(nibble_ashr_use_sse4_2_exit)
1276 #endif
1277 cmp $2, %ecx
1278 ja LABEL(loop_ashr_13_use_sse4_2)
1280 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1282 /*
1283 * The following cases will be handled by ashr_14
1284 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1285 * n(2~15) n - 2 13(15 +(n - 2) - n) ashr_14
1286 */
1287 .p2align 4
1288 LABEL(ashr_14_sse4_2):
1289 pxor %xmm0, %xmm0
1290 movdqa (%rdi), %xmm2
1291 movdqa (%rsi), %xmm1
1292 pcmpeqb %xmm1, %xmm0
1293 pslldq $2, %xmm2
1294 pcmpeqb %xmm1, %xmm2
1295 psubb %xmm0, %xmm2
1296 pmovmskb %xmm2, %r9d
1297 shr %cl, %edx
1298 shr %cl, %r9d
1299 sub %r9d, %edx
1300 jnz LABEL(less32bytes_sse4_2)
1301 movdqa (%rdi), %xmm3
1303 UPDATE_STRNCMP_COUNTER
1305 pxor %xmm0, %xmm0
1306 mov $16, %rcx /* index for loads */
1307 mov $14, %r9d /* byte position left over from less32bytes case */
1308 /*
1309 * Setup %r10 value allows us to detect crossing a page boundary.
1310 * When %r10 goes positive we have crossed a page boundary and
1311 * need to do a nibble.
1312 */
1313 lea 14(%rdi), %r10
1314 and $0xfff, %r10 /* offset into 4K page */
1315 sub $0x1000, %r10 /* subtract 4K pagesize */
1317 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1319 .p2align 4
1320 LABEL(loop_ashr_14_use_sse4_2):
1321 add $16, %r10
1322 jg LABEL(nibble_ashr_14_use_sse4_2)
1324 movdqa (%rdi, %rdx), %xmm0
1325 palignr $14, -16(%rdi, %rdx), %xmm0
1326 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1327 jbe LABEL(use_sse4_2_exit)
1328 #ifdef USE_AS_STRNCMP
1329 sub $16, %r11
1330 jbe LABEL(strcmp_exitz_sse4_2)
1331 #endif
1333 add $16, %rdx
1334 add $16, %r10
1335 jg LABEL(nibble_ashr_14_use_sse4_2)
1337 movdqa (%rdi, %rdx), %xmm0
1338 palignr $14, -16(%rdi, %rdx), %xmm0
1339 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1340 jbe LABEL(use_sse4_2_exit)
1341 #ifdef USE_AS_STRNCMP
1342 sub $16, %r11
1343 jbe LABEL(strcmp_exitz_sse4_2)
1344 #endif
1345 add $16, %rdx
1346 jmp LABEL(loop_ashr_14_use_sse4_2)
1348 .p2align 4
1349 LABEL(nibble_ashr_14_use_sse4_2):
1350 sub $0x1000, %r10
1351 movdqa -16(%rdi, %rdx), %xmm0
1352 psrldq $14, %xmm0
1353 pcmpistri $0x3a,%xmm0, %xmm0
1354 #ifdef USE_AS_STRNCMP
1355 cmp %r11, %rcx
1356 jae LABEL(nibble_ashr_use_sse4_2_exit)
1357 #endif
1358 cmp $1, %ecx
1359 ja LABEL(loop_ashr_14_use_sse4_2)
1361 jmp LABEL(nibble_ashr_use_sse4_2_exit)
1363 /*
1364 * The following cases will be handled by ashr_15
1365 * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
1366 * n(1~15) n - 1 14(15 +(n - 1) - n) ashr_15
1367 */
1368 .p2align 4
1369 LABEL(ashr_15_sse4_2):
1370 pxor %xmm0, %xmm0
1371 movdqa (%rdi), %xmm2
1372 movdqa (%rsi), %xmm1
1373 pcmpeqb %xmm1, %xmm0
1374 pslldq $1, %xmm2
1375 pcmpeqb %xmm1, %xmm2
1376 psubb %xmm0, %xmm2
1377 pmovmskb %xmm2, %r9d
1378 shr %cl, %edx
1379 shr %cl, %r9d
1380 sub %r9d, %edx
1381 jnz LABEL(less32bytes_sse4_2)
1383 movdqa (%rdi), %xmm3
1385 UPDATE_STRNCMP_COUNTER
1387 pxor %xmm0, %xmm0
1388 mov $16, %rcx /* index for loads */
1389 mov $15, %r9d /* byte position left over from less32bytes case */
1390 /*
1391 * Setup %r10 value allows us to detect crossing a page boundary.
1392 * When %r10 goes positive we have crossed a page boundary and
1393 * need to do a nibble.
1394 */
1395 lea 15(%rdi), %r10
1396 and $0xfff, %r10 /* offset into 4K page */
1398 sub $0x1000, %r10 /* subtract 4K pagesize */
1400 mov %rcx, %rdx /* only for offset of sse4 instruction loop*/
1402 .p2align 4
1403 LABEL(loop_ashr_15_use_sse4_2):
1404 add $16, %r10
1405 jg LABEL(nibble_ashr_15_use_sse4_2)
1407 movdqa (%rdi, %rdx), %xmm0
1408 palignr $15, -16(%rdi, %rdx), %xmm0
1409 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1410 jbe LABEL(use_sse4_2_exit)
1411 #ifdef USE_AS_STRNCMP
1412 sub $16, %r11
1413 jbe LABEL(strcmp_exitz_sse4_2)
1414 #endif
1416 add $16, %rdx
1417 add $16, %r10
1418 jg LABEL(nibble_ashr_15_use_sse4_2)
1420 movdqa (%rdi, %rdx), %xmm0
1421 palignr $15, -16(%rdi, %rdx), %xmm0
1422 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1423 jbe LABEL(use_sse4_2_exit)
1424 #ifdef USE_AS_STRNCMP
1425 sub $16, %r11
1426 jbe LABEL(strcmp_exitz_sse4_2)
1427 #endif
1428 add $16, %rdx
1429 jmp LABEL(loop_ashr_15_use_sse4_2)
1431 .p2align 4
1432 LABEL(nibble_ashr_15_use_sse4_2):
1433 sub $0x1000, %r10
1434 movdqa -16(%rdi, %rdx), %xmm0
1435 psrldq $15, %xmm0
1436 pcmpistri $0x3a,%xmm0, %xmm0
1437 #ifdef USE_AS_STRNCMP
1438 cmp %r11, %rcx
1439 jae LABEL(nibble_ashr_use_sse4_2_exit)
1440 #endif
1441 cmp $0, %ecx
1442 ja LABEL(loop_ashr_15_use_sse4_2)
1444 LABEL(nibble_ashr_use_sse4_2_exit):
1445 pcmpistri $0x1a,(%rsi,%rdx), %xmm0
1446 .p2align 4
1447 LABEL(use_sse4_2_exit):
1448 jnc LABEL(strcmp_exitz_sse4_2)
1449 #ifdef USE_AS_STRNCMP
1450 sub %rcx, %r11
1451 jbe LABEL(strcmp_exitz_sse4_2)
1452 #endif
1453 add %rcx, %rdx
1454 lea -16(%rdi, %r9), %rdi
1455 movzbl (%rdi, %rdx), %eax
1456 movzbl (%rsi, %rdx), %edx
1457 test %r8d, %r8d
1458 jz LABEL(use_sse4_2_ret_sse4_2)
1459 xchg %eax, %edx
1460 LABEL(use_sse4_2_ret_sse4_2):
1461 sub %edx, %eax
1462 ret
1464 LABEL(less32bytes_sse4_2):
1465 lea (%rdi, %rax), %rdi /* locate the exact address for first operand(rdi) */
1466 lea (%rsi, %rcx), %rsi /* locate the exact address for second operand(rsi) */
1467 test %r8d, %r8d
1468 jz LABEL(ret_sse4_2)
1469 xchg %rsi, %rdi /* recover original order according to flag(%r8d) */
1471 .p2align 4
1472 LABEL(ret_sse4_2):
1473 LABEL(less16bytes_sse4_2):
1474 bsf %rdx, %rdx /* find and store bit index in %rdx */
1476 #ifdef USE_AS_STRNCMP
1477 sub %rdx, %r11
1478 jbe LABEL(strcmp_exitz_sse4_2)
1479 #endif
1480 movzbl (%rsi, %rdx), %ecx
1481 movzbl (%rdi, %rdx), %eax
1483 sub %ecx, %eax
1484 ret
1486 LABEL(strcmp_exitz_sse4_2):
1487 xor %eax, %eax
1488 ret
1490 .p2align 4
1491 LABEL(Byte0_sse4_2):
1492 movzx (%rsi), %ecx
1493 movzx (%rdi), %eax
1495 sub %ecx, %eax
1496 ret
1497 cfi_endproc
1498 .size STRCMP_SSE42, .-STRCMP_SSE42
1500 /* Put all SSE 4.2 functions together. */
1501 .section .rodata.sse4.2,"a",@progbits
1502 .p2align 3
1503 LABEL(unaligned_table_sse4_2):
1504 .int LABEL(ashr_1_sse4_2) - LABEL(unaligned_table_sse4_2)
1505 .int LABEL(ashr_2_sse4_2) - LABEL(unaligned_table_sse4_2)
1506 .int LABEL(ashr_3_sse4_2) - LABEL(unaligned_table_sse4_2)
1507 .int LABEL(ashr_4_sse4_2) - LABEL(unaligned_table_sse4_2)
1508 .int LABEL(ashr_5_sse4_2) - LABEL(unaligned_table_sse4_2)
1509 .int LABEL(ashr_6_sse4_2) - LABEL(unaligned_table_sse4_2)
1510 .int LABEL(ashr_7_sse4_2) - LABEL(unaligned_table_sse4_2)
1511 .int LABEL(ashr_8_sse4_2) - LABEL(unaligned_table_sse4_2)
1512 .int LABEL(ashr_9_sse4_2) - LABEL(unaligned_table_sse4_2)
1513 .int LABEL(ashr_10_sse4_2) - LABEL(unaligned_table_sse4_2)
1514 .int LABEL(ashr_11_sse4_2) - LABEL(unaligned_table_sse4_2)
1515 .int LABEL(ashr_12_sse4_2) - LABEL(unaligned_table_sse4_2)
1516 .int LABEL(ashr_13_sse4_2) - LABEL(unaligned_table_sse4_2)
1517 .int LABEL(ashr_14_sse4_2) - LABEL(unaligned_table_sse4_2)
1518 .int LABEL(ashr_15_sse4_2) - LABEL(unaligned_table_sse4_2)
1519 .int LABEL(ashr_0_sse4_2) - LABEL(unaligned_table_sse4_2)
1522 # undef ENTRY
1523 # define ENTRY(name) \
1524 .type STRCMP_SSE2, @function; \
1525 .align 16; \
1526 STRCMP_SSE2: cfi_startproc; \
1527 CALL_MCOUNT
1528 # undef END
1529 # define END(name) \
1530 cfi_endproc; .size STRCMP_SSE2, .-STRCMP_SSE2
1531 # undef libc_hidden_builtin_def
1532 /* It doesn't make sense to send libc-internal strcmp calls through a PLT.
1533 The speedup we get from using SSE4.2 instruction is likely eaten away
1534 by the indirect call in the PLT. */
1535 # define libc_hidden_builtin_def(name) \
1536 .globl __GI_STRCMP; __GI_STRCMP = STRCMP_SSE2
1537 #endif
1539 #include "../strcmp.S"