| blob | f2cb1dfc2f6c33a474398692ffc24479e2d90e31 |
1 /* Optimized memcmp implementation for POWER7/PowerPC32.
2 Copyright (C) 2010-2013 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
19 #include <sysdep.h>
20 #include <bp-sym.h>
21 #include <bp-asm.h>
23 /* int [r3] memcmp (const char *s1 [r3],
24 const char *s2 [r4],
25 size_t size [r5]) */
27 .machine power7
28 EALIGN (BP_SYM(memcmp),4,0)
29 CALL_MCOUNT
31 #define rTMP r0
32 #define rRTN r3
33 #define rSTR1 r3 /* first string arg */
34 #define rSTR2 r4 /* second string arg */
35 #define rN r5 /* max string length */
36 #define rWORD1 r6 /* current word in s1 */
37 #define rWORD2 r7 /* current word in s2 */
38 #define rWORD3 r8 /* next word in s1 */
39 #define rWORD4 r9 /* next word in s2 */
40 #define rWORD5 r10 /* next word in s1 */
41 #define rWORD6 r11 /* next word in s2 */
42 #define rBITDIF r12 /* bits that differ in s1 & s2 words */
43 #define rWORD7 r30 /* next word in s1 */
44 #define rWORD8 r31 /* next word in s2 */
46 xor rTMP,rSTR2,rSTR1
47 cmplwi cr6,rN,0
48 cmplwi cr1,rN,12
49 clrlwi. rTMP,rTMP,30
50 clrlwi rBITDIF,rSTR1,30
51 cmplwi cr5,rBITDIF,0
52 beq- cr6,L(zeroLength)
53 dcbt 0,rSTR1
54 dcbt 0,rSTR2
56 /* If less than 8 bytes or not aligned, use the unaligned
57 byte loop. */
59 blt cr1,L(bytealigned)
60 stwu 1,-64(1)
61 cfi_adjust_cfa_offset(64)
62 stw r31,48(1)
63 cfi_offset(31,(48-64))
64 stw r30,44(1)
65 cfi_offset(30,(44-64))
66 bne L(unaligned)
67 /* At this point we know both strings have the same alignment and the
68 compare length is at least 8 bytes. rBITDIF contains the low order
69 2 bits of rSTR1 and cr5 contains the result of the logical compare
70 of rBITDIF to 0. If rBITDIF == 0 then we are already word
71 aligned and can perform the word aligned loop.
73 Otherwise we know the two strings have the same alignment (but not
74 yet word aligned). So we force the string addresses to the next lower
75 word boundary and special case this first word using shift left to
76 eliminate bits preceeding the first byte. Since we want to join the
77 normal (word aligned) compare loop, starting at the second word,
78 we need to adjust the length (rN) and special case the loop
79 versioning for the first word. This insures that the loop count is
80 correct and the first word (shifted) is in the expected register pair. */
81 .align 4
82 L(samealignment):
83 clrrwi rSTR1,rSTR1,2
84 clrrwi rSTR2,rSTR2,2
85 beq cr5,L(Waligned)
86 add rN,rN,rBITDIF
87 slwi r11,rBITDIF,3
88 srwi rTMP,rN,4 /* Divide by 16 */
89 andi. rBITDIF,rN,12 /* Get the word remainder */
90 lwz rWORD1,0(rSTR1)
91 lwz rWORD2,0(rSTR2)
92 cmplwi cr1,rBITDIF,8
93 cmplwi cr7,rN,16
94 clrlwi rN,rN,30
95 beq L(dPs4)
96 mtctr rTMP
97 bgt cr1,L(dPs3)
98 beq cr1,L(dPs2)
100 /* Remainder is 4 */
101 .align 3
102 L(dsP1):
103 slw rWORD5,rWORD1,r11
104 slw rWORD6,rWORD2,r11
105 cmplw cr5,rWORD5,rWORD6
106 blt cr7,L(dP1x)
107 /* Do something useful in this cycle since we have to branch anyway. */
108 lwz rWORD1,4(rSTR1)
109 lwz rWORD2,4(rSTR2)
110 cmplw cr0,rWORD1,rWORD2
111 b L(dP1e)
112 /* Remainder is 8 */
113 .align 4
114 L(dPs2):
115 slw rWORD5,rWORD1,r11
116 slw rWORD6,rWORD2,r11
117 cmplw cr6,rWORD5,rWORD6
118 blt cr7,L(dP2x)
119 /* Do something useful in this cycle since we have to branch anyway. */
120 lwz rWORD7,4(rSTR1)
121 lwz rWORD8,4(rSTR2)
122 cmplw cr5,rWORD7,rWORD8
123 b L(dP2e)
124 /* Remainder is 12 */
125 .align 4
126 L(dPs3):
127 slw rWORD3,rWORD1,r11
128 slw rWORD4,rWORD2,r11
129 cmplw cr1,rWORD3,rWORD4
130 b L(dP3e)
131 /* Count is a multiple of 16, remainder is 0 */
132 .align 4
133 L(dPs4):
134 mtctr rTMP
135 slw rWORD1,rWORD1,r11
136 slw rWORD2,rWORD2,r11
137 cmplw cr0,rWORD1,rWORD2
138 b L(dP4e)
140 /* At this point we know both strings are word aligned and the
141 compare length is at least 8 bytes. */
142 .align 4
143 L(Waligned):
144 andi. rBITDIF,rN,12 /* Get the word remainder */
145 srwi rTMP,rN,4 /* Divide by 16 */
146 cmplwi cr1,rBITDIF,8
147 cmplwi cr7,rN,16
148 clrlwi rN,rN,30
149 beq L(dP4)
150 bgt cr1,L(dP3)
151 beq cr1,L(dP2)
153 /* Remainder is 4 */
154 .align 4
155 L(dP1):
156 mtctr rTMP
157 /* Normally we'd use rWORD7/rWORD8 here, but since we might exit early
158 (8-15 byte compare), we want to use only volatile registers. This
159 means we can avoid restoring non-volatile registers since we did not
160 change any on the early exit path. The key here is the non-early
161 exit path only cares about the condition code (cr5), not about which
162 register pair was used. */
163 lwz rWORD5,0(rSTR1)
164 lwz rWORD6,0(rSTR2)
165 cmplw cr5,rWORD5,rWORD6
166 blt cr7,L(dP1x)
167 lwz rWORD1,4(rSTR1)
168 lwz rWORD2,4(rSTR2)
169 cmplw cr0,rWORD1,rWORD2
170 L(dP1e):
171 lwz rWORD3,8(rSTR1)
172 lwz rWORD4,8(rSTR2)
173 cmplw cr1,rWORD3,rWORD4
174 lwz rWORD5,12(rSTR1)
175 lwz rWORD6,12(rSTR2)
176 cmplw cr6,rWORD5,rWORD6
177 bne cr5,L(dLcr5)
178 bne cr0,L(dLcr0)
180 lwzu rWORD7,16(rSTR1)
181 lwzu rWORD8,16(rSTR2)
182 bne cr1,L(dLcr1)
183 cmplw cr5,rWORD7,rWORD8
184 bdnz L(dLoop)
185 bne cr6,L(dLcr6)
186 lwz r30,44(1)
187 lwz r31,48(1)
188 .align 3
189 L(dP1x):
190 slwi. r12,rN,3
191 bne cr5,L(dLcr5)
192 subfic rN,r12,32 /* Shift count is 32 - (rN * 8). */
193 lwz 1,0(1)
194 bne L(d00)
195 li rRTN,0
196 blr
198 /* Remainder is 8 */
199 .align 4
200 L(dP2):
201 mtctr rTMP
202 lwz rWORD5,0(rSTR1)
203 lwz rWORD6,0(rSTR2)
204 cmplw cr6,rWORD5,rWORD6
205 blt cr7,L(dP2x)
206 lwz rWORD7,4(rSTR1)
207 lwz rWORD8,4(rSTR2)
208 cmplw cr5,rWORD7,rWORD8
209 L(dP2e):
210 lwz rWORD1,8(rSTR1)
211 lwz rWORD2,8(rSTR2)
212 cmplw cr0,rWORD1,rWORD2
213 lwz rWORD3,12(rSTR1)
214 lwz rWORD4,12(rSTR2)
215 cmplw cr1,rWORD3,rWORD4
216 addi rSTR1,rSTR1,4
217 addi rSTR2,rSTR2,4
218 bne cr6,L(dLcr6)
219 bne cr5,L(dLcr5)
220 b L(dLoop2)
221 /* Again we are on a early exit path (16-23 byte compare), we want to
222 only use volatile registers and avoid restoring non-volatile
223 registers. */
224 .align 4
225 L(dP2x):
226 lwz rWORD3,4(rSTR1)
227 lwz rWORD4,4(rSTR2)
228 cmplw cr5,rWORD3,rWORD4
229 slwi. r12,rN,3
230 bne cr6,L(dLcr6)
231 addi rSTR1,rSTR1,4
232 addi rSTR2,rSTR2,4
233 bne cr5,L(dLcr5)
234 subfic rN,r12,32 /* Shift count is 32 - (rN * 8). */
235 lwz 1,0(1)
236 bne L(d00)
237 li rRTN,0
238 blr
240 /* Remainder is 12 */
241 .align 4
242 L(dP3):
243 mtctr rTMP
244 lwz rWORD3,0(rSTR1)
245 lwz rWORD4,0(rSTR2)
246 cmplw cr1,rWORD3,rWORD4
247 L(dP3e):
248 lwz rWORD5,4(rSTR1)
249 lwz rWORD6,4(rSTR2)
250 cmplw cr6,rWORD5,rWORD6
251 blt cr7,L(dP3x)
252 lwz rWORD7,8(rSTR1)
253 lwz rWORD8,8(rSTR2)
254 cmplw cr5,rWORD7,rWORD8
255 lwz rWORD1,12(rSTR1)
256 lwz rWORD2,12(rSTR2)
257 cmplw cr0,rWORD1,rWORD2
258 addi rSTR1,rSTR1,8
259 addi rSTR2,rSTR2,8
260 bne cr1,L(dLcr1)
261 bne cr6,L(dLcr6)
262 b L(dLoop1)
263 /* Again we are on a early exit path (24-31 byte compare), we want to
264 only use volatile registers and avoid restoring non-volatile
265 registers. */
266 .align 4
267 L(dP3x):
268 lwz rWORD1,8(rSTR1)
269 lwz rWORD2,8(rSTR2)
270 cmplw cr5,rWORD1,rWORD2
271 slwi. r12,rN,3
272 bne cr1,L(dLcr1)
273 addi rSTR1,rSTR1,8
274 addi rSTR2,rSTR2,8
275 bne cr6,L(dLcr6)
276 subfic rN,r12,32 /* Shift count is 32 - (rN * 8). */
277 bne cr5,L(dLcr5)
278 lwz 1,0(1)
279 bne L(d00)
280 li rRTN,0
281 blr
283 /* Count is a multiple of 16, remainder is 0 */
284 .align 4
285 L(dP4):
286 mtctr rTMP
287 lwz rWORD1,0(rSTR1)
288 lwz rWORD2,0(rSTR2)
289 cmplw cr0,rWORD1,rWORD2
290 L(dP4e):
291 lwz rWORD3,4(rSTR1)
292 lwz rWORD4,4(rSTR2)
293 cmplw cr1,rWORD3,rWORD4
294 lwz rWORD5,8(rSTR1)
295 lwz rWORD6,8(rSTR2)
296 cmplw cr6,rWORD5,rWORD6
297 lwzu rWORD7,12(rSTR1)
298 lwzu rWORD8,12(rSTR2)
299 cmplw cr5,rWORD7,rWORD8
300 bne cr0,L(dLcr0)
301 bne cr1,L(dLcr1)
302 bdz- L(d24) /* Adjust CTR as we start with +4 */
303 /* This is the primary loop */
304 .align 4
305 L(dLoop):
306 lwz rWORD1,4(rSTR1)
307 lwz rWORD2,4(rSTR2)
308 cmplw cr1,rWORD3,rWORD4
309 bne cr6,L(dLcr6)
310 L(dLoop1):
311 lwz rWORD3,8(rSTR1)
312 lwz rWORD4,8(rSTR2)
313 cmplw cr6,rWORD5,rWORD6
314 bne cr5,L(dLcr5)
315 L(dLoop2):
316 lwz rWORD5,12(rSTR1)
317 lwz rWORD6,12(rSTR2)
318 cmplw cr5,rWORD7,rWORD8
319 bne cr0,L(dLcr0)
320 L(dLoop3):
321 lwzu rWORD7,16(rSTR1)
322 lwzu rWORD8,16(rSTR2)
323 bne cr1,L(dLcr1)
324 cmplw cr0,rWORD1,rWORD2
325 bdnz L(dLoop)
327 L(dL4):
328 cmplw cr1,rWORD3,rWORD4
329 bne cr6,L(dLcr6)
330 cmplw cr6,rWORD5,rWORD6
331 bne cr5,L(dLcr5)
332 cmplw cr5,rWORD7,rWORD8
333 L(d44):
334 bne cr0,L(dLcr0)
335 L(d34):
336 bne cr1,L(dLcr1)
337 L(d24):
338 bne cr6,L(dLcr6)
339 L(d14):
340 slwi. r12,rN,3
341 bne cr5,L(dLcr5)
342 L(d04):
343 lwz r30,44(1)
344 lwz r31,48(1)
345 lwz 1,0(1)
346 subfic rN,r12,32 /* Shift count is 32 - (rN * 8). */
347 beq L(zeroLength)
348 /* At this point we have a remainder of 1 to 3 bytes to compare. Since
349 we are aligned it is safe to load the whole word, and use
350 shift right to eliminate bits beyond the compare length. */
351 L(d00):
352 lwz rWORD1,4(rSTR1)
353 lwz rWORD2,4(rSTR2)
354 srw rWORD1,rWORD1,rN
355 srw rWORD2,rWORD2,rN
356 cmplw rWORD1,rWORD2
357 li rRTN,0
358 beqlr
359 li rRTN,1
360 bgtlr
361 li rRTN,-1
362 blr
364 .align 4
365 L(dLcr0):
366 lwz r30,44(1)
367 lwz r31,48(1)
368 li rRTN,1
369 lwz 1,0(1)
370 bgtlr cr0
371 li rRTN,-1
372 blr
373 .align 4
374 L(dLcr1):
375 lwz r30,44(1)
376 lwz r31,48(1)
377 li rRTN,1
378 lwz 1,0(1)
379 bgtlr cr1
380 li rRTN,-1
381 blr
382 .align 4
383 L(dLcr6):
384 lwz r30,44(1)
385 lwz r31,48(1)
386 li rRTN,1
387 lwz 1,0(1)
388 bgtlr cr6
389 li rRTN,-1
390 blr
391 .align 4
392 L(dLcr5):
393 lwz r30,44(1)
394 lwz r31,48(1)
395 L(dLcr5x):
396 li rRTN,1
397 lwz 1,0(1)
398 bgtlr cr5
399 li rRTN,-1
400 blr
402 .align 4
403 L(bytealigned):
404 cfi_adjust_cfa_offset(-64)
405 mtctr rN
407 /* We need to prime this loop. This loop is swing modulo scheduled
408 to avoid pipe delays. The dependent instruction latencies (load to
409 compare to conditional branch) is 2 to 3 cycles. In this loop each
410 dispatch group ends in a branch and takes 1 cycle. Effectively
411 the first iteration of the loop only serves to load operands and
412 branches based on compares are delayed until the next loop.
414 So we must precondition some registers and condition codes so that
415 we don't exit the loop early on the first iteration. */
416 lbz rWORD1,0(rSTR1)
417 lbz rWORD2,0(rSTR2)
418 bdz L(b11)
419 cmplw cr0,rWORD1,rWORD2
420 lbz rWORD3,1(rSTR1)
421 lbz rWORD4,1(rSTR2)
422 bdz L(b12)
423 cmplw cr1,rWORD3,rWORD4
424 lbzu rWORD5,2(rSTR1)
425 lbzu rWORD6,2(rSTR2)
426 bdz L(b13)
427 .align 4
428 L(bLoop):
429 lbzu rWORD1,1(rSTR1)
430 lbzu rWORD2,1(rSTR2)
431 bne cr0,L(bLcr0)
433 cmplw cr6,rWORD5,rWORD6
434 bdz L(b3i)
436 lbzu rWORD3,1(rSTR1)
437 lbzu rWORD4,1(rSTR2)
438 bne cr1,L(bLcr1)
440 cmplw cr0,rWORD1,rWORD2
441 bdz L(b2i)
443 lbzu rWORD5,1(rSTR1)
444 lbzu rWORD6,1(rSTR2)
445 bne cr6,L(bLcr6)
447 cmplw cr1,rWORD3,rWORD4
448 bdnz L(bLoop)
450 /* We speculatively loading bytes before we have tested the previous
451 bytes. But we must avoid overrunning the length (in the ctr) to
452 prevent these speculative loads from causing a segfault. In this
453 case the loop will exit early (before the all pending bytes are
454 tested. In this case we must complete the pending operations
455 before returning. */
456 L(b1i):
457 bne cr0,L(bLcr0)
458 bne cr1,L(bLcr1)
459 b L(bx56)
460 .align 4
461 L(b2i):
462 bne cr6,L(bLcr6)
463 bne cr0,L(bLcr0)
464 b L(bx34)
465 .align 4
466 L(b3i):
467 bne cr1,L(bLcr1)
468 bne cr6,L(bLcr6)
469 b L(bx12)
470 .align 4
471 L(bLcr0):
472 li rRTN,1
473 bgtlr cr0
474 li rRTN,-1
475 blr
476 L(bLcr1):
477 li rRTN,1
478 bgtlr cr1
479 li rRTN,-1
480 blr
481 L(bLcr6):
482 li rRTN,1
483 bgtlr cr6
484 li rRTN,-1
485 blr
487 L(b13):
488 bne cr0,L(bx12)
489 bne cr1,L(bx34)
490 L(bx56):
491 sub rRTN,rWORD5,rWORD6
492 blr
493 nop
494 L(b12):
495 bne cr0,L(bx12)
496 L(bx34):
497 sub rRTN,rWORD3,rWORD4
498 blr
500 L(b11):
501 L(bx12):
502 sub rRTN,rWORD1,rWORD2
503 blr
505 .align 4
506 L(zeroLengthReturn):
508 L(zeroLength):
509 li rRTN,0
510 blr
512 cfi_adjust_cfa_offset(64)
513 .align 4
514 /* At this point we know the strings have different alignment and the
515 compare length is at least 8 bytes. rBITDIF contains the low order
516 2 bits of rSTR1 and cr5 contains the result of the logical compare
517 of rBITDIF to 0. If rBITDIF == 0 then rStr1 is word aligned and can
518 perform the Wunaligned loop.
520 Otherwise we know that rSTR1 is not aready word aligned yet.
521 So we can force the string addresses to the next lower word
522 boundary and special case this first word using shift left to
523 eliminate bits preceeding the first byte. Since we want to join the
524 normal (Wualigned) compare loop, starting at the second word,
525 we need to adjust the length (rN) and special case the loop
526 versioning for the first W. This insures that the loop count is
527 correct and the first W (shifted) is in the expected resister pair. */
528 #define rSHL r29 /* Unaligned shift left count. */
529 #define rSHR r28 /* Unaligned shift right count. */
530 #define rB r27 /* Left rotation temp for rWORD2. */
531 #define rD r26 /* Left rotation temp for rWORD4. */
532 #define rF r25 /* Left rotation temp for rWORD6. */
533 #define rH r24 /* Left rotation temp for rWORD8. */
534 #define rA r0 /* Right rotation temp for rWORD2. */
535 #define rC r12 /* Right rotation temp for rWORD4. */
536 #define rE r0 /* Right rotation temp for rWORD6. */
537 #define rG r12 /* Right rotation temp for rWORD8. */
538 L(unaligned):
539 stw r29,40(r1)
540 cfi_offset(r29,(40-64))
541 clrlwi rSHL,rSTR2,30
542 stw r28,36(r1)
543 cfi_offset(r28,(36-64))
544 beq cr5,L(Wunaligned)
545 stw r27,32(r1)
546 cfi_offset(r27,(32-64))
547 /* Adjust the logical start of rSTR2 to compensate for the extra bits
548 in the 1st rSTR1 W. */
549 sub r27,rSTR2,rBITDIF
550 /* But do not attempt to address the W before that W that contains
551 the actual start of rSTR2. */
552 clrrwi rSTR2,rSTR2,2
553 stw r26,28(r1)
554 cfi_offset(r26,(28-64))
555 /* Compute the left/right shift counts for the unalign rSTR2,
556 compensating for the logical (W aligned) start of rSTR1. */
557 clrlwi rSHL,r27,30
558 clrrwi rSTR1,rSTR1,2
559 stw r25,24(r1)
560 cfi_offset(r25,(24-64))
561 slwi rSHL,rSHL,3
562 cmplw cr5,r27,rSTR2
563 add rN,rN,rBITDIF
564 slwi r11,rBITDIF,3
565 stw r24,20(r1)
566 cfi_offset(r24,(20-64))
567 subfic rSHR,rSHL,32
568 srwi rTMP,rN,4 /* Divide by 16 */
569 andi. rBITDIF,rN,12 /* Get the W remainder */
570 /* We normally need to load 2 Ws to start the unaligned rSTR2, but in
571 this special case those bits may be discarded anyway. Also we
572 must avoid loading a W where none of the bits are part of rSTR2 as
573 this may cross a page boundary and cause a page fault. */
574 li rWORD8,0
575 blt cr5,L(dus0)
576 lwz rWORD8,0(rSTR2)
577 la rSTR2,4(rSTR2)
578 slw rWORD8,rWORD8,rSHL
580 L(dus0):
581 lwz rWORD1,0(rSTR1)
582 lwz rWORD2,0(rSTR2)
583 cmplwi cr1,rBITDIF,8
584 cmplwi cr7,rN,16
585 srw rG,rWORD2,rSHR
586 clrlwi rN,rN,30
587 beq L(duPs4)
588 mtctr rTMP
589 or rWORD8,rG,rWORD8
590 bgt cr1,L(duPs3)
591 beq cr1,L(duPs2)
593 /* Remainder is 4 */
594 .align 4
595 L(dusP1):
596 slw rB,rWORD2,rSHL
597 slw rWORD7,rWORD1,r11
598 slw rWORD8,rWORD8,r11
599 bge cr7,L(duP1e)
600 /* At this point we exit early with the first word compare
601 complete and remainder of 0 to 3 bytes. See L(du14) for details on
602 how we handle the remaining bytes. */
603 cmplw cr5,rWORD7,rWORD8
604 slwi. rN,rN,3
605 bne cr5,L(duLcr5)
606 cmplw cr7,rN,rSHR
607 beq L(duZeroReturn)
608 li rA,0
609 ble cr7,L(dutrim)
610 lwz rWORD2,4(rSTR2)
611 srw rA,rWORD2,rSHR
612 b L(dutrim)
613 /* Remainder is 8 */
614 .align 4
615 L(duPs2):
616 slw rH,rWORD2,rSHL
617 slw rWORD5,rWORD1,r11
618 slw rWORD6,rWORD8,r11
619 b L(duP2e)
620 /* Remainder is 12 */
621 .align 4
622 L(duPs3):
623 slw rF,rWORD2,rSHL
624 slw rWORD3,rWORD1,r11
625 slw rWORD4,rWORD8,r11
626 b L(duP3e)
627 /* Count is a multiple of 16, remainder is 0 */
628 .align 4
629 L(duPs4):
630 mtctr rTMP
631 or rWORD8,rG,rWORD8
632 slw rD,rWORD2,rSHL
633 slw rWORD1,rWORD1,r11
634 slw rWORD2,rWORD8,r11
635 b L(duP4e)
637 /* At this point we know rSTR1 is word aligned and the
638 compare length is at least 8 bytes. */
639 .align 4
640 L(Wunaligned):
641 stw r27,32(r1)
642 cfi_offset(r27,(32-64))
643 clrrwi rSTR2,rSTR2,2
644 stw r26,28(r1)
645 cfi_offset(r26,(28-64))
646 srwi rTMP,rN,4 /* Divide by 16 */
647 stw r25,24(r1)
648 cfi_offset(r25,(24-64))
649 andi. rBITDIF,rN,12 /* Get the W remainder */
650 stw r24,20(r1)
651 cfi_offset(r24,(24-64))
652 slwi rSHL,rSHL,3
653 lwz rWORD6,0(rSTR2)
654 lwzu rWORD8,4(rSTR2)
655 cmplwi cr1,rBITDIF,8
656 cmplwi cr7,rN,16
657 clrlwi rN,rN,30
658 subfic rSHR,rSHL,32
659 slw rH,rWORD6,rSHL
660 beq L(duP4)
661 mtctr rTMP
662 bgt cr1,L(duP3)
663 beq cr1,L(duP2)
665 /* Remainder is 4 */
666 .align 4
667 L(duP1):
668 srw rG,rWORD8,rSHR
669 lwz rWORD7,0(rSTR1)
670 slw rB,rWORD8,rSHL
671 or rWORD8,rG,rH
672 blt cr7,L(duP1x)
673 L(duP1e):
674 lwz rWORD1,4(rSTR1)
675 lwz rWORD2,4(rSTR2)
676 cmplw cr5,rWORD7,rWORD8
677 srw rA,rWORD2,rSHR
678 slw rD,rWORD2,rSHL
679 or rWORD2,rA,rB
680 lwz rWORD3,8(rSTR1)
681 lwz rWORD4,8(rSTR2)
682 cmplw cr0,rWORD1,rWORD2
683 srw rC,rWORD4,rSHR
684 slw rF,rWORD4,rSHL
685 bne cr5,L(duLcr5)
686 or rWORD4,rC,rD
687 lwz rWORD5,12(rSTR1)
688 lwz rWORD6,12(rSTR2)
689 cmplw cr1,rWORD3,rWORD4
690 srw rE,rWORD6,rSHR
691 slw rH,rWORD6,rSHL
692 bne cr0,L(duLcr0)
693 or rWORD6,rE,rF
694 cmplw cr6,rWORD5,rWORD6
695 b L(duLoop3)
696 .align 4
697 /* At this point we exit early with the first word compare
698 complete and remainder of 0 to 3 bytes. See L(du14) for details on
699 how we handle the remaining bytes. */
700 L(duP1x):
701 cmplw cr5,rWORD7,rWORD8
702 slwi. rN,rN,3
703 bne cr5,L(duLcr5)
704 cmplw cr7,rN,rSHR
705 beq L(duZeroReturn)
706 li rA,0
707 ble cr7,L(dutrim)
708 ld rWORD2,8(rSTR2)
709 srw rA,rWORD2,rSHR
710 b L(dutrim)
711 /* Remainder is 8 */
712 .align 4
713 L(duP2):
714 srw rE,rWORD8,rSHR
715 lwz rWORD5,0(rSTR1)
716 or rWORD6,rE,rH
717 slw rH,rWORD8,rSHL
718 L(duP2e):
719 lwz rWORD7,4(rSTR1)
720 lwz rWORD8,4(rSTR2)
721 cmplw cr6,rWORD5,rWORD6
722 srw rG,rWORD8,rSHR
723 slw rB,rWORD8,rSHL
724 or rWORD8,rG,rH
725 blt cr7,L(duP2x)
726 lwz rWORD1,8(rSTR1)
727 lwz rWORD2,8(rSTR2)
728 cmplw cr5,rWORD7,rWORD8
729 bne cr6,L(duLcr6)
730 srw rA,rWORD2,rSHR
731 slw rD,rWORD2,rSHL
732 or rWORD2,rA,rB
733 lwz rWORD3,12(rSTR1)
734 lwz rWORD4,12(rSTR2)
735 cmplw cr0,rWORD1,rWORD2
736 bne cr5,L(duLcr5)
737 srw rC,rWORD4,rSHR
738 slw rF,rWORD4,rSHL
739 or rWORD4,rC,rD
740 addi rSTR1,rSTR1,4
741 addi rSTR2,rSTR2,4
742 cmplw cr1,rWORD3,rWORD4
743 b L(duLoop2)
744 .align 4
745 L(duP2x):
746 cmplw cr5,rWORD7,rWORD8
747 addi rSTR1,rSTR1,4
748 addi rSTR2,rSTR2,4
749 bne cr6,L(duLcr6)
750 slwi. rN,rN,3
751 bne cr5,L(duLcr5)
752 cmplw cr7,rN,rSHR
753 beq L(duZeroReturn)
754 li rA,0
755 ble cr7,L(dutrim)
756 lwz rWORD2,4(rSTR2)
757 srw rA,rWORD2,rSHR
758 b L(dutrim)
760 /* Remainder is 12 */
761 .align 4
762 L(duP3):
763 srw rC,rWORD8,rSHR
764 lwz rWORD3,0(rSTR1)
765 slw rF,rWORD8,rSHL
766 or rWORD4,rC,rH
767 L(duP3e):
768 lwz rWORD5,4(rSTR1)
769 lwz rWORD6,4(rSTR2)
770 cmplw cr1,rWORD3,rWORD4
771 srw rE,rWORD6,rSHR
772 slw rH,rWORD6,rSHL
773 or rWORD6,rE,rF
774 lwz rWORD7,8(rSTR1)
775 lwz rWORD8,8(rSTR2)
776 cmplw cr6,rWORD5,rWORD6
777 bne cr1,L(duLcr1)
778 srw rG,rWORD8,rSHR
779 slw rB,rWORD8,rSHL
780 or rWORD8,rG,rH
781 blt cr7,L(duP3x)
782 lwz rWORD1,12(rSTR1)
783 lwz rWORD2,12(rSTR2)
784 cmplw cr5,rWORD7,rWORD8
785 bne cr6,L(duLcr6)
786 srw rA,rWORD2,rSHR
787 slw rD,rWORD2,rSHL
788 or rWORD2,rA,rB
789 addi rSTR1,rSTR1,8
790 addi rSTR2,rSTR2,8
791 cmplw cr0,rWORD1,rWORD2
792 b L(duLoop1)
793 .align 4
794 L(duP3x):
795 addi rSTR1,rSTR1,8
796 addi rSTR2,rSTR2,8
797 bne cr1,L(duLcr1)
798 cmplw cr5,rWORD7,rWORD8
799 bne cr6,L(duLcr6)
800 slwi. rN,rN,3
801 bne cr5,L(duLcr5)
802 cmplw cr7,rN,rSHR
803 beq L(duZeroReturn)
804 li rA,0
805 ble cr7,L(dutrim)
806 lwz rWORD2,4(rSTR2)
807 srw rA,rWORD2,rSHR
808 b L(dutrim)
810 /* Count is a multiple of 16, remainder is 0 */
811 .align 4
812 L(duP4):
813 mtctr rTMP
814 srw rA,rWORD8,rSHR
815 lwz rWORD1,0(rSTR1)
816 slw rD,rWORD8,rSHL
817 or rWORD2,rA,rH
818 L(duP4e):
819 lwz rWORD3,4(rSTR1)
820 lwz rWORD4,4(rSTR2)
821 cmplw cr0,rWORD1,rWORD2
822 srw rC,rWORD4,rSHR
823 slw rF,rWORD4,rSHL
824 or rWORD4,rC,rD
825 lwz rWORD5,8(rSTR1)
826 lwz rWORD6,8(rSTR2)
827 cmplw cr1,rWORD3,rWORD4
828 bne cr0,L(duLcr0)
829 srw rE,rWORD6,rSHR
830 slw rH,rWORD6,rSHL
831 or rWORD6,rE,rF
832 lwzu rWORD7,12(rSTR1)
833 lwzu rWORD8,12(rSTR2)
834 cmplw cr6,rWORD5,rWORD6
835 bne cr1,L(duLcr1)
836 srw rG,rWORD8,rSHR
837 slw rB,rWORD8,rSHL
838 or rWORD8,rG,rH
839 cmplw cr5,rWORD7,rWORD8
840 bdz L(du24) /* Adjust CTR as we start with +4 */
841 /* This is the primary loop */
842 .align 4
843 L(duLoop):
844 lwz rWORD1,4(rSTR1)
845 lwz rWORD2,4(rSTR2)
846 cmplw cr1,rWORD3,rWORD4
847 bne cr6,L(duLcr6)
848 srw rA,rWORD2,rSHR
849 slw rD,rWORD2,rSHL
850 or rWORD2,rA,rB
851 L(duLoop1):
852 lwz rWORD3,8(rSTR1)
853 lwz rWORD4,8(rSTR2)
854 cmplw cr6,rWORD5,rWORD6
855 bne cr5,L(duLcr5)
856 srw rC,rWORD4,rSHR
857 slw rF,rWORD4,rSHL
858 or rWORD4,rC,rD
859 L(duLoop2):
860 lwz rWORD5,12(rSTR1)
861 lwz rWORD6,12(rSTR2)
862 cmplw cr5,rWORD7,rWORD8
863 bne cr0,L(duLcr0)
864 srw rE,rWORD6,rSHR
865 slw rH,rWORD6,rSHL
866 or rWORD6,rE,rF
867 L(duLoop3):
868 lwzu rWORD7,16(rSTR1)
869 lwzu rWORD8,16(rSTR2)
870 cmplw cr0,rWORD1,rWORD2
871 bne cr1,L(duLcr1)
872 srw rG,rWORD8,rSHR
873 slw rB,rWORD8,rSHL
874 or rWORD8,rG,rH
875 bdnz L(duLoop)
877 L(duL4):
878 bne cr1,L(duLcr1)
879 cmplw cr1,rWORD3,rWORD4
880 bne cr6,L(duLcr6)
881 cmplw cr6,rWORD5,rWORD6
882 bne cr5,L(duLcr5)
883 cmplw cr5,rWORD7,rWORD8
884 L(du44):
885 bne cr0,L(duLcr0)
886 L(du34):
887 bne cr1,L(duLcr1)
888 L(du24):
889 bne cr6,L(duLcr6)
890 L(du14):
891 slwi. rN,rN,3
892 bne cr5,L(duLcr5)
893 /* At this point we have a remainder of 1 to 3 bytes to compare. We use
894 shift right to eliminate bits beyond the compare length.
896 However it may not be safe to load rWORD2 which may be beyond the
897 string length. So we compare the bit length of the remainder to
898 the right shift count (rSHR). If the bit count is less than or equal
899 we do not need to load rWORD2 (all significant bits are already in
900 rB). */
901 cmplw cr7,rN,rSHR
902 beq L(duZeroReturn)
903 li rA,0
904 ble cr7,L(dutrim)
905 lwz rWORD2,4(rSTR2)
906 srw rA,rWORD2,rSHR
907 .align 4
908 L(dutrim):
909 lwz rWORD1,4(rSTR1)
910 lwz r31,48(1)
911 subfic rN,rN,32 /* Shift count is 32 - (rN * 8). */
912 or rWORD2,rA,rB
913 lwz r30,44(1)
914 lwz r29,40(r1)
915 srw rWORD1,rWORD1,rN
916 srw rWORD2,rWORD2,rN
917 lwz r28,36(r1)
918 lwz r27,32(r1)
919 cmplw rWORD1,rWORD2
920 li rRTN,0
921 beq L(dureturn26)
922 li rRTN,1
923 bgt L(dureturn26)
924 li rRTN,-1
925 b L(dureturn26)
926 .align 4
927 L(duLcr0):
928 lwz r31,48(1)
929 lwz r30,44(1)
930 li rRTN,1
931 bgt cr0,L(dureturn29)
932 lwz r29,40(r1)
933 lwz r28,36(r1)
934 li rRTN,-1
935 b L(dureturn27)
936 .align 4
937 L(duLcr1):
938 lwz r31,48(1)
939 lwz r30,44(1)
940 li rRTN,1
941 bgt cr1,L(dureturn29)
942 lwz r29,40(r1)
943 lwz r28,36(r1)
944 li rRTN,-1
945 b L(dureturn27)
946 .align 4
947 L(duLcr6):
948 lwz r31,48(1)
949 lwz r30,44(1)
950 li rRTN,1
951 bgt cr6,L(dureturn29)
952 lwz r29,40(r1)
953 lwz r28,36(r1)
954 li rRTN,-1
955 b L(dureturn27)
956 .align 4
957 L(duLcr5):
958 lwz r31,48(1)
959 lwz r30,44(1)
960 li rRTN,1
961 bgt cr5,L(dureturn29)
962 lwz r29,40(r1)
963 lwz r28,36(r1)
964 li rRTN,-1
965 b L(dureturn27)
966 .align 3
967 L(duZeroReturn):
968 li rRTN,0
969 .align 4
970 L(dureturn):
971 lwz r31,48(1)
972 lwz r30,44(1)
973 L(dureturn29):
974 lwz r29,40(r1)
975 lwz r28,36(r1)
976 L(dureturn27):
977 lwz r27,32(r1)
978 L(dureturn26):
979 lwz r26,28(r1)
980 L(dureturn25):
981 lwz r25,24(r1)
982 lwz r24,20(r1)
983 lwz 1,0(1)
984 blr
985 END (BP_SYM (memcmp))
986 libc_hidden_builtin_def (memcmp)
987 weak_alias (memcmp,bcmp)