| blob | d529b492fc9ea854e5bc2786714596129d0a4aca |
1 /* Optimized memcmp implementation for POWER7/PowerPC32.
2 Copyright (C) 2010 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA
18 02110-1301 USA. */
20 #include <sysdep.h>
21 #include <bp-sym.h>
22 #include <bp-asm.h>
24 /* int [r3] memcmp (const char *s1 [r3],
25 const char *s2 [r4],
26 size_t size [r5]) */
28 .machine power7
29 EALIGN (BP_SYM(memcmp),4,0)
30 CALL_MCOUNT
32 #define rTMP r0
33 #define rRTN r3
34 #define rSTR1 r3 /* first string arg */
35 #define rSTR2 r4 /* second string arg */
36 #define rN r5 /* max string length */
37 #define rWORD1 r6 /* current word in s1 */
38 #define rWORD2 r7 /* current word in s2 */
39 #define rWORD3 r8 /* next word in s1 */
40 #define rWORD4 r9 /* next word in s2 */
41 #define rWORD5 r10 /* next word in s1 */
42 #define rWORD6 r11 /* next word in s2 */
43 #define rBITDIF r12 /* bits that differ in s1 & s2 words */
44 #define rWORD7 r30 /* next word in s1 */
45 #define rWORD8 r31 /* next word in s2 */
47 xor rTMP,rSTR2,rSTR1
48 cmplwi cr6,rN,0
49 cmplwi cr1,rN,12
50 clrlwi. rTMP,rTMP,30
51 clrlwi rBITDIF,rSTR1,30
52 cmplwi cr5,rBITDIF,0
53 beq- cr6,L(zeroLength)
54 dcbt 0,rSTR1
55 dcbt 0,rSTR2
57 /* If less than 8 bytes or not aligned, use the unaligned
58 byte loop. */
60 blt cr1,L(bytealigned)
61 stwu 1,-64(1)
62 cfi_adjust_cfa_offset(64)
63 stw r31,48(1)
64 cfi_offset(31,(48-64))
65 stw r30,44(1)
66 cfi_offset(30,(44-64))
67 bne L(unaligned)
68 /* At this point we know both strings have the same alignment and the
69 compare length is at least 8 bytes. rBITDIF contains the low order
70 2 bits of rSTR1 and cr5 contains the result of the logical compare
71 of rBITDIF to 0. If rBITDIF == 0 then we are already word
72 aligned and can perform the word aligned loop.
74 Otherwise we know the two strings have the same alignment (but not
75 yet word aligned). So we force the string addresses to the next lower
76 word boundary and special case this first word using shift left to
77 eliminate bits preceeding the first byte. Since we want to join the
78 normal (word aligned) compare loop, starting at the second word,
79 we need to adjust the length (rN) and special case the loop
80 versioning for the first word. This insures that the loop count is
81 correct and the first word (shifted) is in the expected register pair. */
82 .align 4
83 L(samealignment):
84 clrrwi rSTR1,rSTR1,2
85 clrrwi rSTR2,rSTR2,2
86 beq cr5,L(Waligned)
87 add rN,rN,rBITDIF
88 slwi r11,rBITDIF,3
89 srwi rTMP,rN,4 /* Divide by 16 */
90 andi. rBITDIF,rN,12 /* Get the word remainder */
91 lwz rWORD1,0(rSTR1)
92 lwz rWORD2,0(rSTR2)
93 cmplwi cr1,rBITDIF,8
94 cmplwi cr7,rN,16
95 clrlwi rN,rN,30
96 beq L(dPs4)
97 mtctr rTMP
98 bgt cr1,L(dPs3)
99 beq cr1,L(dPs2)
101 /* Remainder is 4 */
102 .align 3
103 L(dsP1):
104 slw rWORD5,rWORD1,r11
105 slw rWORD6,rWORD2,r11
106 cmplw cr5,rWORD5,rWORD6
107 blt cr7,L(dP1x)
108 /* Do something useful in this cycle since we have to branch anyway. */
109 lwz rWORD1,4(rSTR1)
110 lwz rWORD2,4(rSTR2)
111 cmplw cr0,rWORD1,rWORD2
112 b L(dP1e)
113 /* Remainder is 8 */
114 .align 4
115 L(dPs2):
116 slw rWORD5,rWORD1,r11
117 slw rWORD6,rWORD2,r11
118 cmplw cr6,rWORD5,rWORD6
119 blt cr7,L(dP2x)
120 /* Do something useful in this cycle since we have to branch anyway. */
121 lwz rWORD7,4(rSTR1)
122 lwz rWORD8,4(rSTR2)
123 cmplw cr5,rWORD7,rWORD8
124 b L(dP2e)
125 /* Remainder is 12 */
126 .align 4
127 L(dPs3):
128 slw rWORD3,rWORD1,r11
129 slw rWORD4,rWORD2,r11
130 cmplw cr1,rWORD3,rWORD4
131 b L(dP3e)
132 /* Count is a multiple of 16, remainder is 0 */
133 .align 4
134 L(dPs4):
135 mtctr rTMP
136 slw rWORD1,rWORD1,r11
137 slw rWORD2,rWORD2,r11
138 cmplw cr0,rWORD1,rWORD2
139 b L(dP4e)
141 /* At this point we know both strings are word aligned and the
142 compare length is at least 8 bytes. */
143 .align 4
144 L(Waligned):
145 andi. rBITDIF,rN,12 /* Get the word remainder */
146 srwi rTMP,rN,4 /* Divide by 16 */
147 cmplwi cr1,rBITDIF,8
148 cmplwi cr7,rN,16
149 clrlwi rN,rN,30
150 beq L(dP4)
151 bgt cr1,L(dP3)
152 beq cr1,L(dP2)
154 /* Remainder is 4 */
155 .align 4
156 L(dP1):
157 mtctr rTMP
158 /* Normally we'd use rWORD7/rWORD8 here, but since we might exit early
159 (8-15 byte compare), we want to use only volatile registers. This
160 means we can avoid restoring non-volatile registers since we did not
161 change any on the early exit path. The key here is the non-early
162 exit path only cares about the condition code (cr5), not about which
163 register pair was used. */
164 lwz rWORD5,0(rSTR1)
165 lwz rWORD6,0(rSTR2)
166 cmplw cr5,rWORD5,rWORD6
167 blt cr7,L(dP1x)
168 lwz rWORD1,4(rSTR1)
169 lwz rWORD2,4(rSTR2)
170 cmplw cr0,rWORD1,rWORD2
171 L(dP1e):
172 lwz rWORD3,8(rSTR1)
173 lwz rWORD4,8(rSTR2)
174 cmplw cr1,rWORD3,rWORD4
175 lwz rWORD5,12(rSTR1)
176 lwz rWORD6,12(rSTR2)
177 cmplw cr6,rWORD5,rWORD6
178 bne cr5,L(dLcr5)
179 bne cr0,L(dLcr0)
181 lwzu rWORD7,16(rSTR1)
182 lwzu rWORD8,16(rSTR2)
183 bne cr1,L(dLcr1)
184 cmplw cr5,rWORD7,rWORD8
185 bdnz L(dLoop)
186 bne cr6,L(dLcr6)
187 lwz r30,44(1)
188 lwz r31,48(1)
189 .align 3
190 L(dP1x):
191 slwi. r12,rN,3
192 bne cr5,L(dLcr5)
193 subfic rN,r12,32 /* Shift count is 32 - (rN * 8). */
194 lwz 1,0(1)
195 bne L(d00)
196 li rRTN,0
197 blr
199 /* Remainder is 8 */
200 .align 4
201 L(dP2):
202 mtctr rTMP
203 lwz rWORD5,0(rSTR1)
204 lwz rWORD6,0(rSTR2)
205 cmplw cr6,rWORD5,rWORD6
206 blt cr7,L(dP2x)
207 lwz rWORD7,4(rSTR1)
208 lwz rWORD8,4(rSTR2)
209 cmplw cr5,rWORD7,rWORD8
210 L(dP2e):
211 lwz rWORD1,8(rSTR1)
212 lwz rWORD2,8(rSTR2)
213 cmplw cr0,rWORD1,rWORD2
214 lwz rWORD3,12(rSTR1)
215 lwz rWORD4,12(rSTR2)
216 cmplw cr1,rWORD3,rWORD4
217 addi rSTR1,rSTR1,4
218 addi rSTR2,rSTR2,4
219 bne cr6,L(dLcr6)
220 bne cr5,L(dLcr5)
221 b L(dLoop2)
222 /* Again we are on a early exit path (16-23 byte compare), we want to
223 only use volatile registers and avoid restoring non-volatile
224 registers. */
225 .align 4
226 L(dP2x):
227 lwz rWORD3,4(rSTR1)
228 lwz rWORD4,4(rSTR2)
229 cmplw cr5,rWORD3,rWORD4
230 slwi. r12,rN,3
231 bne cr6,L(dLcr6)
232 addi rSTR1,rSTR1,4
233 addi rSTR2,rSTR2,4
234 bne cr5,L(dLcr5)
235 subfic rN,r12,32 /* Shift count is 32 - (rN * 8). */
236 lwz 1,0(1)
237 bne L(d00)
238 li rRTN,0
239 blr
241 /* Remainder is 12 */
242 .align 4
243 L(dP3):
244 mtctr rTMP
245 lwz rWORD3,0(rSTR1)
246 lwz rWORD4,0(rSTR2)
247 cmplw cr1,rWORD3,rWORD4
248 L(dP3e):
249 lwz rWORD5,4(rSTR1)
250 lwz rWORD6,4(rSTR2)
251 cmplw cr6,rWORD5,rWORD6
252 blt cr7,L(dP3x)
253 lwz rWORD7,8(rSTR1)
254 lwz rWORD8,8(rSTR2)
255 cmplw cr5,rWORD7,rWORD8
256 lwz rWORD1,12(rSTR1)
257 lwz rWORD2,12(rSTR2)
258 cmplw cr0,rWORD1,rWORD2
259 addi rSTR1,rSTR1,8
260 addi rSTR2,rSTR2,8
261 bne cr1,L(dLcr1)
262 bne cr6,L(dLcr6)
263 b L(dLoop1)
264 /* Again we are on a early exit path (24-31 byte compare), we want to
265 only use volatile registers and avoid restoring non-volatile
266 registers. */
267 .align 4
268 L(dP3x):
269 lwz rWORD1,8(rSTR1)
270 lwz rWORD2,8(rSTR2)
271 cmplw cr5,rWORD1,rWORD2
272 slwi. r12,rN,3
273 bne cr1,L(dLcr1)
274 addi rSTR1,rSTR1,8
275 addi rSTR2,rSTR2,8
276 bne cr6,L(dLcr6)
277 subfic rN,r12,32 /* Shift count is 32 - (rN * 8). */
278 bne cr5,L(dLcr5)
279 lwz 1,0(1)
280 bne L(d00)
281 li rRTN,0
282 blr
284 /* Count is a multiple of 16, remainder is 0 */
285 .align 4
286 L(dP4):
287 mtctr rTMP
288 lwz rWORD1,0(rSTR1)
289 lwz rWORD2,0(rSTR2)
290 cmplw cr0,rWORD1,rWORD2
291 L(dP4e):
292 lwz rWORD3,4(rSTR1)
293 lwz rWORD4,4(rSTR2)
294 cmplw cr1,rWORD3,rWORD4
295 lwz rWORD5,8(rSTR1)
296 lwz rWORD6,8(rSTR2)
297 cmplw cr6,rWORD5,rWORD6
298 lwzu rWORD7,12(rSTR1)
299 lwzu rWORD8,12(rSTR2)
300 cmplw cr5,rWORD7,rWORD8
301 bne cr0,L(dLcr0)
302 bne cr1,L(dLcr1)
303 bdz- L(d24) /* Adjust CTR as we start with +4 */
304 /* This is the primary loop */
305 .align 4
306 L(dLoop):
307 lwz rWORD1,4(rSTR1)
308 lwz rWORD2,4(rSTR2)
309 cmplw cr1,rWORD3,rWORD4
310 bne cr6,L(dLcr6)
311 L(dLoop1):
312 lwz rWORD3,8(rSTR1)
313 lwz rWORD4,8(rSTR2)
314 cmplw cr6,rWORD5,rWORD6
315 bne cr5,L(dLcr5)
316 L(dLoop2):
317 lwz rWORD5,12(rSTR1)
318 lwz rWORD6,12(rSTR2)
319 cmplw cr5,rWORD7,rWORD8
320 bne cr0,L(dLcr0)
321 L(dLoop3):
322 lwzu rWORD7,16(rSTR1)
323 lwzu rWORD8,16(rSTR2)
324 bne cr1,L(dLcr1)
325 cmplw cr0,rWORD1,rWORD2
326 bdnz L(dLoop)
328 L(dL4):
329 cmplw cr1,rWORD3,rWORD4
330 bne cr6,L(dLcr6)
331 cmplw cr6,rWORD5,rWORD6
332 bne cr5,L(dLcr5)
333 cmplw cr5,rWORD7,rWORD8
334 L(d44):
335 bne cr0,L(dLcr0)
336 L(d34):
337 bne cr1,L(dLcr1)
338 L(d24):
339 bne cr6,L(dLcr6)
340 L(d14):
341 slwi. r12,rN,3
342 bne cr5,L(dLcr5)
343 L(d04):
344 lwz r30,44(1)
345 lwz r31,48(1)
346 lwz 1,0(1)
347 subfic rN,r12,32 /* Shift count is 32 - (rN * 8). */
348 beq L(zeroLength)
349 /* At this point we have a remainder of 1 to 3 bytes to compare. Since
350 we are aligned it is safe to load the whole word, and use
351 shift right to eliminate bits beyond the compare length. */
352 L(d00):
353 lwz rWORD1,4(rSTR1)
354 lwz rWORD2,4(rSTR2)
355 srw rWORD1,rWORD1,rN
356 srw rWORD2,rWORD2,rN
357 cmplw rWORD1,rWORD2
358 li rRTN,0
359 beqlr
360 li rRTN,1
361 bgtlr
362 li rRTN,-1
363 blr
365 .align 4
366 L(dLcr0):
367 lwz r30,44(1)
368 lwz r31,48(1)
369 li rRTN,1
370 lwz 1,0(1)
371 bgtlr cr0
372 li rRTN,-1
373 blr
374 .align 4
375 L(dLcr1):
376 lwz r30,44(1)
377 lwz r31,48(1)
378 li rRTN,1
379 lwz 1,0(1)
380 bgtlr cr1
381 li rRTN,-1
382 blr
383 .align 4
384 L(dLcr6):
385 lwz r30,44(1)
386 lwz r31,48(1)
387 li rRTN,1
388 lwz 1,0(1)
389 bgtlr cr6
390 li rRTN,-1
391 blr
392 .align 4
393 L(dLcr5):
394 lwz r30,44(1)
395 lwz r31,48(1)
396 L(dLcr5x):
397 li rRTN,1
398 lwz 1,0(1)
399 bgtlr cr5
400 li rRTN,-1
401 blr
403 .align 4
404 L(bytealigned):
405 cfi_adjust_cfa_offset(-64)
406 mtctr rN
408 /* We need to prime this loop. This loop is swing modulo scheduled
409 to avoid pipe delays. The dependent instruction latencies (load to
410 compare to conditional branch) is 2 to 3 cycles. In this loop each
411 dispatch group ends in a branch and takes 1 cycle. Effectively
412 the first iteration of the loop only serves to load operands and
413 branches based on compares are delayed until the next loop.
415 So we must precondition some registers and condition codes so that
416 we don't exit the loop early on the first iteration. */
417 lbz rWORD1,0(rSTR1)
418 lbz rWORD2,0(rSTR2)
419 bdz L(b11)
420 cmplw cr0,rWORD1,rWORD2
421 lbz rWORD3,1(rSTR1)
422 lbz rWORD4,1(rSTR2)
423 bdz L(b12)
424 cmplw cr1,rWORD3,rWORD4
425 lbzu rWORD5,2(rSTR1)
426 lbzu rWORD6,2(rSTR2)
427 bdz L(b13)
428 .align 4
429 L(bLoop):
430 lbzu rWORD1,1(rSTR1)
431 lbzu rWORD2,1(rSTR2)
432 bne cr0,L(bLcr0)
434 cmplw cr6,rWORD5,rWORD6
435 bdz L(b3i)
437 lbzu rWORD3,1(rSTR1)
438 lbzu rWORD4,1(rSTR2)
439 bne cr1,L(bLcr1)
441 cmplw cr0,rWORD1,rWORD2
442 bdz L(b2i)
444 lbzu rWORD5,1(rSTR1)
445 lbzu rWORD6,1(rSTR2)
446 bne cr6,L(bLcr6)
448 cmplw cr1,rWORD3,rWORD4
449 bdnz L(bLoop)
451 /* We speculatively loading bytes before we have tested the previous
452 bytes. But we must avoid overrunning the length (in the ctr) to
453 prevent these speculative loads from causing a segfault. In this
454 case the loop will exit early (before the all pending bytes are
455 tested. In this case we must complete the pending operations
456 before returning. */
457 L(b1i):
458 bne cr0,L(bLcr0)
459 bne cr1,L(bLcr1)
460 b L(bx56)
461 .align 4
462 L(b2i):
463 bne cr6,L(bLcr6)
464 bne cr0,L(bLcr0)
465 b L(bx34)
466 .align 4
467 L(b3i):
468 bne cr1,L(bLcr1)
469 bne cr6,L(bLcr6)
470 b L(bx12)
471 .align 4
472 L(bLcr0):
473 li rRTN,1
474 bgtlr cr0
475 li rRTN,-1
476 blr
477 L(bLcr1):
478 li rRTN,1
479 bgtlr cr1
480 li rRTN,-1
481 blr
482 L(bLcr6):
483 li rRTN,1
484 bgtlr cr6
485 li rRTN,-1
486 blr
488 L(b13):
489 bne cr0,L(bx12)
490 bne cr1,L(bx34)
491 L(bx56):
492 sub rRTN,rWORD5,rWORD6
493 blr
494 nop
495 L(b12):
496 bne cr0,L(bx12)
497 L(bx34):
498 sub rRTN,rWORD3,rWORD4
499 blr
501 L(b11):
502 L(bx12):
503 sub rRTN,rWORD1,rWORD2
504 blr
506 .align 4
507 L(zeroLengthReturn):
509 L(zeroLength):
510 li rRTN,0
511 blr
513 cfi_adjust_cfa_offset(64)
514 .align 4
515 /* At this point we know the strings have different alignment and the
516 compare length is at least 8 bytes. rBITDIF contains the low order
517 2 bits of rSTR1 and cr5 contains the result of the logical compare
518 of rBITDIF to 0. If rBITDIF == 0 then rStr1 is word aligned and can
519 perform the Wunaligned loop.
521 Otherwise we know that rSTR1 is not aready word aligned yet.
522 So we can force the string addresses to the next lower word
523 boundary and special case this first word using shift left to
524 eliminate bits preceeding the first byte. Since we want to join the
525 normal (Wualigned) compare loop, starting at the second word,
526 we need to adjust the length (rN) and special case the loop
527 versioning for the first W. This insures that the loop count is
528 correct and the first W (shifted) is in the expected resister pair. */
529 #define rSHL r29 /* Unaligned shift left count. */
530 #define rSHR r28 /* Unaligned shift right count. */
531 #define rB r27 /* Left rotation temp for rWORD2. */
532 #define rD r26 /* Left rotation temp for rWORD4. */
533 #define rF r25 /* Left rotation temp for rWORD6. */
534 #define rH r24 /* Left rotation temp for rWORD8. */
535 #define rA r0 /* Right rotation temp for rWORD2. */
536 #define rC r12 /* Right rotation temp for rWORD4. */
537 #define rE r0 /* Right rotation temp for rWORD6. */
538 #define rG r12 /* Right rotation temp for rWORD8. */
539 L(unaligned):
540 stw r29,40(r1)
541 cfi_offset(r29,(40-64))
542 clrlwi rSHL,rSTR2,30
543 stw r28,36(r1)
544 cfi_offset(r28,(36-64))
545 beq cr5,L(Wunaligned)
546 stw r27,32(r1)
547 cfi_offset(r27,(32-64))
548 /* Adjust the logical start of rSTR2 to compensate for the extra bits
549 in the 1st rSTR1 W. */
550 sub r27,rSTR2,rBITDIF
551 /* But do not attempt to address the W before that W that contains
552 the actual start of rSTR2. */
553 clrrwi rSTR2,rSTR2,2
554 stw r26,28(r1)
555 cfi_offset(r26,(28-64))
556 /* Compute the left/right shift counts for the unalign rSTR2,
557 compensating for the logical (W aligned) start of rSTR1. */
558 clrlwi rSHL,r27,30
559 clrrwi rSTR1,rSTR1,2
560 stw r25,24(r1)
561 cfi_offset(r25,(24-64))
562 slwi rSHL,rSHL,3
563 cmplw cr5,r27,rSTR2
564 add rN,rN,rBITDIF
565 slwi r11,rBITDIF,3
566 stw r24,20(r1)
567 cfi_offset(r24,(20-64))
568 subfic rSHR,rSHL,32
569 srwi rTMP,rN,4 /* Divide by 16 */
570 andi. rBITDIF,rN,12 /* Get the W remainder */
571 /* We normally need to load 2 Ws to start the unaligned rSTR2, but in
572 this special case those bits may be discarded anyway. Also we
573 must avoid loading a W where none of the bits are part of rSTR2 as
574 this may cross a page boundary and cause a page fault. */
575 li rWORD8,0
576 blt cr5,L(dus0)
577 lwz rWORD8,0(rSTR2)
578 la rSTR2,4(rSTR2)
579 slw rWORD8,rWORD8,rSHL
581 L(dus0):
582 lwz rWORD1,0(rSTR1)
583 lwz rWORD2,0(rSTR2)
584 cmplwi cr1,rBITDIF,8
585 cmplwi cr7,rN,16
586 srw rG,rWORD2,rSHR
587 clrlwi rN,rN,30
588 beq L(duPs4)
589 mtctr rTMP
590 or rWORD8,rG,rWORD8
591 bgt cr1,L(duPs3)
592 beq cr1,L(duPs2)
594 /* Remainder is 4 */
595 .align 4
596 L(dusP1):
597 slw rB,rWORD2,rSHL
598 slw rWORD7,rWORD1,r11
599 slw rWORD8,rWORD8,r11
600 bge cr7,L(duP1e)
601 /* At this point we exit early with the first word compare
602 complete and remainder of 0 to 3 bytes. See L(du14) for details on
603 how we handle the remaining bytes. */
604 cmplw cr5,rWORD7,rWORD8
605 slwi. rN,rN,3
606 bne cr5,L(duLcr5)
607 cmplw cr7,rN,rSHR
608 beq L(duZeroReturn)
609 li rA,0
610 ble cr7,L(dutrim)
611 lwz rWORD2,4(rSTR2)
612 srw rA,rWORD2,rSHR
613 b L(dutrim)
614 /* Remainder is 8 */
615 .align 4
616 L(duPs2):
617 slw rH,rWORD2,rSHL
618 slw rWORD5,rWORD1,r11
619 slw rWORD6,rWORD8,r11
620 b L(duP2e)
621 /* Remainder is 12 */
622 .align 4
623 L(duPs3):
624 slw rF,rWORD2,rSHL
625 slw rWORD3,rWORD1,r11
626 slw rWORD4,rWORD8,r11
627 b L(duP3e)
628 /* Count is a multiple of 16, remainder is 0 */
629 .align 4
630 L(duPs4):
631 mtctr rTMP
632 or rWORD8,rG,rWORD8
633 slw rD,rWORD2,rSHL
634 slw rWORD1,rWORD1,r11
635 slw rWORD2,rWORD8,r11
636 b L(duP4e)
638 /* At this point we know rSTR1 is word aligned and the
639 compare length is at least 8 bytes. */
640 .align 4
641 L(Wunaligned):
642 stw r27,32(r1)
643 cfi_offset(r27,(32-64))
644 clrrwi rSTR2,rSTR2,2
645 stw r26,28(r1)
646 cfi_offset(r26,(28-64))
647 srwi rTMP,rN,4 /* Divide by 16 */
648 stw r25,24(r1)
649 cfi_offset(r25,(24-64))
650 andi. rBITDIF,rN,12 /* Get the W remainder */
651 stw r24,20(r1)
652 cfi_offset(r24,(24-64))
653 slwi rSHL,rSHL,3
654 lwz rWORD6,0(rSTR2)
655 lwzu rWORD8,4(rSTR2)
656 cmplwi cr1,rBITDIF,8
657 cmplwi cr7,rN,16
658 clrlwi rN,rN,30
659 subfic rSHR,rSHL,32
660 slw rH,rWORD6,rSHL
661 beq L(duP4)
662 mtctr rTMP
663 bgt cr1,L(duP3)
664 beq cr1,L(duP2)
666 /* Remainder is 4 */
667 .align 4
668 L(duP1):
669 srw rG,rWORD8,rSHR
670 lwz rWORD7,0(rSTR1)
671 slw rB,rWORD8,rSHL
672 or rWORD8,rG,rH
673 blt cr7,L(duP1x)
674 L(duP1e):
675 lwz rWORD1,4(rSTR1)
676 lwz rWORD2,4(rSTR2)
677 cmplw cr5,rWORD7,rWORD8
678 srw rA,rWORD2,rSHR
679 slw rD,rWORD2,rSHL
680 or rWORD2,rA,rB
681 lwz rWORD3,8(rSTR1)
682 lwz rWORD4,8(rSTR2)
683 cmplw cr0,rWORD1,rWORD2
684 srw rC,rWORD4,rSHR
685 slw rF,rWORD4,rSHL
686 bne cr5,L(duLcr5)
687 or rWORD4,rC,rD
688 lwz rWORD5,12(rSTR1)
689 lwz rWORD6,12(rSTR2)
690 cmplw cr1,rWORD3,rWORD4
691 srw rE,rWORD6,rSHR
692 slw rH,rWORD6,rSHL
693 bne cr0,L(duLcr0)
694 or rWORD6,rE,rF
695 cmplw cr6,rWORD5,rWORD6
696 b L(duLoop3)
697 .align 4
698 /* At this point we exit early with the first word compare
699 complete and remainder of 0 to 3 bytes. See L(du14) for details on
700 how we handle the remaining bytes. */
701 L(duP1x):
702 cmplw cr5,rWORD7,rWORD8
703 slwi. rN,rN,3
704 bne cr5,L(duLcr5)
705 cmplw cr7,rN,rSHR
706 beq L(duZeroReturn)
707 li rA,0
708 ble cr7,L(dutrim)
709 ld rWORD2,8(rSTR2)
710 srw rA,rWORD2,rSHR
711 b L(dutrim)
712 /* Remainder is 8 */
713 .align 4
714 L(duP2):
715 srw rE,rWORD8,rSHR
716 lwz rWORD5,0(rSTR1)
717 or rWORD6,rE,rH
718 slw rH,rWORD8,rSHL
719 L(duP2e):
720 lwz rWORD7,4(rSTR1)
721 lwz rWORD8,4(rSTR2)
722 cmplw cr6,rWORD5,rWORD6
723 srw rG,rWORD8,rSHR
724 slw rB,rWORD8,rSHL
725 or rWORD8,rG,rH
726 blt cr7,L(duP2x)
727 lwz rWORD1,8(rSTR1)
728 lwz rWORD2,8(rSTR2)
729 cmplw cr5,rWORD7,rWORD8
730 bne cr6,L(duLcr6)
731 srw rA,rWORD2,rSHR
732 slw rD,rWORD2,rSHL
733 or rWORD2,rA,rB
734 lwz rWORD3,12(rSTR1)
735 lwz rWORD4,12(rSTR2)
736 cmplw cr0,rWORD1,rWORD2
737 bne cr5,L(duLcr5)
738 srw rC,rWORD4,rSHR
739 slw rF,rWORD4,rSHL
740 or rWORD4,rC,rD
741 addi rSTR1,rSTR1,4
742 addi rSTR2,rSTR2,4
743 cmplw cr1,rWORD3,rWORD4
744 b L(duLoop2)
745 .align 4
746 L(duP2x):
747 cmplw cr5,rWORD7,rWORD8
748 addi rSTR1,rSTR1,4
749 addi rSTR2,rSTR2,4
750 bne cr6,L(duLcr6)
751 slwi. rN,rN,3
752 bne cr5,L(duLcr5)
753 cmplw cr7,rN,rSHR
754 beq L(duZeroReturn)
755 li rA,0
756 ble cr7,L(dutrim)
757 lwz rWORD2,4(rSTR2)
758 srw rA,rWORD2,rSHR
759 b L(dutrim)
761 /* Remainder is 12 */
762 .align 4
763 L(duP3):
764 srw rC,rWORD8,rSHR
765 lwz rWORD3,0(rSTR1)
766 slw rF,rWORD8,rSHL
767 or rWORD4,rC,rH
768 L(duP3e):
769 lwz rWORD5,4(rSTR1)
770 lwz rWORD6,4(rSTR2)
771 cmplw cr1,rWORD3,rWORD4
772 srw rE,rWORD6,rSHR
773 slw rH,rWORD6,rSHL
774 or rWORD6,rE,rF
775 lwz rWORD7,8(rSTR1)
776 lwz rWORD8,8(rSTR2)
777 cmplw cr6,rWORD5,rWORD6
778 bne cr1,L(duLcr1)
779 srw rG,rWORD8,rSHR
780 slw rB,rWORD8,rSHL
781 or rWORD8,rG,rH
782 blt cr7,L(duP3x)
783 lwz rWORD1,12(rSTR1)
784 lwz rWORD2,12(rSTR2)
785 cmplw cr5,rWORD7,rWORD8
786 bne cr6,L(duLcr6)
787 srw rA,rWORD2,rSHR
788 slw rD,rWORD2,rSHL
789 or rWORD2,rA,rB
790 addi rSTR1,rSTR1,8
791 addi rSTR2,rSTR2,8
792 cmplw cr0,rWORD1,rWORD2
793 b L(duLoop1)
794 .align 4
795 L(duP3x):
796 addi rSTR1,rSTR1,8
797 addi rSTR2,rSTR2,8
798 bne cr1,L(duLcr1)
799 cmplw cr5,rWORD7,rWORD8
800 bne cr6,L(duLcr6)
801 slwi. rN,rN,3
802 bne cr5,L(duLcr5)
803 cmplw cr7,rN,rSHR
804 beq L(duZeroReturn)
805 li rA,0
806 ble cr7,L(dutrim)
807 lwz rWORD2,4(rSTR2)
808 srw rA,rWORD2,rSHR
809 b L(dutrim)
811 /* Count is a multiple of 16, remainder is 0 */
812 .align 4
813 L(duP4):
814 mtctr rTMP
815 srw rA,rWORD8,rSHR
816 lwz rWORD1,0(rSTR1)
817 slw rD,rWORD8,rSHL
818 or rWORD2,rA,rH
819 L(duP4e):
820 lwz rWORD3,4(rSTR1)
821 lwz rWORD4,4(rSTR2)
822 cmplw cr0,rWORD1,rWORD2
823 srw rC,rWORD4,rSHR
824 slw rF,rWORD4,rSHL
825 or rWORD4,rC,rD
826 lwz rWORD5,8(rSTR1)
827 lwz rWORD6,8(rSTR2)
828 cmplw cr1,rWORD3,rWORD4
829 bne cr0,L(duLcr0)
830 srw rE,rWORD6,rSHR
831 slw rH,rWORD6,rSHL
832 or rWORD6,rE,rF
833 lwzu rWORD7,12(rSTR1)
834 lwzu rWORD8,12(rSTR2)
835 cmplw cr6,rWORD5,rWORD6
836 bne cr1,L(duLcr1)
837 srw rG,rWORD8,rSHR
838 slw rB,rWORD8,rSHL
839 or rWORD8,rG,rH
840 cmplw cr5,rWORD7,rWORD8
841 bdz L(du24) /* Adjust CTR as we start with +4 */
842 /* This is the primary loop */
843 .align 4
844 L(duLoop):
845 lwz rWORD1,4(rSTR1)
846 lwz rWORD2,4(rSTR2)
847 cmplw cr1,rWORD3,rWORD4
848 bne cr6,L(duLcr6)
849 srw rA,rWORD2,rSHR
850 slw rD,rWORD2,rSHL
851 or rWORD2,rA,rB
852 L(duLoop1):
853 lwz rWORD3,8(rSTR1)
854 lwz rWORD4,8(rSTR2)
855 cmplw cr6,rWORD5,rWORD6
856 bne cr5,L(duLcr5)
857 srw rC,rWORD4,rSHR
858 slw rF,rWORD4,rSHL
859 or rWORD4,rC,rD
860 L(duLoop2):
861 lwz rWORD5,12(rSTR1)
862 lwz rWORD6,12(rSTR2)
863 cmplw cr5,rWORD7,rWORD8
864 bne cr0,L(duLcr0)
865 srw rE,rWORD6,rSHR
866 slw rH,rWORD6,rSHL
867 or rWORD6,rE,rF
868 L(duLoop3):
869 lwzu rWORD7,16(rSTR1)
870 lwzu rWORD8,16(rSTR2)
871 cmplw cr0,rWORD1,rWORD2
872 bne cr1,L(duLcr1)
873 srw rG,rWORD8,rSHR
874 slw rB,rWORD8,rSHL
875 or rWORD8,rG,rH
876 bdnz L(duLoop)
878 L(duL4):
879 bne cr1,L(duLcr1)
880 cmplw cr1,rWORD3,rWORD4
881 bne cr6,L(duLcr6)
882 cmplw cr6,rWORD5,rWORD6
883 bne cr5,L(duLcr5)
884 cmplw cr5,rWORD7,rWORD8
885 L(du44):
886 bne cr0,L(duLcr0)
887 L(du34):
888 bne cr1,L(duLcr1)
889 L(du24):
890 bne cr6,L(duLcr6)
891 L(du14):
892 slwi. rN,rN,3
893 bne cr5,L(duLcr5)
894 /* At this point we have a remainder of 1 to 3 bytes to compare. We use
895 shift right to eliminate bits beyond the compare length.
897 However it may not be safe to load rWORD2 which may be beyond the
898 string length. So we compare the bit length of the remainder to
899 the right shift count (rSHR). If the bit count is less than or equal
900 we do not need to load rWORD2 (all significant bits are already in
901 rB). */
902 cmplw cr7,rN,rSHR
903 beq L(duZeroReturn)
904 li rA,0
905 ble cr7,L(dutrim)
906 lwz rWORD2,4(rSTR2)
907 srw rA,rWORD2,rSHR
908 .align 4
909 L(dutrim):
910 lwz rWORD1,4(rSTR1)
911 lwz r31,48(1)
912 subfic rN,rN,32 /* Shift count is 32 - (rN * 8). */
913 or rWORD2,rA,rB
914 lwz r30,44(1)
915 lwz r29,40(r1)
916 srw rWORD1,rWORD1,rN
917 srw rWORD2,rWORD2,rN
918 lwz r28,36(r1)
919 lwz r27,32(r1)
920 cmplw rWORD1,rWORD2
921 li rRTN,0
922 beq L(dureturn26)
923 li rRTN,1
924 bgt L(dureturn26)
925 li rRTN,-1
926 b L(dureturn26)
927 .align 4
928 L(duLcr0):
929 lwz r31,48(1)
930 lwz r30,44(1)
931 li rRTN,1
932 bgt cr0,L(dureturn29)
933 lwz r29,40(r1)
934 lwz r28,36(r1)
935 li rRTN,-1
936 b L(dureturn27)
937 .align 4
938 L(duLcr1):
939 lwz r31,48(1)
940 lwz r30,44(1)
941 li rRTN,1
942 bgt cr1,L(dureturn29)
943 lwz r29,40(r1)
944 lwz r28,36(r1)
945 li rRTN,-1
946 b L(dureturn27)
947 .align 4
948 L(duLcr6):
949 lwz r31,48(1)
950 lwz r30,44(1)
951 li rRTN,1
952 bgt cr6,L(dureturn29)
953 lwz r29,40(r1)
954 lwz r28,36(r1)
955 li rRTN,-1
956 b L(dureturn27)
957 .align 4
958 L(duLcr5):
959 lwz r31,48(1)
960 lwz r30,44(1)
961 li rRTN,1
962 bgt cr5,L(dureturn29)
963 lwz r29,40(r1)
964 lwz r28,36(r1)
965 li rRTN,-1
966 b L(dureturn27)
967 .align 3
968 L(duZeroReturn):
969 li rRTN,0
970 .align 4
971 L(dureturn):
972 lwz r31,48(1)
973 lwz r30,44(1)
974 L(dureturn29):
975 lwz r29,40(r1)
976 lwz r28,36(r1)
977 L(dureturn27):
978 lwz r27,32(r1)
979 L(dureturn26):
980 lwz r26,28(r1)
981 L(dureturn25):
982 lwz r25,24(r1)
983 lwz r24,20(r1)
984 lwz 1,0(1)
985 blr
986 END (BP_SYM (memcmp))
987 libc_hidden_builtin_def (memcmp)
988 weak_alias (memcmp,bcmp)