| blob | f1afffb4e728a5b45db6f571be9daf84e7bcd02d |
1 /* Optimized memcmp implementation for POWER7/PowerPC64.
2 Copyright (C) 2010 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA
18 02110-1301 USA. */
20 #include <sysdep.h>
21 #include <bp-sym.h>
22 #include <bp-asm.h>
24 /* int [r3] memcmp (const char *s1 [r3],
25 const char *s2 [r4],
26 size_t size [r5]) */
28 .machine power7
29 EALIGN (BP_SYM(memcmp),4,0)
30 CALL_MCOUNT 3
32 #define rTMP r0
33 #define rRTN r3
34 #define rSTR1 r3 /* first string arg */
35 #define rSTR2 r4 /* second string arg */
36 #define rN r5 /* max string length */
37 /* Note: The Bounded pointer support in this code is broken. This code
38 was inherited from PPC32 and and that support was never completed.
39 Current PPC gcc does not support -fbounds-check or -fbounded-pointers. */
40 #define rWORD1 r6 /* current word in s1 */
41 #define rWORD2 r7 /* current word in s2 */
42 #define rWORD3 r8 /* next word in s1 */
43 #define rWORD4 r9 /* next word in s2 */
44 #define rWORD5 r10 /* next word in s1 */
45 #define rWORD6 r11 /* next word in s2 */
46 #define rBITDIF r12 /* bits that differ in s1 & s2 words */
47 #define rWORD7 r30 /* next word in s1 */
48 #define rWORD8 r31 /* next word in s2 */
50 xor rTMP,rSTR2,rSTR1
51 cmpldi cr6,rN,0
52 cmpldi cr1,rN,12
53 clrldi. rTMP,rTMP,61
54 clrldi rBITDIF,rSTR1,61
55 cmpldi cr5,rBITDIF,0
56 beq- cr6,L(zeroLength)
57 dcbt 0,rSTR1
58 dcbt 0,rSTR2
59 /* If less than 8 bytes or not aligned, use the unalligned
60 byte loop. */
61 blt cr1,L(bytealigned)
62 std rWORD8,-8(r1)
63 cfi_offset(rWORD8,-8)
64 std rWORD7,-16(r1)
65 cfi_offset(rWORD7,-16)
66 bne L(unaligned)
67 /* At this point we know both strings have the same alignment and the
68 compare length is at least 8 bytes. rBITDIF containes the low order
69 3 bits of rSTR1 and cr5 contains the result of the logical compare
70 of rBITDIF to 0. If rBITDIF == 0 then we are already double word
71 aligned and can perform the DWaligned loop.
73 Otherwise we know the two strings have the same alignment (but not
74 yet DW). So we can force the string addresses to the next lower DW
75 boundary and special case this first DW word using shift left to
76 ellimiate bits preceeding the first byte. Since we want to join the
77 normal (DWaligned) compare loop, starting at the second double word,
78 we need to adjust the length (rN) and special case the loop
79 versioning for the first DW. This insures that the loop count is
80 correct and the first DW (shifted) is in the expected resister pair. */
81 .align 4
82 L(samealignment):
83 clrrdi rSTR1,rSTR1,3
84 clrrdi rSTR2,rSTR2,3
85 beq cr5,L(DWaligned)
86 add rN,rN,rBITDIF
87 sldi r11,rBITDIF,3
88 srdi rTMP,rN,5 /* Divide by 32 */
89 andi. rBITDIF,rN,24 /* Get the DW remainder */
90 ld rWORD1,0(rSTR1)
91 ld rWORD2,0(rSTR2)
92 cmpldi cr1,rBITDIF,16
93 cmpldi cr7,rN,32
94 clrldi rN,rN,61
95 beq L(dPs4)
96 mtctr rTMP
97 bgt cr1,L(dPs3)
98 beq cr1,L(dPs2)
100 /* Remainder is 8 */
101 .align 3
102 L(dsP1):
103 sld rWORD5,rWORD1,r11
104 sld rWORD6,rWORD2,r11
105 cmpld cr5,rWORD5,rWORD6
106 blt cr7,L(dP1x)
107 /* Do something useful in this cycle since we have to branch anyway. */
108 ld rWORD1,8(rSTR1)
109 ld rWORD2,8(rSTR2)
110 cmpld cr0,rWORD1,rWORD2
111 b L(dP1e)
112 /* Remainder is 16 */
113 .align 4
114 L(dPs2):
115 sld rWORD5,rWORD1,r11
116 sld rWORD6,rWORD2,r11
117 cmpld cr6,rWORD5,rWORD6
118 blt cr7,L(dP2x)
119 /* Do something useful in this cycle since we have to branch anyway. */
120 ld rWORD7,8(rSTR1)
121 ld rWORD8,8(rSTR2)
122 cmpld cr5,rWORD7,rWORD8
123 b L(dP2e)
124 /* Remainder is 24 */
125 .align 4
126 L(dPs3):
127 sld rWORD3,rWORD1,r11
128 sld rWORD4,rWORD2,r11
129 cmpld cr1,rWORD3,rWORD4
130 b L(dP3e)
131 /* Count is a multiple of 32, remainder is 0 */
132 .align 4
133 L(dPs4):
134 mtctr rTMP
135 sld rWORD1,rWORD1,r11
136 sld rWORD2,rWORD2,r11
137 cmpld cr0,rWORD1,rWORD2
138 b L(dP4e)
140 /* At this point we know both strings are double word aligned and the
141 compare length is at least 8 bytes. */
142 .align 4
143 L(DWaligned):
144 andi. rBITDIF,rN,24 /* Get the DW remainder */
145 srdi rTMP,rN,5 /* Divide by 32 */
146 cmpldi cr1,rBITDIF,16
147 cmpldi cr7,rN,32
148 clrldi rN,rN,61
149 beq L(dP4)
150 bgt cr1,L(dP3)
151 beq cr1,L(dP2)
153 /* Remainder is 8 */
154 .align 4
155 L(dP1):
156 mtctr rTMP
157 /* Normally we'd use rWORD7/rWORD8 here, but since we might exit early
158 (8-15 byte compare), we want to use only volitile registers. This
159 means we can avoid restoring non-volitile registers since we did not
160 change any on the early exit path. The key here is the non-early
161 exit path only cares about the condition code (cr5), not about which
162 register pair was used. */
163 ld rWORD5,0(rSTR1)
164 ld rWORD6,0(rSTR2)
165 cmpld cr5,rWORD5,rWORD6
166 blt cr7,L(dP1x)
167 ld rWORD1,8(rSTR1)
168 ld rWORD2,8(rSTR2)
169 cmpld cr0,rWORD1,rWORD2
170 L(dP1e):
171 ld rWORD3,16(rSTR1)
172 ld rWORD4,16(rSTR2)
173 cmpld cr1,rWORD3,rWORD4
174 ld rWORD5,24(rSTR1)
175 ld rWORD6,24(rSTR2)
176 cmpld cr6,rWORD5,rWORD6
177 bne cr5,L(dLcr5)
178 bne cr0,L(dLcr0)
180 ldu rWORD7,32(rSTR1)
181 ldu rWORD8,32(rSTR2)
182 bne cr1,L(dLcr1)
183 cmpld cr5,rWORD7,rWORD8
184 bdnz L(dLoop)
185 bne cr6,L(dLcr6)
186 ld rWORD8,-8(r1)
187 ld rWORD7,-16(r1)
188 .align 3
189 L(dP1x):
190 sldi. r12,rN,3
191 bne cr5,L(dLcr5)
192 subfic rN,r12,64 /* Shift count is 64 - (rN * 8). */
193 bne L(d00)
194 li rRTN,0
195 blr
197 /* Remainder is 16 */
198 .align 4
199 L(dP2):
200 mtctr rTMP
201 ld rWORD5,0(rSTR1)
202 ld rWORD6,0(rSTR2)
203 cmpld cr6,rWORD5,rWORD6
204 blt cr7,L(dP2x)
205 ld rWORD7,8(rSTR1)
206 ld rWORD8,8(rSTR2)
207 cmpld cr5,rWORD7,rWORD8
208 L(dP2e):
209 ld rWORD1,16(rSTR1)
210 ld rWORD2,16(rSTR2)
211 cmpld cr0,rWORD1,rWORD2
212 ld rWORD3,24(rSTR1)
213 ld rWORD4,24(rSTR2)
214 cmpld cr1,rWORD3,rWORD4
215 addi rSTR1,rSTR1,8
216 addi rSTR2,rSTR2,8
217 bne cr6,L(dLcr6)
218 bne cr5,L(dLcr5)
219 b L(dLoop2)
220 /* Again we are on a early exit path (16-23 byte compare), we want to
221 only use volitile registers and avoid restoring non-volitile
222 registers. */
223 .align 4
224 L(dP2x):
225 ld rWORD3,8(rSTR1)
226 ld rWORD4,8(rSTR2)
227 cmpld cr5,rWORD3,rWORD4
228 sldi. r12,rN,3
229 bne cr6,L(dLcr6)
230 addi rSTR1,rSTR1,8
231 addi rSTR2,rSTR2,8
232 bne cr5,L(dLcr5)
233 subfic rN,r12,64 /* Shift count is 64 - (rN * 8). */
234 bne L(d00)
235 li rRTN,0
236 blr
238 /* Remainder is 24 */
239 .align 4
240 L(dP3):
241 mtctr rTMP
242 ld rWORD3,0(rSTR1)
243 ld rWORD4,0(rSTR2)
244 cmpld cr1,rWORD3,rWORD4
245 L(dP3e):
246 ld rWORD5,8(rSTR1)
247 ld rWORD6,8(rSTR2)
248 cmpld cr6,rWORD5,rWORD6
249 blt cr7,L(dP3x)
250 ld rWORD7,16(rSTR1)
251 ld rWORD8,16(rSTR2)
252 cmpld cr5,rWORD7,rWORD8
253 ld rWORD1,24(rSTR1)
254 ld rWORD2,24(rSTR2)
255 cmpld cr0,rWORD1,rWORD2
256 addi rSTR1,rSTR1,16
257 addi rSTR2,rSTR2,16
258 bne cr1,L(dLcr1)
259 bne cr6,L(dLcr6)
260 b L(dLoop1)
261 /* Again we are on a early exit path (24-31 byte compare), we want to
262 only use volitile registers and avoid restoring non-volitile
263 registers. */
264 .align 4
265 L(dP3x):
266 ld rWORD1,16(rSTR1)
267 ld rWORD2,16(rSTR2)
268 cmpld cr5,rWORD1,rWORD2
269 sldi. r12,rN,3
270 bne cr1,L(dLcr1)
271 addi rSTR1,rSTR1,16
272 addi rSTR2,rSTR2,16
273 bne cr6,L(dLcr6)
274 subfic rN,r12,64 /* Shift count is 64 - (rN * 8). */
275 bne cr5,L(dLcr5)
276 bne L(d00)
277 li rRTN,0
278 blr
280 /* Count is a multiple of 32, remainder is 0 */
281 .align 4
282 L(dP4):
283 mtctr rTMP
284 ld rWORD1,0(rSTR1)
285 ld rWORD2,0(rSTR2)
286 cmpld cr0,rWORD1,rWORD2
287 L(dP4e):
288 ld rWORD3,8(rSTR1)
289 ld rWORD4,8(rSTR2)
290 cmpld cr1,rWORD3,rWORD4
291 ld rWORD5,16(rSTR1)
292 ld rWORD6,16(rSTR2)
293 cmpld cr6,rWORD5,rWORD6
294 ldu rWORD7,24(rSTR1)
295 ldu rWORD8,24(rSTR2)
296 cmpld cr5,rWORD7,rWORD8
297 bne cr0,L(dLcr0)
298 bne cr1,L(dLcr1)
299 bdz- L(d24) /* Adjust CTR as we start with +4 */
300 /* This is the primary loop */
301 .align 4
302 L(dLoop):
303 ld rWORD1,8(rSTR1)
304 ld rWORD2,8(rSTR2)
305 cmpld cr1,rWORD3,rWORD4
306 bne cr6,L(dLcr6)
307 L(dLoop1):
308 ld rWORD3,16(rSTR1)
309 ld rWORD4,16(rSTR2)
310 cmpld cr6,rWORD5,rWORD6
311 bne cr5,L(dLcr5)
312 L(dLoop2):
313 ld rWORD5,24(rSTR1)
314 ld rWORD6,24(rSTR2)
315 cmpld cr5,rWORD7,rWORD8
316 bne cr0,L(dLcr0)
317 L(dLoop3):
318 ldu rWORD7,32(rSTR1)
319 ldu rWORD8,32(rSTR2)
320 bne cr1,L(dLcr1)
321 cmpld cr0,rWORD1,rWORD2
322 bdnz L(dLoop)
324 L(dL4):
325 cmpld cr1,rWORD3,rWORD4
326 bne cr6,L(dLcr6)
327 cmpld cr6,rWORD5,rWORD6
328 bne cr5,L(dLcr5)
329 cmpld cr5,rWORD7,rWORD8
330 L(d44):
331 bne cr0,L(dLcr0)
332 L(d34):
333 bne cr1,L(dLcr1)
334 L(d24):
335 bne cr6,L(dLcr6)
336 L(d14):
337 sldi. r12,rN,3
338 bne cr5,L(dLcr5)
339 L(d04):
340 ld rWORD8,-8(r1)
341 ld rWORD7,-16(r1)
342 subfic rN,r12,64 /* Shift count is 64 - (rN * 8). */
343 beq L(zeroLength)
344 /* At this point we have a remainder of 1 to 7 bytes to compare. Since
345 we are aligned it is safe to load the whole double word, and use
346 shift right double to elliminate bits beyond the compare length. */
347 L(d00):
348 ld rWORD1,8(rSTR1)
349 ld rWORD2,8(rSTR2)
350 srd rWORD1,rWORD1,rN
351 srd rWORD2,rWORD2,rN
352 cmpld cr5,rWORD1,rWORD2
353 bne cr5,L(dLcr5x)
354 li rRTN,0
355 blr
356 .align 4
357 L(dLcr0):
358 ld rWORD8,-8(r1)
359 ld rWORD7,-16(r1)
360 li rRTN,1
361 bgtlr cr0
362 li rRTN,-1
363 blr
364 .align 4
365 L(dLcr1):
366 ld rWORD8,-8(r1)
367 ld rWORD7,-16(r1)
368 li rRTN,1
369 bgtlr cr1
370 li rRTN,-1
371 blr
372 .align 4
373 L(dLcr6):
374 ld rWORD8,-8(r1)
375 ld rWORD7,-16(r1)
376 li rRTN,1
377 bgtlr cr6
378 li rRTN,-1
379 blr
380 .align 4
381 L(dLcr5):
382 ld rWORD8,-8(r1)
383 ld rWORD7,-16(r1)
384 L(dLcr5x):
385 li rRTN,1
386 bgtlr cr5
387 li rRTN,-1
388 blr
390 .align 4
391 L(bytealigned):
392 mtctr rN
393 beq cr6,L(zeroLength)
395 /* We need to prime this loop. This loop is swing modulo scheduled
396 to avoid pipe delays. The dependent instruction latencies (load to
397 compare to conditional branch) is 2 to 3 cycles. In this loop each
398 dispatch group ends in a branch and takes 1 cycle. Effectively
399 the first iteration of the loop only serves to load operands and
400 branches based on compares are delayed until the next loop.
402 So we must precondition some registers and condition codes so that
403 we don't exit the loop early on the first iteration. */
405 lbz rWORD1,0(rSTR1)
406 lbz rWORD2,0(rSTR2)
407 bdz L(b11)
408 cmpld cr0,rWORD1,rWORD2
409 lbz rWORD3,1(rSTR1)
410 lbz rWORD4,1(rSTR2)
411 bdz L(b12)
412 cmpld cr1,rWORD3,rWORD4
413 lbzu rWORD5,2(rSTR1)
414 lbzu rWORD6,2(rSTR2)
415 bdz L(b13)
416 .align 4
417 L(bLoop):
418 lbzu rWORD1,1(rSTR1)
419 lbzu rWORD2,1(rSTR2)
420 bne cr0,L(bLcr0)
422 cmpld cr6,rWORD5,rWORD6
423 bdz L(b3i)
425 lbzu rWORD3,1(rSTR1)
426 lbzu rWORD4,1(rSTR2)
427 bne cr1,L(bLcr1)
429 cmpld cr0,rWORD1,rWORD2
430 bdz L(b2i)
432 lbzu rWORD5,1(rSTR1)
433 lbzu rWORD6,1(rSTR2)
434 bne cr6,L(bLcr6)
436 cmpld cr1,rWORD3,rWORD4
437 bdnz L(bLoop)
439 /* We speculatively loading bytes before we have tested the previous
440 bytes. But we must avoid overrunning the length (in the ctr) to
441 prevent these speculative loads from causing a segfault. In this
442 case the loop will exit early (before the all pending bytes are
443 tested. In this case we must complete the pending operations
444 before returning. */
445 L(b1i):
446 bne cr0,L(bLcr0)
447 bne cr1,L(bLcr1)
448 b L(bx56)
449 .align 4
450 L(b2i):
451 bne cr6,L(bLcr6)
452 bne cr0,L(bLcr0)
453 b L(bx34)
454 .align 4
455 L(b3i):
456 bne cr1,L(bLcr1)
457 bne cr6,L(bLcr6)
458 b L(bx12)
459 .align 4
460 L(bLcr0):
461 li rRTN,1
462 bgtlr cr0
463 li rRTN,-1
464 blr
465 L(bLcr1):
466 li rRTN,1
467 bgtlr cr1
468 li rRTN,-1
469 blr
470 L(bLcr6):
471 li rRTN,1
472 bgtlr cr6
473 li rRTN,-1
474 blr
476 L(b13):
477 bne cr0,L(bx12)
478 bne cr1,L(bx34)
479 L(bx56):
480 sub rRTN,rWORD5,rWORD6
481 blr
482 nop
483 L(b12):
484 bne cr0,L(bx12)
485 L(bx34):
486 sub rRTN,rWORD3,rWORD4
487 blr
488 L(b11):
489 L(bx12):
490 sub rRTN,rWORD1,rWORD2
491 blr
492 .align 4
493 L(zeroLengthReturn):
494 ld rWORD8,-8(r1)
495 ld rWORD7,-16(r1)
496 L(zeroLength):
497 li rRTN,0
498 blr
500 .align 4
501 /* At this point we know the strings have different alignment and the
502 compare length is at least 8 bytes. rBITDIF containes the low order
503 3 bits of rSTR1 and cr5 contains the result of the logical compare
504 of rBITDIF to 0. If rBITDIF == 0 then rStr1 is double word
505 aligned and can perform the DWunaligned loop.
507 Otherwise we know that rSTR1 is not aready DW aligned yet.
508 So we can force the string addresses to the next lower DW
509 boundary and special case this first DW word using shift left to
510 ellimiate bits preceeding the first byte. Since we want to join the
511 normal (DWaligned) compare loop, starting at the second double word,
512 we need to adjust the length (rN) and special case the loop
513 versioning for the first DW. This insures that the loop count is
514 correct and the first DW (shifted) is in the expected resister pair. */
515 #define rSHL r29 /* Unaligned shift left count. */
516 #define rSHR r28 /* Unaligned shift right count. */
517 #define rB r27 /* Left rotation temp for rWORD2. */
518 #define rD r26 /* Left rotation temp for rWORD4. */
519 #define rF r25 /* Left rotation temp for rWORD6. */
520 #define rH r24 /* Left rotation temp for rWORD8. */
521 #define rA r0 /* Right rotation temp for rWORD2. */
522 #define rC r12 /* Right rotation temp for rWORD4. */
523 #define rE r0 /* Right rotation temp for rWORD6. */
524 #define rG r12 /* Right rotation temp for rWORD8. */
525 L(unaligned):
526 std r29,-24(r1)
527 cfi_offset(r29,-24)
528 clrldi rSHL,rSTR2,61
529 beq cr6,L(duzeroLength)
530 std r28,-32(r1)
531 cfi_offset(r28,-32)
532 beq cr5,L(DWunaligned)
533 std r27,-40(r1)
534 cfi_offset(r27,-40)
535 /* Adjust the logical start of rSTR2 ro compensate for the extra bits
536 in the 1st rSTR1 DW. */
537 sub r27,rSTR2,rBITDIF
538 /* But do not attempt to address the DW before that DW that contains
539 the actual start of rSTR2. */
540 clrrdi rSTR2,rSTR2,3
541 std r26,-48(r1)
542 cfi_offset(r26,-48)
543 /* Compute the leaft/right shift counts for the unalign rSTR2,
544 compensating for the logical (DW aligned) start of rSTR1. */
545 clrldi rSHL,r27,61
546 clrrdi rSTR1,rSTR1,3
547 std r25,-56(r1)
548 cfi_offset(r25,-56)
549 sldi rSHL,rSHL,3
550 cmpld cr5,r27,rSTR2
551 add rN,rN,rBITDIF
552 sldi r11,rBITDIF,3
553 std r24,-64(r1)
554 cfi_offset(r24,-64)
555 subfic rSHR,rSHL,64
556 srdi rTMP,rN,5 /* Divide by 32 */
557 andi. rBITDIF,rN,24 /* Get the DW remainder */
558 /* We normally need to load 2 DWs to start the unaligned rSTR2, but in
559 this special case those bits may be discarded anyway. Also we
560 must avoid loading a DW where none of the bits are part of rSTR2 as
561 this may cross a page boundary and cause a page fault. */
562 li rWORD8,0
563 blt cr5,L(dus0)
564 ld rWORD8,0(rSTR2)
565 la rSTR2,8(rSTR2)
566 sld rWORD8,rWORD8,rSHL
568 L(dus0):
569 ld rWORD1,0(rSTR1)
570 ld rWORD2,0(rSTR2)
571 cmpldi cr1,rBITDIF,16
572 cmpldi cr7,rN,32
573 srd rG,rWORD2,rSHR
574 clrldi rN,rN,61
575 beq L(duPs4)
576 mtctr rTMP
577 or rWORD8,rG,rWORD8
578 bgt cr1,L(duPs3)
579 beq cr1,L(duPs2)
581 /* Remainder is 8 */
582 .align 4
583 L(dusP1):
584 sld rB,rWORD2,rSHL
585 sld rWORD7,rWORD1,r11
586 sld rWORD8,rWORD8,r11
587 bge cr7,L(duP1e)
588 /* At this point we exit early with the first double word compare
589 complete and remainder of 0 to 7 bytes. See L(du14) for details on
590 how we handle the remaining bytes. */
591 cmpld cr5,rWORD7,rWORD8
592 sldi. rN,rN,3
593 bne cr5,L(duLcr5)
594 cmpld cr7,rN,rSHR
595 beq L(duZeroReturn)
596 li rA,0
597 ble cr7,L(dutrim)
598 ld rWORD2,8(rSTR2)
599 srd rA,rWORD2,rSHR
600 b L(dutrim)
601 /* Remainder is 16 */
602 .align 4
603 L(duPs2):
604 sld rH,rWORD2,rSHL
605 sld rWORD5,rWORD1,r11
606 sld rWORD6,rWORD8,r11
607 b L(duP2e)
608 /* Remainder is 24 */
609 .align 4
610 L(duPs3):
611 sld rF,rWORD2,rSHL
612 sld rWORD3,rWORD1,r11
613 sld rWORD4,rWORD8,r11
614 b L(duP3e)
615 /* Count is a multiple of 32, remainder is 0 */
616 .align 4
617 L(duPs4):
618 mtctr rTMP
619 or rWORD8,rG,rWORD8
620 sld rD,rWORD2,rSHL
621 sld rWORD1,rWORD1,r11
622 sld rWORD2,rWORD8,r11
623 b L(duP4e)
625 /* At this point we know rSTR1 is double word aligned and the
626 compare length is at least 8 bytes. */
627 .align 4
628 L(DWunaligned):
629 std r27,-40(r1)
630 cfi_offset(r27,-40)
631 clrrdi rSTR2,rSTR2,3
632 std r26,-48(r1)
633 cfi_offset(r26,-48)
634 srdi rTMP,rN,5 /* Divide by 32 */
635 std r25,-56(r1)
636 cfi_offset(r25,-56)
637 andi. rBITDIF,rN,24 /* Get the DW remainder */
638 std r24,-64(r1)
639 cfi_offset(r24,-64)
640 sldi rSHL,rSHL,3
641 ld rWORD6,0(rSTR2)
642 ldu rWORD8,8(rSTR2)
643 cmpldi cr1,rBITDIF,16
644 cmpldi cr7,rN,32
645 clrldi rN,rN,61
646 subfic rSHR,rSHL,64
647 sld rH,rWORD6,rSHL
648 beq L(duP4)
649 mtctr rTMP
650 bgt cr1,L(duP3)
651 beq cr1,L(duP2)
653 /* Remainder is 8 */
654 .align 4
655 L(duP1):
656 srd rG,rWORD8,rSHR
657 ld rWORD7,0(rSTR1)
658 sld rB,rWORD8,rSHL
659 or rWORD8,rG,rH
660 blt cr7,L(duP1x)
661 L(duP1e):
662 ld rWORD1,8(rSTR1)
663 ld rWORD2,8(rSTR2)
664 cmpld cr5,rWORD7,rWORD8
665 srd rA,rWORD2,rSHR
666 sld rD,rWORD2,rSHL
667 or rWORD2,rA,rB
668 ld rWORD3,16(rSTR1)
669 ld rWORD4,16(rSTR2)
670 cmpld cr0,rWORD1,rWORD2
671 srd rC,rWORD4,rSHR
672 sld rF,rWORD4,rSHL
673 bne cr5,L(duLcr5)
674 or rWORD4,rC,rD
675 ld rWORD5,24(rSTR1)
676 ld rWORD6,24(rSTR2)
677 cmpld cr1,rWORD3,rWORD4
678 srd rE,rWORD6,rSHR
679 sld rH,rWORD6,rSHL
680 bne cr0,L(duLcr0)
681 or rWORD6,rE,rF
682 cmpld cr6,rWORD5,rWORD6
683 b L(duLoop3)
684 .align 4
685 /* At this point we exit early with the first double word compare
686 complete and remainder of 0 to 7 bytes. See L(du14) for details on
687 how we handle the remaining bytes. */
688 L(duP1x):
689 cmpld cr5,rWORD7,rWORD8
690 sldi. rN,rN,3
691 bne cr5,L(duLcr5)
692 cmpld cr7,rN,rSHR
693 beq L(duZeroReturn)
694 li rA,0
695 ble cr7,L(dutrim)
696 ld rWORD2,8(rSTR2)
697 srd rA,rWORD2,rSHR
698 b L(dutrim)
699 /* Remainder is 16 */
700 .align 4
701 L(duP2):
702 srd rE,rWORD8,rSHR
703 ld rWORD5,0(rSTR1)
704 or rWORD6,rE,rH
705 sld rH,rWORD8,rSHL
706 L(duP2e):
707 ld rWORD7,8(rSTR1)
708 ld rWORD8,8(rSTR2)
709 cmpld cr6,rWORD5,rWORD6
710 srd rG,rWORD8,rSHR
711 sld rB,rWORD8,rSHL
712 or rWORD8,rG,rH
713 blt cr7,L(duP2x)
714 ld rWORD1,16(rSTR1)
715 ld rWORD2,16(rSTR2)
716 cmpld cr5,rWORD7,rWORD8
717 bne cr6,L(duLcr6)
718 srd rA,rWORD2,rSHR
719 sld rD,rWORD2,rSHL
720 or rWORD2,rA,rB
721 ld rWORD3,24(rSTR1)
722 ld rWORD4,24(rSTR2)
723 cmpld cr0,rWORD1,rWORD2
724 bne cr5,L(duLcr5)
725 srd rC,rWORD4,rSHR
726 sld rF,rWORD4,rSHL
727 or rWORD4,rC,rD
728 addi rSTR1,rSTR1,8
729 addi rSTR2,rSTR2,8
730 cmpld cr1,rWORD3,rWORD4
731 b L(duLoop2)
732 .align 4
733 L(duP2x):
734 cmpld cr5,rWORD7,rWORD8
735 addi rSTR1,rSTR1,8
736 addi rSTR2,rSTR2,8
737 bne cr6,L(duLcr6)
738 sldi. rN,rN,3
739 bne cr5,L(duLcr5)
740 cmpld cr7,rN,rSHR
741 beq L(duZeroReturn)
742 li rA,0
743 ble cr7,L(dutrim)
744 ld rWORD2,8(rSTR2)
745 srd rA,rWORD2,rSHR
746 b L(dutrim)
748 /* Remainder is 24 */
749 .align 4
750 L(duP3):
751 srd rC,rWORD8,rSHR
752 ld rWORD3,0(rSTR1)
753 sld rF,rWORD8,rSHL
754 or rWORD4,rC,rH
755 L(duP3e):
756 ld rWORD5,8(rSTR1)
757 ld rWORD6,8(rSTR2)
758 cmpld cr1,rWORD3,rWORD4
759 srd rE,rWORD6,rSHR
760 sld rH,rWORD6,rSHL
761 or rWORD6,rE,rF
762 ld rWORD7,16(rSTR1)
763 ld rWORD8,16(rSTR2)
764 cmpld cr6,rWORD5,rWORD6
765 bne cr1,L(duLcr1)
766 srd rG,rWORD8,rSHR
767 sld rB,rWORD8,rSHL
768 or rWORD8,rG,rH
769 blt cr7,L(duP3x)
770 ld rWORD1,24(rSTR1)
771 ld rWORD2,24(rSTR2)
772 cmpld cr5,rWORD7,rWORD8
773 bne cr6,L(duLcr6)
774 srd rA,rWORD2,rSHR
775 sld rD,rWORD2,rSHL
776 or rWORD2,rA,rB
777 addi rSTR1,rSTR1,16
778 addi rSTR2,rSTR2,16
779 cmpld cr0,rWORD1,rWORD2
780 b L(duLoop1)
781 .align 4
782 L(duP3x):
783 addi rSTR1,rSTR1,16
784 addi rSTR2,rSTR2,16
785 bne cr1,L(duLcr1)
786 cmpld cr5,rWORD7,rWORD8
787 bne cr6,L(duLcr6)
788 sldi. rN,rN,3
789 bne cr5,L(duLcr5)
790 cmpld cr7,rN,rSHR
791 beq L(duZeroReturn)
792 li rA,0
793 ble cr7,L(dutrim)
794 ld rWORD2,8(rSTR2)
795 srd rA,rWORD2,rSHR
796 b L(dutrim)
798 /* Count is a multiple of 32, remainder is 0 */
799 .align 4
800 L(duP4):
801 mtctr rTMP
802 srd rA,rWORD8,rSHR
803 ld rWORD1,0(rSTR1)
804 sld rD,rWORD8,rSHL
805 or rWORD2,rA,rH
806 L(duP4e):
807 ld rWORD3,8(rSTR1)
808 ld rWORD4,8(rSTR2)
809 cmpld cr0,rWORD1,rWORD2
810 srd rC,rWORD4,rSHR
811 sld rF,rWORD4,rSHL
812 or rWORD4,rC,rD
813 ld rWORD5,16(rSTR1)
814 ld rWORD6,16(rSTR2)
815 cmpld cr1,rWORD3,rWORD4
816 bne cr0,L(duLcr0)
817 srd rE,rWORD6,rSHR
818 sld rH,rWORD6,rSHL
819 or rWORD6,rE,rF
820 ldu rWORD7,24(rSTR1)
821 ldu rWORD8,24(rSTR2)
822 cmpld cr6,rWORD5,rWORD6
823 bne cr1,L(duLcr1)
824 srd rG,rWORD8,rSHR
825 sld rB,rWORD8,rSHL
826 or rWORD8,rG,rH
827 cmpld cr5,rWORD7,rWORD8
828 bdz L(du24) /* Adjust CTR as we start with +4 */
829 /* This is the primary loop */
830 .align 4
831 L(duLoop):
832 ld rWORD1,8(rSTR1)
833 ld rWORD2,8(rSTR2)
834 cmpld cr1,rWORD3,rWORD4
835 bne cr6,L(duLcr6)
836 srd rA,rWORD2,rSHR
837 sld rD,rWORD2,rSHL
838 or rWORD2,rA,rB
839 L(duLoop1):
840 ld rWORD3,16(rSTR1)
841 ld rWORD4,16(rSTR2)
842 cmpld cr6,rWORD5,rWORD6
843 bne cr5,L(duLcr5)
844 srd rC,rWORD4,rSHR
845 sld rF,rWORD4,rSHL
846 or rWORD4,rC,rD
847 L(duLoop2):
848 ld rWORD5,24(rSTR1)
849 ld rWORD6,24(rSTR2)
850 cmpld cr5,rWORD7,rWORD8
851 bne cr0,L(duLcr0)
852 srd rE,rWORD6,rSHR
853 sld rH,rWORD6,rSHL
854 or rWORD6,rE,rF
855 L(duLoop3):
856 ldu rWORD7,32(rSTR1)
857 ldu rWORD8,32(rSTR2)
858 cmpld cr0,rWORD1,rWORD2
859 bne- cr1,L(duLcr1)
860 srd rG,rWORD8,rSHR
861 sld rB,rWORD8,rSHL
862 or rWORD8,rG,rH
863 bdnz L(duLoop)
865 L(duL4):
866 bne cr1,L(duLcr1)
867 cmpld cr1,rWORD3,rWORD4
868 bne cr6,L(duLcr6)
869 cmpld cr6,rWORD5,rWORD6
870 bne cr5,L(duLcr5)
871 cmpld cr5,rWORD7,rWORD8
872 L(du44):
873 bne cr0,L(duLcr0)
874 L(du34):
875 bne cr1,L(duLcr1)
876 L(du24):
877 bne cr6,L(duLcr6)
878 L(du14):
879 sldi. rN,rN,3
880 bne cr5,L(duLcr5)
881 /* At this point we have a remainder of 1 to 7 bytes to compare. We use
882 shift right double to elliminate bits beyond the compare length.
883 This allows the use of double word subtract to compute the final
884 result.
886 However it may not be safe to load rWORD2 which may be beyond the
887 string length. So we compare the bit length of the remainder to
888 the right shift count (rSHR). If the bit count is less than or equal
889 we do not need to load rWORD2 (all significant bits are already in
890 rB). */
891 cmpld cr7,rN,rSHR
892 beq L(duZeroReturn)
893 li rA,0
894 ble cr7,L(dutrim)
895 ld rWORD2,8(rSTR2)
896 srd rA,rWORD2,rSHR
897 .align 4
898 L(dutrim):
899 ld rWORD1,8(rSTR1)
900 ld rWORD8,-8(r1)
901 subfic rN,rN,64 /* Shift count is 64 - (rN * 8). */
902 or rWORD2,rA,rB
903 ld rWORD7,-16(r1)
904 ld r29,-24(r1)
905 srd rWORD1,rWORD1,rN
906 srd rWORD2,rWORD2,rN
907 ld r28,-32(r1)
908 ld r27,-40(r1)
909 li rRTN,0
910 cmpld cr0,rWORD1,rWORD2
911 ld r26,-48(r1)
912 ld r25,-56(r1)
913 beq cr0,L(dureturn24)
914 li rRTN,1
915 ld r24,-64(r1)
916 bgtlr cr0
917 li rRTN,-1
918 blr
919 .align 4
920 L(duLcr0):
921 ld rWORD8,-8(r1)
922 ld rWORD7,-16(r1)
923 li rRTN,1
924 bgt cr0,L(dureturn29)
925 ld r29,-24(r1)
926 ld r28,-32(r1)
927 li rRTN,-1
928 b L(dureturn27)
929 .align 4
930 L(duLcr1):
931 ld rWORD8,-8(r1)
932 ld rWORD7,-16(r1)
933 li rRTN,1
934 bgt cr1,L(dureturn29)
935 ld r29,-24(r1)
936 ld r28,-32(r1)
937 li rRTN,-1
938 b L(dureturn27)
939 .align 4
940 L(duLcr6):
941 ld rWORD8,-8(r1)
942 ld rWORD7,-16(r1)
943 li rRTN,1
944 bgt cr6,L(dureturn29)
945 ld r29,-24(r1)
946 ld r28,-32(r1)
947 li rRTN,-1
948 b L(dureturn27)
949 .align 4
950 L(duLcr5):
951 ld rWORD8,-8(r1)
952 ld rWORD7,-16(r1)
953 li rRTN,1
954 bgt cr5,L(dureturn29)
955 ld r29,-24(r1)
956 ld r28,-32(r1)
957 li rRTN,-1
958 b L(dureturn27)
959 .align 3
960 L(duZeroReturn):
961 li rRTN,0
962 .align 4
963 L(dureturn):
964 ld rWORD8,-8(r1)
965 ld rWORD7,-16(r1)
966 L(dureturn29):
967 ld r29,-24(r1)
968 ld r28,-32(r1)
969 L(dureturn27):
970 ld r27,-40(r1)
971 L(dureturn26):
972 ld r26,-48(r1)
973 L(dureturn25):
974 ld r25,-56(r1)
975 L(dureturn24):
976 ld r24,-64(r1)
977 blr
978 L(duzeroLength):
979 li rRTN,0
980 blr
982 END (BP_SYM (memcmp))
983 libc_hidden_builtin_def (memcmp)
984 weak_alias (memcmp,bcmp)