| blob | 2b6091027dae3e6868cce480776846a8904c2c21 |
1 /*
2 * String handling functions for PowerPC.
3 *
4 * Copyright (C) 1996 Paul Mackerras.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11 #include <linux/config.h>
12 #include <asm/processor.h>
13 #include <asm/cache.h>
14 #include <asm/errno.h>
15 #include <asm/ppc_asm.h>
17 #define COPY_16_BYTES \
18 lwz r7,4(r4); \
19 lwz r8,8(r4); \
20 lwz r9,12(r4); \
21 lwzu r10,16(r4); \
22 stw r7,4(r6); \
23 stw r8,8(r6); \
24 stw r9,12(r6); \
25 stwu r10,16(r6)
27 #define COPY_16_BYTES_WITHEX(n) \
28 8 ## n ## 0: \
29 lwz r7,4(r4); \
30 8 ## n ## 1: \
31 lwz r8,8(r4); \
32 8 ## n ## 2: \
33 lwz r9,12(r4); \
34 8 ## n ## 3: \
35 lwzu r10,16(r4); \
36 8 ## n ## 4: \
37 stw r7,4(r6); \
38 8 ## n ## 5: \
39 stw r8,8(r6); \
40 8 ## n ## 6: \
41 stw r9,12(r6); \
42 8 ## n ## 7: \
43 stwu r10,16(r6)
45 #define COPY_16_BYTES_EXCODE(n) \
46 9 ## n ## 0: \
47 addi r5,r5,-(16 * n); \
48 b 104f; \
49 9 ## n ## 1: \
50 addi r5,r5,-(16 * n); \
51 b 105f; \
52 .section __ex_table,"a"; \
53 .align 2; \
54 .long 8 ## n ## 0b,9 ## n ## 0b; \
55 .long 8 ## n ## 1b,9 ## n ## 0b; \
56 .long 8 ## n ## 2b,9 ## n ## 0b; \
57 .long 8 ## n ## 3b,9 ## n ## 0b; \
58 .long 8 ## n ## 4b,9 ## n ## 1b; \
59 .long 8 ## n ## 5b,9 ## n ## 1b; \
60 .long 8 ## n ## 6b,9 ## n ## 1b; \
61 .long 8 ## n ## 7b,9 ## n ## 1b; \
62 .text
64 .text
65 .stabs "arch/ppc/lib/",N_SO,0,0,0f
66 .stabs "string.S",N_SO,0,0,0f
68 CACHELINE_BYTES = L1_CACHE_LINE_SIZE
69 LG_CACHELINE_BYTES = LG_L1_CACHE_LINE_SIZE
70 CACHELINE_MASK = (L1_CACHE_LINE_SIZE-1)
72 _GLOBAL(strcpy)
73 addi r5,r3,-1
74 addi r4,r4,-1
75 1: lbzu r0,1(r4)
76 cmpwi 0,r0,0
77 stbu r0,1(r5)
78 bne 1b
79 blr
81 /* This clears out any unused part of the destination buffer,
82 just as the libc version does. -- paulus */
83 _GLOBAL(strncpy)
84 cmpwi 0,r5,0
85 beqlr
86 mtctr r5
87 addi r6,r3,-1
88 addi r4,r4,-1
89 1: lbzu r0,1(r4)
90 cmpwi 0,r0,0
91 stbu r0,1(r6)
92 bdnzf 2,1b /* dec ctr, branch if ctr != 0 && !cr0.eq */
93 bnelr /* if we didn't hit a null char, we're done */
94 mfctr r5
95 cmpwi 0,r5,0 /* any space left in destination buffer? */
96 beqlr /* we know r0 == 0 here */
97 2: stbu r0,1(r6) /* clear it out if so */
98 bdnz 2b
99 blr
101 _GLOBAL(strcat)
102 addi r5,r3,-1
103 addi r4,r4,-1
104 1: lbzu r0,1(r5)
105 cmpwi 0,r0,0
106 bne 1b
107 addi r5,r5,-1
108 1: lbzu r0,1(r4)
109 cmpwi 0,r0,0
110 stbu r0,1(r5)
111 bne 1b
112 blr
114 _GLOBAL(strcmp)
115 addi r5,r3,-1
116 addi r4,r4,-1
117 1: lbzu r3,1(r5)
118 cmpwi 1,r3,0
119 lbzu r0,1(r4)
120 subf. r3,r0,r3
121 beqlr 1
122 beq 1b
123 blr
125 _GLOBAL(strlen)
126 addi r4,r3,-1
127 1: lbzu r0,1(r4)
128 cmpwi 0,r0,0
129 bne 1b
130 subf r3,r3,r4
131 blr
133 /*
134 * Use dcbz on the complete cache lines in the destination
135 * to set them to zero. This requires that the destination
136 * area is cacheable. -- paulus
137 */
138 _GLOBAL(cacheable_memzero)
139 mr r5,r4
140 li r4,0
141 addi r6,r3,-4
142 cmplwi 0,r5,4
143 blt 7f
144 stwu r4,4(r6)
145 beqlr
146 andi. r0,r6,3
147 add r5,r0,r5
148 subf r6,r0,r6
149 clrlwi r7,r6,32-LG_CACHELINE_BYTES
150 add r8,r7,r5
151 srwi r9,r8,LG_CACHELINE_BYTES
152 addic. r9,r9,-1 /* total number of complete cachelines */
153 ble 2f
154 xori r0,r7,CACHELINE_MASK & ~3
155 srwi. r0,r0,2
156 beq 3f
157 mtctr r0
158 4: stwu r4,4(r6)
159 bdnz 4b
160 3: mtctr r9
161 li r7,4
162 #if !defined(CONFIG_8xx)
163 10: dcbz r7,r6
164 #else
165 10: stw r4, 4(r6)
166 stw r4, 8(r6)
167 stw r4, 12(r6)
168 stw r4, 16(r6)
169 #if CACHE_LINE_SIZE >= 32
170 stw r4, 20(r6)
171 stw r4, 24(r6)
172 stw r4, 28(r6)
173 stw r4, 32(r6)
174 #endif /* CACHE_LINE_SIZE */
175 #endif
176 addi r6,r6,CACHELINE_BYTES
177 bdnz 10b
178 clrlwi r5,r8,32-LG_CACHELINE_BYTES
179 addi r5,r5,4
180 2: srwi r0,r5,2
181 mtctr r0
182 bdz 6f
183 1: stwu r4,4(r6)
184 bdnz 1b
185 6: andi. r5,r5,3
186 7: cmpwi 0,r5,0
187 beqlr
188 mtctr r5
189 addi r6,r6,3
190 8: stbu r4,1(r6)
191 bdnz 8b
192 blr
194 _GLOBAL(memset)
195 rlwimi r4,r4,8,16,23
196 rlwimi r4,r4,16,0,15
197 addi r6,r3,-4
198 cmplwi 0,r5,4
199 blt 7f
200 stwu r4,4(r6)
201 beqlr
202 andi. r0,r6,3
203 add r5,r0,r5
204 subf r6,r0,r6
205 srwi r0,r5,2
206 mtctr r0
207 bdz 6f
208 1: stwu r4,4(r6)
209 bdnz 1b
210 6: andi. r5,r5,3
211 7: cmpwi 0,r5,0
212 beqlr
213 mtctr r5
214 addi r6,r6,3
215 8: stbu r4,1(r6)
216 bdnz 8b
217 blr
219 _GLOBAL(bcopy)
220 mr r6,r3
221 mr r3,r4
222 mr r4,r6
223 b memcpy
225 /*
226 * This version uses dcbz on the complete cache lines in the
227 * destination area to reduce memory traffic. This requires that
228 * the destination area is cacheable.
229 * We only use this version if the source and dest don't overlap.
230 * -- paulus.
231 */
232 _GLOBAL(cacheable_memcpy)
233 add r7,r3,r5 /* test if the src & dst overlap */
234 add r8,r4,r5
235 cmplw 0,r4,r7
236 cmplw 1,r3,r8
237 crand 0,0,4 /* cr0.lt &= cr1.lt */
238 blt memcpy /* if regions overlap */
240 addi r4,r4,-4
241 addi r6,r3,-4
242 neg r0,r3
243 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
244 beq 58f
246 cmplw 0,r5,r0 /* is this more than total to do? */
247 blt 63f /* if not much to do */
248 andi. r8,r0,3 /* get it word-aligned first */
249 subf r5,r0,r5
250 mtctr r8
251 beq+ 61f
252 70: lbz r9,4(r4) /* do some bytes */
253 stb r9,4(r6)
254 addi r4,r4,1
255 addi r6,r6,1
256 bdnz 70b
257 61: srwi. r0,r0,2
258 mtctr r0
259 beq 58f
260 72: lwzu r9,4(r4) /* do some words */
261 stwu r9,4(r6)
262 bdnz 72b
264 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
265 clrlwi r5,r5,32-LG_CACHELINE_BYTES
266 li r11,4
267 mtctr r0
268 beq 63f
269 53:
270 #if !defined(CONFIG_8xx)
271 dcbz r11,r6
272 #endif
273 COPY_16_BYTES
274 #if L1_CACHE_LINE_SIZE >= 32
275 COPY_16_BYTES
276 #if L1_CACHE_LINE_SIZE >= 64
277 COPY_16_BYTES
278 COPY_16_BYTES
279 #if L1_CACHE_LINE_SIZE >= 128
280 COPY_16_BYTES
281 COPY_16_BYTES
282 COPY_16_BYTES
283 COPY_16_BYTES
284 #endif
285 #endif
286 #endif
287 bdnz 53b
289 63: srwi. r0,r5,2
290 mtctr r0
291 beq 64f
292 30: lwzu r0,4(r4)
293 stwu r0,4(r6)
294 bdnz 30b
296 64: andi. r0,r5,3
297 mtctr r0
298 beq+ 65f
299 40: lbz r0,4(r4)
300 stb r0,4(r6)
301 addi r4,r4,1
302 addi r6,r6,1
303 bdnz 40b
304 65: blr
306 _GLOBAL(memmove)
307 cmplw 0,r3,r4
308 bgt backwards_memcpy
309 /* fall through */
311 _GLOBAL(memcpy)
312 srwi. r7,r5,3
313 addi r6,r3,-4
314 addi r4,r4,-4
315 beq 2f /* if less than 8 bytes to do */
316 andi. r0,r6,3 /* get dest word aligned */
317 mtctr r7
318 bne 5f
319 1: lwz r7,4(r4)
320 lwzu r8,8(r4)
321 stw r7,4(r6)
322 stwu r8,8(r6)
323 bdnz 1b
324 andi. r5,r5,7
325 2: cmplwi 0,r5,4
326 blt 3f
327 lwzu r0,4(r4)
328 addi r5,r5,-4
329 stwu r0,4(r6)
330 3: cmpwi 0,r5,0
331 beqlr
332 mtctr r5
333 addi r4,r4,3
334 addi r6,r6,3
335 4: lbzu r0,1(r4)
336 stbu r0,1(r6)
337 bdnz 4b
338 blr
339 5: subfic r0,r0,4
340 mtctr r0
341 6: lbz r7,4(r4)
342 addi r4,r4,1
343 stb r7,4(r6)
344 addi r6,r6,1
345 bdnz 6b
346 subf r5,r0,r5
347 rlwinm. r7,r5,32-3,3,31
348 beq 2b
349 mtctr r7
350 b 1b
352 _GLOBAL(backwards_memcpy)
353 rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */
354 add r6,r3,r5
355 add r4,r4,r5
356 beq 2f
357 andi. r0,r6,3
358 mtctr r7
359 bne 5f
360 1: lwz r7,-4(r4)
361 lwzu r8,-8(r4)
362 stw r7,-4(r6)
363 stwu r8,-8(r6)
364 bdnz 1b
365 andi. r5,r5,7
366 2: cmplwi 0,r5,4
367 blt 3f
368 lwzu r0,-4(r4)
369 subi r5,r5,4
370 stwu r0,-4(r6)
371 3: cmpwi 0,r5,0
372 beqlr
373 mtctr r5
374 4: lbzu r0,-1(r4)
375 stbu r0,-1(r6)
376 bdnz 4b
377 blr
378 5: mtctr r0
379 6: lbzu r7,-1(r4)
380 stbu r7,-1(r6)
381 bdnz 6b
382 subf r5,r0,r5
383 rlwinm. r7,r5,32-3,3,31
384 beq 2b
385 mtctr r7
386 b 1b
388 _GLOBAL(memcmp)
389 cmpwi 0,r5,0
390 ble- 2f
391 mtctr r5
392 addi r6,r3,-1
393 addi r4,r4,-1
394 1: lbzu r3,1(r6)
395 lbzu r0,1(r4)
396 subf. r3,r0,r3
397 bdnzt 2,1b
398 blr
399 2: li r3,0
400 blr
402 _GLOBAL(memchr)
403 cmpwi 0,r5,0
404 ble- 2f
405 mtctr r5
406 addi r3,r3,-1
407 1: lbzu r0,1(r3)
408 cmpw 0,r0,r4
409 bdnzf 2,1b
410 beqlr
411 2: li r3,0
412 blr
414 _GLOBAL(__copy_tofrom_user)
415 addi r4,r4,-4
416 addi r6,r3,-4
417 neg r0,r3
418 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
419 beq 58f
421 cmplw 0,r5,r0 /* is this more than total to do? */
422 blt 63f /* if not much to do */
423 andi. r8,r0,3 /* get it word-aligned first */
424 mtctr r8
425 beq+ 61f
426 70: lbz r9,4(r4) /* do some bytes */
427 71: stb r9,4(r6)
428 addi r4,r4,1
429 addi r6,r6,1
430 bdnz 70b
431 61: subf r5,r0,r5
432 srwi. r0,r0,2
433 mtctr r0
434 beq 58f
435 72: lwzu r9,4(r4) /* do some words */
436 73: stwu r9,4(r6)
437 bdnz 72b
439 .section __ex_table,"a"
440 .align 2
441 .long 70b,100f
442 .long 71b,101f
443 .long 72b,102f
444 .long 73b,103f
445 .text
447 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
448 clrlwi r5,r5,32-LG_CACHELINE_BYTES
449 li r11,4
450 beq 63f
452 #ifdef CONFIG_8xx
453 /* Don't use prefetch on 8xx */
454 mtctr r0
455 53: COPY_16_BYTES_WITHEX(0)
456 bdnz 53b
458 #else /* not CONFIG_8xx */
459 /* Here we decide how far ahead to prefetch the source */
460 li r3,4
461 cmpwi r0,1
462 li r7,0
463 ble 114f
464 li r7,1
465 #if MAX_COPY_PREFETCH > 1
466 /* Heuristically, for large transfers we prefetch
467 MAX_COPY_PREFETCH cachelines ahead. For small transfers
468 we prefetch 1 cacheline ahead. */
469 cmpwi r0,MAX_COPY_PREFETCH
470 ble 112f
471 li r7,MAX_COPY_PREFETCH
472 112: mtctr r7
473 111: dcbt r3,r4
474 addi r3,r3,CACHELINE_BYTES
475 bdnz 111b
476 #else
477 dcbt r3,r4
478 addi r3,r3,CACHELINE_BYTES
479 #endif /* MAX_COPY_PREFETCH > 1 */
481 114: subf r8,r7,r0
482 mr r0,r7
483 mtctr r8
485 53: dcbt r3,r4
486 54: dcbz r11,r6
487 .section __ex_table,"a"
488 .align 2
489 .long 54b,105f
490 .text
491 /* the main body of the cacheline loop */
492 COPY_16_BYTES_WITHEX(0)
493 #if L1_CACHE_LINE_SIZE >= 32
494 COPY_16_BYTES_WITHEX(1)
495 #if L1_CACHE_LINE_SIZE >= 64
496 COPY_16_BYTES_WITHEX(2)
497 COPY_16_BYTES_WITHEX(3)
498 #if L1_CACHE_LINE_SIZE >= 128
499 COPY_16_BYTES_WITHEX(4)
500 COPY_16_BYTES_WITHEX(5)
501 COPY_16_BYTES_WITHEX(6)
502 COPY_16_BYTES_WITHEX(7)
503 #endif
504 #endif
505 #endif
506 bdnz 53b
507 cmpwi r0,0
508 li r3,4
509 li r7,0
510 bne 114b
511 #endif /* CONFIG_8xx */
513 63: srwi. r0,r5,2
514 mtctr r0
515 beq 64f
516 30: lwzu r0,4(r4)
517 31: stwu r0,4(r6)
518 bdnz 30b
520 64: andi. r0,r5,3
521 mtctr r0
522 beq+ 65f
523 40: lbz r0,4(r4)
524 41: stb r0,4(r6)
525 addi r4,r4,1
526 addi r6,r6,1
527 bdnz 40b
528 65: li r3,0
529 blr
531 /* read fault, initial single-byte copy */
532 100: li r9,0
533 b 90f
534 /* write fault, initial single-byte copy */
535 101: li r9,1
536 90: subf r5,r8,r5
537 li r3,0
538 b 99f
539 /* read fault, initial word copy */
540 102: li r9,0
541 b 91f
542 /* write fault, initial word copy */
543 103: li r9,1
544 91: li r3,2
545 b 99f
547 /*
548 * this stuff handles faults in the cacheline loop and branches to either
549 * 104f (if in read part) or 105f (if in write part), after updating r5
550 */
551 COPY_16_BYTES_EXCODE(0)
552 #if L1_CACHE_LINE_SIZE >= 32
553 COPY_16_BYTES_EXCODE(1)
554 #if L1_CACHE_LINE_SIZE >= 64
555 COPY_16_BYTES_EXCODE(2)
556 COPY_16_BYTES_EXCODE(3)
557 #if L1_CACHE_LINE_SIZE >= 128
558 COPY_16_BYTES_EXCODE(4)
559 COPY_16_BYTES_EXCODE(5)
560 COPY_16_BYTES_EXCODE(6)
561 COPY_16_BYTES_EXCODE(7)
562 #endif
563 #endif
564 #endif
566 /* read fault in cacheline loop */
567 104: li r9,0
568 b 92f
569 /* fault on dcbz (effectively a write fault) */
570 /* or write fault in cacheline loop */
571 105: li r9,1
572 92: li r3,LG_CACHELINE_BYTES
573 b 99f
574 /* read fault in final word loop */
575 108: li r9,0
576 b 93f
577 /* write fault in final word loop */
578 109: li r9,1
579 93: andi. r5,r5,3
580 li r3,2
581 b 99f
582 /* read fault in final byte loop */
583 110: li r9,0
584 b 94f
585 /* write fault in final byte loop */
586 111: li r9,1
587 94: li r5,0
588 li r3,0
589 /*
590 * At this stage the number of bytes not copied is
591 * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
592 */
593 99: mfctr r0
594 slw r3,r0,r3
595 add. r3,r3,r5
596 beq 120f /* shouldn't happen */
597 cmpwi 0,r9,0
598 bne 120f
599 /* for a read fault, first try to continue the copy one byte at a time */
600 mtctr r3
601 130: lbz r0,4(r4)
602 131: stb r0,4(r6)
603 addi r4,r4,1
604 addi r6,r6,1
605 bdnz 130b
606 /* then clear out the destination: r3 bytes starting at 4(r6) */
607 132: mfctr r3
608 srwi. r0,r3,2
609 li r9,0
610 mtctr r0
611 beq 113f
612 112: stwu r9,4(r6)
613 bdnz 112b
614 113: andi. r0,r3,3
615 mtctr r0
616 beq 120f
617 114: stb r9,4(r6)
618 addi r6,r6,1
619 bdnz 114b
620 120: blr
622 .section __ex_table,"a"
623 .align 2
624 .long 30b,108b
625 .long 31b,109b
626 .long 40b,110b
627 .long 41b,111b
628 .long 130b,132b
629 .long 131b,120b
630 .long 112b,120b
631 .long 114b,120b
632 .text
634 _GLOBAL(__clear_user)
635 addi r6,r3,-4
636 li r3,0
637 li r5,0
638 cmplwi 0,r4,4
639 blt 7f
640 /* clear a single word */
641 11: stwu r5,4(r6)
642 beqlr
643 /* clear word sized chunks */
644 andi. r0,r6,3
645 add r4,r0,r4
646 subf r6,r0,r6
647 srwi r0,r4,2
648 andi. r4,r4,3
649 mtctr r0
650 bdz 7f
651 1: stwu r5,4(r6)
652 bdnz 1b
653 /* clear byte sized chunks */
654 7: cmpwi 0,r4,0
655 beqlr
656 mtctr r4
657 addi r6,r6,3
658 8: stbu r5,1(r6)
659 bdnz 8b
660 blr
661 90: mr r3,r4
662 blr
663 91: mfctr r3
664 slwi r3,r3,2
665 add r3,r3,r4
666 blr
667 92: mfctr r3
668 blr
670 .section __ex_table,"a"
671 .align 2
672 .long 11b,90b
673 .long 1b,91b
674 .long 8b,92b
675 .text
677 _GLOBAL(__strncpy_from_user)
678 addi r6,r3,-1
679 addi r4,r4,-1
680 cmpwi 0,r5,0
681 beq 2f
682 mtctr r5
683 1: lbzu r0,1(r4)
684 cmpwi 0,r0,0
685 stbu r0,1(r6)
686 bdnzf 2,1b /* dec ctr, branch if ctr != 0 && !cr0.eq */
687 beq 3f
688 2: addi r6,r6,1
689 3: subf r3,r3,r6
690 blr
691 99: li r3,-EFAULT
692 blr
694 .section __ex_table,"a"
695 .align 2
696 .long 1b,99b
697 .text
699 /* r3 = str, r4 = len (> 0), r5 = top (highest addr) */
700 _GLOBAL(__strnlen_user)
701 addi r7,r3,-1
702 subf r6,r7,r5 /* top+1 - str */
703 cmplw 0,r4,r6
704 bge 0f
705 mr r6,r4
706 0: mtctr r6 /* ctr = min(len, top - str) */
707 1: lbzu r0,1(r7) /* get next byte */
708 cmpwi 0,r0,0
709 bdnzf 2,1b /* loop if --ctr != 0 && byte != 0 */
710 addi r7,r7,1
711 subf r3,r3,r7 /* number of bytes we have looked at */
712 beqlr /* return if we found a 0 byte */
713 cmpw 0,r3,r4 /* did we look at all len bytes? */
714 blt 99f /* if not, must have hit top */
715 addi r3,r4,1 /* return len + 1 to indicate no null found */
716 blr
717 99: li r3,0 /* bad address, return 0 */
718 blr
720 .section __ex_table,"a"
721 .align 2
722 .long 1b,99b