| blob | a6c9abf7d94a588b3596166e02a3e4008b606a05 |
1 /* Copyright (C) 2014-2015 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library; if not, see
16 <http://www.gnu.org/licenses/>. */
18 #include <sysdep.h>
20 /* Implements the functions
22 char * [r3] strncpy (char *dst [r3], const char *src [r4], size_t n [r5])
24 AND
26 char * [r3] stpncpy (char *dst [r3], const char *src [r4], size_t n [r5])
28 The algorithm is as follows:
29 > if src and dest are 8 byte aligned, perform double word copy
30 else
31 > copy byte by byte on unaligned addresses.
33 The aligned comparison are made using cmpb instructions. */
35 /* The focus on optimization for performance improvements are as follows:
36 1. data alignment [gain from aligned memory access on read/write]
37 2. POWER7 gains performance with loop unrolling/unwinding
38 [gain by reduction of branch penalty].
39 3. The final pad with null bytes is done by calling an optimized
40 memset. */
42 #ifdef USE_AS_STPNCPY
43 # define FUNC_NAME __stpncpy
44 #else
45 # define FUNC_NAME strncpy
46 #endif
48 #define FRAMESIZE (FRAME_MIN_SIZE+32)
50 #ifndef MEMSET
51 /* For builds with no IFUNC support, local calls should be made to internal
52 GLIBC symbol (created by libc_hidden_builtin_def). */
53 # ifdef SHARED
54 # define MEMSET __GI_memset
55 # else
56 # define MEMSET memset
57 # endif
58 #endif
60 .machine power7
61 EALIGN(FUNC_NAME, 4, 0)
62 CALL_MCOUNT 3
64 mflr r0 /* load link register LR to r0 */
65 or r10, r3, r4 /* to verify source and destination */
66 rldicl. r8, r10, 0, 61 /* is double word aligned .. ? */
68 std r19, -8(r1) /* save callers register , r19 */
69 std r18, -16(r1) /* save callers register , r18 */
70 std r0, 16(r1) /* store the link register */
71 stdu r1, -FRAMESIZE(r1) /* create the stack frame */
73 mr r9, r3 /* save r3 into r9 for use */
74 mr r18, r3 /* save r3 for retCode of strncpy */
75 bne 0, L(unaligned)
77 L(aligned):
78 srdi r11, r5, 3 /* compute count for CTR ; count = n/8 */
79 cmpldi cr7, r11, 3 /* if count > 4 ; perform unrolling 4 times */
80 ble 7, L(update1)
82 ld r10, 0(r4) /* load doubleWord from src */
83 cmpb r8, r10, r8 /* compare src with NULL ,we read just now */
84 cmpdi cr7, r8, 0 /* if cmpb returned NULL ; we continue */
85 bne cr7, L(update3)
87 std r10, 0(r3) /* copy doubleword at offset=0 */
88 ld r10, 8(r4) /* load next doubleword from offset=8 */
89 cmpb r8, r10, r8 /* compare src with NULL , we read just now */
90 cmpdi cr7, r8, 0 /* if cmpb returned NULL ; we continue */
91 bne 7,L(HopBy8)
93 addi r8, r11, -4
94 mr r7, r3
95 srdi r8, r8, 2
96 mr r6, r4
97 addi r8, r8, 1
98 li r12, 0
99 mtctr r8
100 b L(dwordCopy)
102 .p2align 4
103 L(dWordUnroll):
104 std r8, 16(r9)
105 ld r8, 24(r4) /* load dword,perform loop unrolling again */
106 cmpb r10, r8, r10
107 cmpdi cr7, r10, 0
108 bne cr7, L(HopBy24)
110 std r8, 24(r7) /* copy dword at offset=24 */
111 addi r9, r9, 32
112 addi r4, r4, 32
113 bdz L(leftDwords) /* continue with loop on counter */
115 ld r3, 32(r6)
116 cmpb r8, r3, r10
117 cmpdi cr7, r8, 0
118 bne cr7, L(update2)
120 std r3, 32(r7)
121 ld r10, 40(r6)
122 cmpb r8, r10, r8
123 cmpdi cr7, r8, 0
124 bne cr7, L(HopBy40)
126 mr r6, r4 /* update values */
127 mr r7, r9
128 mr r11, r0
129 mr r5, r19
131 L(dwordCopy):
132 std r10, 8(r9) /* copy dword at offset=8 */
133 addi r19, r5, -32
134 addi r0, r11, -4
135 ld r8, 16(r4)
136 cmpb r10, r8, r12
137 cmpdi cr7, r10, 0
138 beq cr7, L(dWordUnroll)
140 addi r9, r9, 16 /* increment dst by 16 */
141 addi r4, r4, 16 /* increment src by 16 */
142 addi r5, r5, -16 /* decrement length 'n' by 16 */
143 addi r0, r11, -2 /* decrement loop counter */
145 L(dWordUnrollOFF):
146 ld r10, 0(r4) /* load first dword */
147 li r8, 0 /* load mask */
148 cmpb r8, r10, r8
149 cmpdi cr7, r8, 0
150 bne cr7, L(byte_by_byte)
151 mtctr r0
152 li r7, 0
153 b L(CopyDword)
155 .p2align 4
156 L(loadDWordandCompare):
157 ld r10, 0(r4)
158 cmpb r8, r10, r7
159 cmpdi cr7, r8, 0
160 bne cr7, L(byte_by_byte)
162 L(CopyDword):
163 addi r9, r9, 8
164 std r10, -8(r9)
165 addi r4, r4, 8
166 addi r5, r5, -8
167 bdnz L(loadDWordandCompare)
169 L(byte_by_byte):
170 cmpldi cr7, r5, 3
171 ble cr7, L(verifyByte)
172 srdi r10, r5, 2
173 mr r19, r9
174 mtctr r10
175 b L(firstByteUnroll)
177 .p2align 4
178 L(bytes_unroll):
179 lbz r10, 1(r4) /* load byte from src */
180 cmpdi cr7, r10, 0 /* compare for NULL */
181 stb r10, 1(r19) /* store byte to dst */
182 beq cr7, L(updtDestComputeN2ndByte)
184 addi r4, r4, 4 /* advance src */
186 lbz r10, -2(r4) /* perform loop unrolling for byte r/w */
187 cmpdi cr7, r10, 0
188 stb r10, 2(r19)
189 beq cr7, L(updtDestComputeN3rdByte)
191 lbz r10, -1(r4) /* perform loop unrolling for byte r/w */
192 addi r19, r19, 4
193 cmpdi cr7, r10, 0
194 stb r10, -1(r19)
195 beq cr7, L(ComputeNByte)
197 bdz L(update0)
199 L(firstByteUnroll):
200 lbz r10, 0(r4) /* perform loop unrolling for byte r/w */
201 cmpdi cr7, 10, 0
202 stb r10, 0(r19)
203 bne cr7, L(bytes_unroll)
204 addi r19, r19, 1
206 L(ComputeNByte):
207 subf r9, r19, r9 /* compute 'n'n bytes to fill */
208 add r8, r9, r5
210 L(zeroFill):
211 cmpdi cr7, r8, 0 /* compare if length is zero */
212 beq cr7, L(update3return)
214 mr r3, r19 /* fill buffer with */
215 li r4, 0 /* zero fill buffer */
216 mr r5, r8 /* how many bytes to fill buffer with */
217 bl MEMSET /* call optimized memset */
218 nop
220 L(update3return):
221 #ifdef USE_AS_STPNCPY
222 addi r3, r19, -1 /* update return value */
223 #endif
225 L(hop2return):
226 #ifndef USE_AS_STPNCPY
227 mr r3, r18 /* set return value */
228 #endif
229 addi r1, r1, FRAMESIZE /* restore stack pointer */
230 ld r0, 16(r1) /* read the saved link register */
231 ld r18, -16(r1) /* restore callers save register, r18 */
232 ld r19, -8(r1) /* restore callers save register, r19 */
233 mtlr r0 /* branch to link register */
234 blr /* return */
236 .p2align 4
237 L(update0):
238 mr r9, r19
240 .p2align 4
241 L(verifyByte):
242 rldicl. r8, r5, 0, 62
243 #ifdef USE_AS_STPNCPY
244 mr r3, r9
245 #endif
246 beq cr0, L(hop2return)
247 mtctr r8
248 addi r4, r4, -1
249 mr r19, r9
250 b L(oneBYone)
252 .p2align 4
253 L(proceed):
254 bdz L(done)
256 L(oneBYone):
257 lbzu r10, 1(r4) /* copy byte */
258 addi r19, r19, 1
259 addi r8, r8, -1
260 cmpdi cr7, r10, 0
261 stb r10, -1(r19)
262 bne cr7, L(proceed)
263 b L(zeroFill)
265 .p2align 4
266 L(done):
267 addi r1, r1, FRAMESIZE /* restore stack pointer */
268 #ifdef USE_AS_STPNCPY
269 mr r3, r19 /* set the return value */
270 #else
271 mr r3, r18 /* set the return value */
272 #endif
273 ld r0, 16(r1) /* read the saved link register */
274 ld r18, -16(r1) /* restore callers save register, r18 */
275 ld r19, -8(r1) /* restore callers save register, r19 */
276 mtlr r0 /* branch to link register */
277 blr /* return */
279 L(update1):
280 mr r0, r11
281 mr r19, r5
283 .p2align 4
284 L(leftDwords):
285 cmpdi cr7, r0, 0
286 mr r5, r19
287 bne cr7, L(dWordUnrollOFF)
288 b L(byte_by_byte)
290 .p2align 4
291 L(updtDestComputeN2ndByte):
292 addi r19, r19, 2 /* update dst by 2 */
293 subf r9, r19, r9 /* compute distance covered */
294 add r8, r9, r5
295 b L(zeroFill)
297 .p2align 4
298 L(updtDestComputeN3rdByte):
299 addi r19, r19, 3 /* update dst by 3 */
300 subf r9, r19, r9 /* compute distance covered */
301 add r8, r9, r5
302 b L(zeroFill)
304 .p2align 4
305 L(HopBy24):
306 addi r9, r9, 24 /* increment dst by 24 */
307 addi r4, r4, 24 /* increment src by 24 */
308 addi r5, r5, -24 /* decrement length 'n' by 24 */
309 addi r0, r11, -3 /* decrement loop counter */
310 b L(dWordUnrollOFF)
312 .p2align 4
313 L(update2):
314 mr r5, r19
315 b L(dWordUnrollOFF)
317 .p2align 4
318 L(HopBy40):
319 addi r9, r7, 40 /* increment dst by 40 */
320 addi r4, r6, 40 /* increment src by 40 */
321 addi r5, r5, -40 /* decrement length 'n' by 40 */
322 addi r0, r11, -5 /* decrement loop counter */
323 b L(dWordUnrollOFF)
325 L(update3):
326 mr r0, r11
327 b L(dWordUnrollOFF)
329 L(HopBy8):
330 addi r9, r3, 8 /* increment dst by 8 */
331 addi r4, r4, 8 /* increment src by 8 */
332 addi r5, r5, -8 /* decrement length 'n' by 8 */
333 addi r0, r11, -1 /* decrement loop counter */
334 b L(dWordUnrollOFF)
336 L(unaligned):
337 cmpdi r5, 16 /* Proceed byte by byte for less than 16 */
338 ble L(byte_by_byte)
339 rldicl r7, r3, 0, 61
340 rldicl r6, r4, 0, 61
341 cmpdi r6, 0 /* Check src alignment */
342 beq L(srcaligndstunalign)
343 /* src is unaligned */
344 rlwinm r10, r4, 3,26,28 /* Calculate padding. */
345 clrrdi r4, r4, 3 /* Align the addr to dw boundary */
346 ld r8, 0(r4) /* Load doubleword from memory. */
347 li r0, 0
348 /* Discard bits not part of the string */
349 #ifdef __LITTLE_ENDIAN__
350 srd r7, r8, r10
351 #else
352 sld r7, r8, r10
353 #endif
354 cmpb r0, r7, r0 /* Compare each byte against null */
355 /* Discard bits not part of the string */
356 #ifdef __LITTLE_ENDIAN__
357 sld r0, r0, r10
358 #else
359 srd r0, r0, r10
360 #endif
361 cmpdi r0, 0
362 bne L(bytebybyte) /* if it has null, copy byte by byte */
363 subfic r6, r6, 8
364 rlwinm r12, r3, 3,26,28 /* Calculate padding in bits. */
365 rldicl r9, r3, 0, 61 /* Calculate padding in bytes. */
366 addi r3, r3, -1
368 cmpdi r12, 0 /* check dest alignment */
369 beq L(srcunaligndstalign)
371 /* both src and dst unaligned */
372 #ifdef __LITTLE_ENDIAN__
373 sld r8, r7, r10
374 mr r11, r10
375 addi r11, r11, -8 /* Adjust byte pointer on loaded dw */
376 #else
377 srd r8, r7, r10
378 subfic r11, r10, 64
379 #endif
380 /* dst alignment is greater then src alignment? */
381 cmpd cr7, r12, r10
382 ble cr7, L(dst_align_small)
383 /* src alignment is less than dst */
385 /* Calculate the dst alignment difference */
386 subfic r7, r9, 8
387 mtctr r7
389 /* Write until dst is aligned */
390 cmpdi r0, r7, 4
391 blt L(storebyte1) /* less than 4, store byte by byte */
392 beq L(equal1) /* if its 4, store word */
393 addi r0, r7, -4 /* greater than 4, so stb and stw */
394 mtctr r0
395 L(storebyte1):
396 #ifdef __LITTLE_ENDIAN__
397 addi r11, r11, 8 /* Adjust byte pointer on loaded dw */
398 #else
399 addi r11, r11, -8
400 #endif
401 srd r7, r8, r11
402 stbu r7, 1(r3)
403 addi r5, r5, -1
404 bdnz L(storebyte1)
406 subfic r7, r9, 8 /* Check the remaining bytes */
407 cmpdi r0, r7, 4
408 blt L(proceed1)
410 .align 4
411 L(equal1):
412 #ifdef __LITTLE_ENDIAN__
413 addi r11, r11, 8 /* Adjust byte pointer on loaded dw */
414 srd r7, r8, r11
415 #else
416 subfic r11, r11, 64
417 sld r7, r8, r11
418 srdi r7, r7, 32
419 #endif
420 stw r7, 1(r3)
421 addi r3, r3, 4
422 addi r5, r5, -4
424 L(proceed1):
425 mr r7, r8
426 /* calculate the Left over bytes to be written */
427 subfic r11, r10, 64
428 subfic r12, r12, 64
429 subf r12, r12, r11 /* remaining bytes on second dw */
430 subfic r10, r12, 64 /* remaining bytes on first dw */
431 subfic r9, r9, 8
432 subf r6, r9, r6 /* recalculate padding */
433 L(srcunaligndstalign):
434 addi r3, r3, 1
435 subfic r12, r10, 64 /* remaining bytes on second dw */
436 addi r4, r4, 8
437 li r0,0
438 b L(storedouble)
440 .align 4
441 L(dst_align_small):
442 mtctr r6
443 /* Write until src is aligned */
444 L(storebyte2):
445 #ifdef __LITTLE_ENDIAN__
446 addi r11, r11, 8 /* Adjust byte pointer on dw */
447 #else
448 addi r11, r11, -8
449 #endif
450 srd r7, r8, r11
451 stbu r7, 1(r3)
452 addi r5, r5, -1
453 bdnz L(storebyte2)
455 addi r4, r4, 8 /* Increment src pointer */
456 addi r3, r3, 1 /* Increment dst pointer */
457 mr r9, r3
458 li r8, 0
459 cmpd cr7, r12, r10
460 beq cr7, L(aligned)
461 rldicl r6, r3, 0, 61 /* Recalculate padding */
462 mr r7, r6
464 /* src is algined */
465 L(srcaligndstunalign):
466 mr r9, r3
467 mr r6, r7
468 ld r8, 0(r4)
469 subfic r10, r7, 8
470 mr r7, r8
471 li r0, 0 /* Check null */
472 cmpb r0, r8, r0
473 cmpdi r0, 0
474 bne L(byte_by_byte) /* Do byte by byte if there is NULL */
475 rlwinm r12, r3, 3,26,28 /* Calculate padding */
476 addi r3, r3, -1
477 /* write byte by byte until aligned */
478 #ifdef __LITTLE_ENDIAN__
479 li r11, -8
480 #else
481 li r11, 64
482 #endif
483 mtctr r10
484 cmpdi r0, r10, 4
485 blt L(storebyte)
486 beq L(equal)
487 addi r0, r10, -4
488 mtctr r0
489 L(storebyte):
490 #ifdef __LITTLE_ENDIAN__
491 addi r11, r11, 8 /* Adjust byte pointer on dw */
492 #else
493 addi r11, r11, -8
494 #endif
495 srd r7, r8, r11
496 stbu r7, 1(r3)
497 addi r5, r5, -1
498 bdnz L(storebyte)
500 cmpdi r0, r10, 4
501 blt L(align)
503 .align 4
504 L(equal):
505 #ifdef __LITTLE_ENDIAN__
506 addi r11, r11, 8
507 srd r7, r8, r11
508 #else
509 subfic r11, r11, 64
510 sld r7, r8, r11
511 srdi r7, r7, 32
512 #endif
513 stw r7, 1(r3)
514 addi r5, r5, -4
515 addi r3, r3, 4
516 L(align):
517 addi r3, r3, 1
518 addi r4, r4, 8 /* Increment src pointer */
519 subfic r10, r12, 64
520 li r0, 0
521 /* dst addr aligned to 8 */
522 L(storedouble):
523 cmpdi r5, 8
524 ble L(null1)
525 ld r7, 0(r4) /* load next dw */
526 cmpb r0, r7, r0
527 cmpdi r0, 0 /* check for null on each new dw */
528 bne L(null)
529 #ifdef __LITTLE_ENDIAN__
530 srd r9, r8, r10 /* bytes from first dw */
531 sld r11, r7, r12 /* bytes from second dw */
532 #else
533 sld r9, r8, r10
534 srd r11, r7, r12
535 #endif
536 or r11, r9, r11 /* make as a single dw */
537 std r11, 0(r3) /* store as std on aligned addr */
538 mr r8, r7 /* still few bytes left to be written */
539 addi r3, r3, 8 /* increment dst addr */
540 addi r4, r4, 8 /* increment src addr */
541 addi r5, r5, -8
542 b L(storedouble) /* Loop until NULL */
544 .align 4
546 /* We've hit the end of the string. Do the rest byte-by-byte. */
547 L(null):
548 addi r3, r3, -1
549 mr r10, r12
550 mtctr r6
551 #ifdef __LITTLE_ENDIAN__
552 subfic r10, r10, 64
553 addi r10, r10, -8
554 #endif
555 cmpdi r0, r5, 4
556 blt L(loop)
557 cmpdi r0, r6, 4
558 blt L(loop)
560 /* we can still use stw if leftover >= 4 */
561 #ifdef __LITTLE_ENDIAN__
562 addi r10, r10, 8
563 srd r11, r8, r10
564 #else
565 subfic r10, r10, 64
566 sld r11, r8, r10
567 srdi r11, r11, 32
568 #endif
569 stw r11, 1(r3)
570 addi r5, r5, -4
571 addi r3, r3, 4
572 cmpdi r0, r5, 0
573 beq L(g1)
574 cmpdi r0, r6, 4
575 beq L(bytebybyte1)
576 addi r10, r10, 32
577 #ifdef __LITTLE_ENDIAN__
578 addi r10, r10, -8
579 #else
580 subfic r10, r10, 64
581 #endif
582 addi r0, r6, -4
583 mtctr r0
584 /* remaining byte by byte part of first dw */
585 L(loop):
586 #ifdef __LITTLE_ENDIAN__
587 addi r10, r10, 8
588 #else
589 addi r10, r10, -8
590 #endif
591 srd r0, r8, r10
592 stbu r0, 1(r3)
593 addi r5, r5, -1
594 cmpdi r0, r5, 0
595 beq L(g1)
596 bdnz L(loop)
597 L(bytebybyte1):
598 addi r3, r3, 1
599 /* remaining byte by byte part of second dw */
600 L(bytebybyte):
601 addi r3, r3, -8
602 addi r4, r4, -1
604 #ifdef __LITTLE_ENDIAN__
605 extrdi. r0, r7, 8, 56
606 stbu r7, 8(r3)
607 addi r5, r5, -1
608 beq L(g2)
609 cmpdi r5, 0
610 beq L(g1)
611 extrdi. r0, r7, 8, 48
612 stbu r0, 1(r3)
613 addi r5, r5, -1
614 beq L(g2)
615 cmpdi r5, 0
616 beq L(g1)
617 extrdi. r0, r7, 8, 40
618 stbu r0, 1(r3)
619 addi r5, r5, -1
620 beq L(g2)
621 cmpdi r5, 0
622 beq L(g1)
623 extrdi. r0, r7, 8, 32
624 stbu r0, 1(r3)
625 addi r5, r5, -1
626 beq L(g2)
627 cmpdi r5, 0
628 beq L(g1)
629 extrdi. r0, r7, 8, 24
630 stbu r0, 1(r3)
631 addi r5, r5, -1
632 beq L(g2)
633 cmpdi r5, 0
634 beq L(g1)
635 extrdi. r0, r7, 8, 16
636 stbu r0, 1(r3)
637 addi r5, r5, -1
638 beq L(g2)
639 cmpdi r5, 0
640 beq L(g1)
641 extrdi. r0, r7, 8, 8
642 stbu r0, 1(r3)
643 addi r5, r5, -1
644 beq L(g2)
645 cmpdi r5, 0
646 beq L(g1)
647 extrdi r0, r7, 8, 0
648 stbu r0, 1(r3)
649 addi r5, r5, -1
650 b L(g2)
651 #else
652 extrdi. r0, r7, 8, 0
653 stbu r0, 8(r3)
654 addi r5, r5, -1
655 beq L(g2)
656 cmpdi r5, 0
657 beq L(g1)
658 extrdi. r0, r7, 8, 8
659 stbu r0, 1(r3)
660 addi r5, r5, -1
661 beq L(g2)
662 cmpdi r5, 0
663 beq L(g1)
664 extrdi. r0, r7, 8, 16
665 stbu r0, 1(r3)
666 addi r5, r5, -1
667 beq L(g2)
668 cmpdi r5, 0
669 beq L(g1)
670 extrdi. r0, r7, 8, 24
671 stbu r0, 1(r3)
672 addi r5, r5, -1
673 beq L(g2)
674 cmpdi r5, 0
675 beq L(g1)
676 extrdi. r0, r7, 8, 32
677 stbu r0, 1(r3)
678 addi r5, r5, -1
679 beq L(g2)
680 cmpdi r5, 0
681 beq L(g1)
682 extrdi. r0, r7, 8, 40
683 stbu r0, 1(r3)
684 addi r5, r5, -1
685 beq L(g2)
686 cmpdi r5, 0
687 beq L(g1)
688 extrdi. r0, r7, 8, 48
689 stbu r0, 1(r3)
690 addi r5, r5, -1
691 beq L(g2)
692 cmpdi r5, 0
693 beq L(g1)
694 stbu r7, 1(r3)
695 addi r5, r5, -1
696 b L(g2)
697 #endif
698 L(g1):
699 #ifdef USE_AS_STPNCPY
700 addi r3, r3, 1
701 #endif
702 L(g2):
703 addi r3, r3, 1
704 mr r19, r3
705 mr r8, r5
706 b L(zeroFill)
707 L(null1):
708 mr r9, r3
709 subf r4, r6, r4
710 b L(byte_by_byte)
711 END(FUNC_NAME)
712 #ifndef USE_AS_STPNCPY
713 libc_hidden_builtin_def (strncpy)
714 #endif