| blob | 98798e5d6b3e728848386f2e1d270370b2cda3f1 |
1 /* Optimized memcpy implementation for PowerPC64.
2 Copyright (C) 2003, 2006 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA
18 02110-1301 USA. */
20 #include <sysdep.h>
21 #include <bp-sym.h>
22 #include <bp-asm.h>
24 /* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]);
25 Returns 'dst'.
27 Memcpy handles short copies (< 32-bytes) using a binary move blocks
28 (no loops) of lwz/stw. The tail (remaining 1-3) bytes is handled
29 with the appropriate combination of byte and halfword load/stores.
30 There is minimal effort to optimize the alignment of short moves.
31 The 64-bit implementations of POWER3 and POWER4 do a reasonable job
32 of handling unligned load/stores that do not cross 32-byte boundries.
34 Longer moves (>= 32-bytes) justify the effort to get at least the
35 destination doubleword (8-byte) aligned. Further optimization is
36 posible when both source and destination are doubleword aligned.
37 Each case has a optimized unrolled loop. */
39 EALIGN (BP_SYM (memcpy), 5, 0)
40 CALL_MCOUNT 3
42 cmpldi cr1,5,31
43 neg 0,3
44 std 3,-16(1)
45 std 31,-8(1)
46 cfi_offset(31,-8)
47 andi. 11,3,7 /* check alignement of dst. */
48 clrldi 0,0,61 /* Number of bytes until the 1st doubleword of dst. */
49 clrldi 10,4,61 /* check alignement of src. */
50 cmpldi cr6,5,8
51 ble- cr1,.L2 /* If move < 32 bytes use short move code. */
52 cmpld cr6,10,11
53 mr 12,4
54 srdi 9,5,3 /* Number of full double words remaining. */
55 mtcrf 0x01,0
56 mr 31,5
57 beq .L0
59 subf 31,0,5
60 /* Move 0-7 bytes as needed to get the destination doubleword alligned. */
61 1: bf 31,2f
62 lbz 6,0(12)
63 addi 12,12,1
64 stb 6,0(3)
65 addi 3,3,1
66 2: bf 30,4f
67 lhz 6,0(12)
68 addi 12,12,2
69 sth 6,0(3)
70 addi 3,3,2
71 4: bf 29,0f
72 lwz 6,0(12)
73 addi 12,12,4
74 stw 6,0(3)
75 addi 3,3,4
76 0:
77 clrldi 10,12,61 /* check alignement of src again. */
78 srdi 9,31,3 /* Number of full double words remaining. */
80 /* Copy doublewords from source to destination, assumpting the
81 destination is aligned on a doubleword boundary.
83 At this point we know there are at least 25 bytes left (32-7) to copy.
84 The next step is to determine if the source is also doubleword aligned.
85 If not branch to the unaligned move code at .L6. which uses
86 a load, shift, store strategy.
88 Otherwise source and destination are doubleword aligned, and we can
89 the optimized doubleword copy loop. */
90 .L0:
91 clrldi 11,31,61
92 mtcrf 0x01,9
93 cmpldi cr1,11,0
94 bne- cr6,.L6 /* If source is not DW aligned. */
96 /* Move doublewords where destination and source are DW aligned.
97 Use a unrolled loop to copy 4 doubleword (32-bytes) per iteration.
98 If the the copy is not an exact multiple of 32 bytes, 1-3
99 doublewords are copied as needed to set up the main loop. After
100 the main loop exits there may be a tail of 1-7 bytes. These byte
101 are copied a word/halfword/byte at a time as needed to preserve
102 alignment.
104 For POWER6 the L1 is store-through and the L2 is store-in. The
105 L2 is clocked at half CPU clock so we can store 16 bytes every
106 other cycle. POWER6 also has a load/store bypass so we can do
107 load, load, store, store every 2 cycles.
109 For POWER6 unaligned loads will take a 20+ cycle hicup for any
110 L1 cache miss that crosses a 32- or 128-byte boundary. Store
111 is more forgiving and does not take a hicup until page or
112 segment boundaries. So we require doubleword alignment for
113 the source but may take a risk and only require word alignment
114 for the destination. */
116 srdi 8,31,5
117 cmpldi cr1,9,4
118 cmpldi cr6,11,0
119 mr 11,12
121 bf 30,1f
122 ld 6,0(12)
123 ld 7,8(12)
124 addi 11,12,16
125 mtctr 8
126 std 6,0(3)
127 std 7,8(3)
128 addi 10,3,16
129 bf 31,4f
130 ld 0,16(12)
131 std 0,16(3)
132 blt cr1,3f
133 addi 11,12,24
134 addi 10,3,24
135 b 4f
136 .align 4
137 1:
138 mr 10,3
139 mtctr 8
140 bf 31,4f
141 ld 6,0(12)
142 addi 11,12,8
143 std 6,0(3)
144 addi 10,3,8
146 .align 4
147 4:
148 ld 6,0(11)
149 ld 7,8(11)
150 std 6,0(10)
151 std 7,8(10)
152 ld 8,16(11)
153 ld 0,24(11)
154 std 8,16(10)
155 std 0,24(10)
156 bdz 3f
158 ld 6,0+32(11)
159 ld 7,8+32(11)
160 std 6,0+32(10)
161 std 7,8+32(10)
162 ld 8,16+32(11)
163 ld 0,24+32(11)
164 std 8,16+32(10)
165 std 0,24+32(10)
166 bdz 3f
168 ld 6,0+64(11)
169 ld 7,8+64(11)
170 std 6,0+64(10)
171 std 7,8+64(10)
172 ld 8,16+64(11)
173 ld 0,24+64(11)
174 std 8,16+64(10)
175 std 0,24+64(10)
176 bdz 3f
178 ld 6,0+96(11)
179 ld 7,8+96(11)
180 std 6,0+96(10)
181 std 7,8+96(10)
182 ld 8,16+96(11)
183 ld 0,24+96(11)
184 addi 11,11,128
185 std 8,16+96(10)
186 std 0,24+96(10)
187 addi 10,10,128
188 bdnz 4b
189 3:
191 rldicr 0,31,0,60
192 mtcrf 0x01,31
193 beq cr6,0f
194 .L9:
195 add 3,3,0
196 add 12,12,0
198 /* At this point we have a tail of 0-7 bytes and we know that the
199 destiniation is double word aligned. */
200 4: bf 29,2f
201 lwz 6,0(12)
202 addi 12,12,4
203 stw 6,0(3)
204 addi 3,3,4
205 2: bf 30,1f
206 lhz 6,0(12)
207 addi 12,12,2
208 sth 6,0(3)
209 addi 3,3,2
210 1: bf 31,0f
211 lbz 6,0(12)
212 stb 6,0(3)
213 0:
214 /* Return original dst pointer. */
215 ld 31,-8(1)
216 ld 3,-16(1)
217 blr
219 /* Copy up to 31 bytes. This divided into two cases 0-8 bytes and 9-31
220 bytes. Each case is handled without loops, using binary (1,2,4,8)
221 tests.
223 In the short (0-8 byte) case no attempt is made to force alignment
224 of either source or destination. The hardware will handle the
225 unaligned load/stores with small delays for crossing 32- 128-byte,
226 and 4096-byte boundaries. Since these short moves are unlikely to be
227 unaligned or cross these boundaries, the overhead to force
228 alignment is not justified.
230 The longer (9-31 byte) move is more likely to cross 32- or 128-byte
231 boundaries. Since only loads are sensitive to the 32-/128-byte
232 boundaries it is more important to align the source then the
233 destination. If the source is not already word aligned, we first
234 move 1-3 bytes as needed. Since we are only word aligned we don't
235 use double word load/stores to insure that all loads are aligned.
236 While the destination and stores may still be unaligned, this
237 is only an issue for page (4096 byte boundary) crossing, which
238 should be rare for these short moves. The hardware handles this
239 case automatically with a small (~20 cycle) delay. */
240 .align 4
241 .L2:
242 mtcrf 0x01,5
243 neg 8,4
244 clrrdi 11,4,2
245 andi. 0,8,3
246 ble cr6,.LE8 /* Handle moves of 0-8 bytes. */
247 /* At least 9 bytes left. Get the source word aligned. */
248 cmpldi cr1,5,16
249 mr 10,5
250 mr 12,4
251 cmpldi cr6,0,2
252 beq L(dus_tail) /* If the source is already word aligned skip this. */
253 /* Copy 1-3 bytes to get source address word aligned. */
254 lwz 6,0(11)
255 subf 10,0,5
256 add 12,4,0
257 blt cr6,5f
258 srdi 7,6,16
259 bgt cr6,3f
260 sth 6,0(3)
261 b 7f
262 .align 4
263 3:
264 stb 7,0(3)
265 sth 6,1(3)
266 b 7f
267 .align 4
268 5:
269 stb 6,0(3)
270 7:
271 cmpldi cr1,10,16
272 add 3,3,0
273 mtcrf 0x01,10
274 .align 4
275 L(dus_tail):
276 /* At least 6 bytes left and the source is word aligned. This allows
277 some speculative loads up front. */
278 /* We need to special case the fall-through because the biggest delays
279 are due to address computation not being ready in time for the
280 AGEN. */
281 lwz 6,0(12)
282 lwz 7,4(12)
283 blt cr1,L(dus_tail8)
284 cmpldi cr0,10,24
285 L(dus_tail16): /* Move 16 bytes. */
286 stw 6,0(3)
287 stw 7,4(3)
288 lwz 6,8(12)
289 lwz 7,12(12)
290 stw 6,8(3)
291 stw 7,12(3)
292 /* Move 8 bytes more. */
293 bf 28,L(dus_tail16p8)
294 cmpldi cr1,10,28
295 lwz 6,16(12)
296 lwz 7,20(12)
297 stw 6,16(3)
298 stw 7,20(3)
299 /* Move 4 bytes more. */
300 bf 29,L(dus_tail16p4)
301 lwz 6,24(12)
302 stw 6,24(3)
303 addi 12,12,28
304 addi 3,3,28
305 bgt cr1,L(dus_tail2)
306 /* exactly 28 bytes. Return original dst pointer and exit. */
307 ld 3,-16(1)
308 blr
309 .align 4
310 L(dus_tail16p8): /* less then 8 bytes left. */
311 beq cr1,L(dus_tailX) /* exactly 16 bytes, early exit. */
312 cmpldi cr1,10,20
313 bf 29,L(dus_tail16p2)
314 /* Move 4 bytes more. */
315 lwz 6,16(12)
316 stw 6,16(3)
317 addi 12,12,20
318 addi 3,3,20
319 bgt cr1,L(dus_tail2)
320 /* exactly 20 bytes. Return original dst pointer and exit. */
321 ld 3,-16(1)
322 blr
323 .align 4
324 L(dus_tail16p4): /* less then 4 bytes left. */
325 addi 12,12,24
326 addi 3,3,24
327 bgt cr0,L(dus_tail2)
328 /* exactly 24 bytes. Return original dst pointer and exit. */
329 ld 3,-16(1)
330 blr
331 .align 4
332 L(dus_tail16p2): /* 16 bytes moved, less then 4 bytes left. */
333 addi 12,12,16
334 addi 3,3,16
335 b L(dus_tail2)
337 .align 4
338 L(dus_tail8): /* Move 8 bytes. */
339 /* r6, r7 already loaded speculatively. */
340 cmpldi cr1,10,8
341 cmpldi cr0,10,12
342 bf 28,L(dus_tail4)
343 stw 6,0(3)
344 stw 7,4(3)
345 /* Move 4 bytes more. */
346 bf 29,L(dus_tail8p4)
347 lwz 6,8(12)
348 stw 6,8(3)
349 addi 12,12,12
350 addi 3,3,12
351 bgt cr0,L(dus_tail2)
352 /* exactly 12 bytes. Return original dst pointer and exit. */
353 ld 3,-16(1)
354 blr
355 .align 4
356 L(dus_tail8p4): /* less then 4 bytes left. */
357 addi 12,12,8
358 addi 3,3,8
359 bgt cr1,L(dus_tail2)
360 /* exactly 8 bytes. Return original dst pointer and exit. */
361 ld 3,-16(1)
362 blr
364 .align 4
365 L(dus_tail4): /* Move 4 bytes. */
366 /* r6 already loaded speculatively. If we are here we know there is
367 more then 4 bytes left. So there is no need to test. */
368 addi 12,12,4
369 stw 6,0(3)
370 addi 3,3,4
371 L(dus_tail2): /* Move 2-3 bytes. */
372 bf 30,L(dus_tail1)
373 lhz 6,0(12)
374 sth 6,0(3)
375 bf 31,L(dus_tailX)
376 lbz 7,2(12)
377 stb 7,2(3)
378 ld 3,-16(1)
379 blr
380 L(dus_tail1): /* Move 1 byte. */
381 bf 31,L(dus_tailX)
382 lbz 6,0(12)
383 stb 6,0(3)
384 L(dus_tailX):
385 /* Return original dst pointer. */
386 ld 3,-16(1)
387 blr
389 /* Special case to copy 0-8 bytes. */
390 .align 4
391 .LE8:
392 mr 12,4
393 bne cr6,L(dus_4)
394 /* Exactly 8 bytes. We may cross a 32-/128-byte boundry and take a ~20
395 cycle delay. This case should be rare and any attempt to avoid this
396 would take most of 20 cycles any way. */
397 ld 6,0(4)
398 std 6,0(3)
399 /* Return original dst pointer. */
400 ld 3,-16(1)
401 blr
402 .align 4
403 L(dus_4):
404 bf 29,L(dus_tail2)
405 lwz 6,0(4)
406 stw 6,0(3)
407 bf 30,L(dus_5)
408 lhz 7,4(4)
409 sth 7,4(3)
410 bf 31,L(dus_0)
411 lbz 8,6(4)
412 stb 8,6(3)
413 ld 3,-16(1)
414 blr
415 .align 4
416 L(dus_5):
417 bf 31,L(dus_0)
418 lbz 6,4(4)
419 stb 6,4(3)
420 L(dus_0):
421 /* Return original dst pointer. */
422 ld 3,-16(1)
423 blr
425 .align 4
426 .L6:
428 /* Copy doublewords where the destination is aligned but the source is
429 not. Use aligned doubleword loads from the source, shifted to realign
430 the data, to allow aligned destination stores. */
431 addi 11,9,-1 /* loop DW count is one less than total */
432 subf 5,10,12 /* Move source addr to previous full double word. */
433 cmpldi cr5, 10, 2
434 cmpldi cr0, 10, 4
435 mr 4,3
436 srdi 8,11,2 /* calculate the 32 byte loop count */
437 ld 6,0(5) /* pre load 1st full doubleword. */
438 mtcrf 0x01,11
439 cmpldi cr6,9,4
440 mtctr 8
441 ld 7,8(5) /* pre load 2nd full doubleword. */
442 bge cr0, L(du4_do)
443 blt cr5, L(du1_do)
444 beq cr5, L(du2_do)
445 b L(du3_do)
447 .align 4
448 L(du1_do):
449 bf 30,L(du1_1dw)
451 /* there are at least two DWs to copy */
452 sldi 0,6, 8
453 srdi 8,7, 64-8
454 or 0,0,8
455 ld 6,16(5)
456 std 0,0(4)
457 sldi 0,7, 8
458 srdi 8,6, 64-8
459 or 0,0,8
460 ld 7,24(5)
461 std 0,8(4)
462 addi 4,4,16
463 addi 5,5,32
464 blt cr6,L(du1_fini) /* if total DWs = 3, then bypass loop */
465 bf 31,L(du1_loop)
466 /* there is a third DW to copy */
467 sldi 0,6, 8
468 srdi 8,7, 64-8
469 or 0,0,8
470 std 0,0(4)
471 mr 6,7
472 ld 7,0(5)
473 addi 5,5,8
474 addi 4,4,8
475 beq cr6,L(du1_fini) /* if total DWs = 4, then bypass loop */
476 b L(du1_loop)
477 .align 4
478 L(du1_1dw):
479 sldi 0,6, 8
480 srdi 8,7, 64-8
481 addi 5,5,16
482 or 0,0,8
483 bf 31,L(du1_loop)
484 mr 6,7
485 ld 7,0(5)
486 addi 5,5,8
487 std 0,0(4)
488 addi 4,4,8
489 .align 4
490 /* copy 32 bytes at a time */
491 L(du1_loop):
492 sldi 0,6, 8
493 srdi 8,7, 64-8
494 or 0,0,8
495 ld 6,0(5)
496 std 0,0(4)
497 sldi 0,7, 8
498 srdi 8,6, 64-8
499 or 0,0,8
500 ld 7,8(5)
501 std 0,8(4)
502 sldi 0,6, 8
503 srdi 8,7, 64-8
504 or 0,0,8
505 ld 6,16(5)
506 std 0,16(4)
507 sldi 0,7, 8
508 srdi 8,6, 64-8
509 or 0,0,8
510 ld 7,24(5)
511 std 0,24(4)
512 addi 5,5,32
513 addi 4,4,32
514 bdnz+ L(du1_loop)
515 .align 4
516 L(du1_fini):
517 /* calculate and store the final DW */
518 sldi 0,6, 8
519 srdi 8,7, 64-8
520 or 0,0,8
521 std 0,0(4)
522 b L(du_done)
524 .align 4
525 L(du2_do):
526 bf 30,L(du2_1dw)
528 /* there are at least two DWs to copy */
529 sldi 0,6, 16
530 srdi 8,7, 64-16
531 or 0,0,8
532 ld 6,16(5)
533 std 0,0(4)
534 sldi 0,7, 16
535 srdi 8,6, 64-16
536 or 0,0,8
537 ld 7,24(5)
538 std 0,8(4)
539 addi 4,4,16
540 addi 5,5,32
541 blt cr6,L(du2_fini) /* if total DWs = 3, then bypass loop */
542 bf 31,L(du2_loop)
543 /* there is a third DW to copy */
544 sldi 0,6, 16
545 srdi 8,7, 64-16
546 or 0,0,8
547 std 0,0(4)
548 mr 6,7
549 ld 7,0(5)
550 addi 5,5,8
551 addi 4,4,8
552 beq cr6,L(du2_fini) /* if total DWs = 4, then bypass loop */
553 b L(du2_loop)
554 .align 4
555 L(du2_1dw):
556 sldi 0,6, 16
557 srdi 8,7, 64-16
558 addi 5,5,16
559 or 0,0,8
560 bf 31,L(du2_loop)
561 mr 6,7
562 ld 7,0(5)
563 addi 5,5,8
564 std 0,0(4)
565 addi 4,4,8
566 .align 4
567 /* copy 32 bytes at a time */
568 L(du2_loop):
569 sldi 0,6, 16
570 srdi 8,7, 64-16
571 or 0,0,8
572 ld 6,0(5)
573 std 0,0(4)
574 sldi 0,7, 16
575 srdi 8,6, 64-16
576 or 0,0,8
577 ld 7,8(5)
578 std 0,8(4)
579 sldi 0,6, 16
580 srdi 8,7, 64-16
581 or 0,0,8
582 ld 6,16(5)
583 std 0,16(4)
584 sldi 0,7, 16
585 srdi 8,6, 64-16
586 or 0,0,8
587 ld 7,24(5)
588 std 0,24(4)
589 addi 5,5,32
590 addi 4,4,32
591 bdnz+ L(du2_loop)
592 .align 4
593 L(du2_fini):
594 /* calculate and store the final DW */
595 sldi 0,6, 16
596 srdi 8,7, 64-16
597 or 0,0,8
598 std 0,0(4)
599 b L(du_done)
601 .align 4
602 L(du3_do):
603 bf 30,L(du3_1dw)
605 /* there are at least two DWs to copy */
606 sldi 0,6, 24
607 srdi 8,7, 64-24
608 or 0,0,8
609 ld 6,16(5)
610 std 0,0(4)
611 sldi 0,7, 24
612 srdi 8,6, 64-24
613 or 0,0,8
614 ld 7,24(5)
615 std 0,8(4)
616 addi 4,4,16
617 addi 5,5,32
618 blt cr6,L(du3_fini) /* if total DWs = 3, then bypass loop */
619 bf 31,L(du3_loop)
620 /* there is a third DW to copy */
621 sldi 0,6, 24
622 srdi 8,7, 64-24
623 or 0,0,8
624 std 0,0(4)
625 mr 6,7
626 ld 7,0(5)
627 addi 5,5,8
628 addi 4,4,8
629 beq cr6,L(du3_fini) /* if total DWs = 4, then bypass loop */
630 b L(du3_loop)
631 .align 4
632 L(du3_1dw):
633 sldi 0,6, 24
634 srdi 8,7, 64-24
635 addi 5,5,16
636 or 0,0,8
637 bf 31,L(du3_loop)
638 mr 6,7
639 ld 7,0(5)
640 addi 5,5,8
641 std 0,0(4)
642 addi 4,4,8
643 .align 4
644 /* copy 32 bytes at a time */
645 L(du3_loop):
646 sldi 0,6, 24
647 srdi 8,7, 64-24
648 or 0,0,8
649 ld 6,0(5)
650 std 0,0(4)
651 sldi 0,7, 24
652 srdi 8,6, 64-24
653 or 0,0,8
654 ld 7,8(5)
655 std 0,8(4)
656 sldi 0,6, 24
657 srdi 8,7, 64-24
658 or 0,0,8
659 ld 6,16(5)
660 std 0,16(4)
661 sldi 0,7, 24
662 srdi 8,6, 64-24
663 or 0,0,8
664 ld 7,24(5)
665 std 0,24(4)
666 addi 5,5,32
667 addi 4,4,32
668 bdnz+ L(du3_loop)
669 .align 4
670 L(du3_fini):
671 /* calculate and store the final DW */
672 sldi 0,6, 24
673 srdi 8,7, 64-24
674 or 0,0,8
675 std 0,0(4)
676 b L(du_done)
678 .align 4
679 L(du4_do):
680 cmpldi cr5, 10, 6
681 beq cr0, L(du4_dox)
682 blt cr5, L(du5_do)
683 beq cr5, L(du6_do)
684 b L(du7_do)
685 L(du4_dox):
686 bf 30,L(du4_1dw)
688 /* there are at least two DWs to copy */
689 sldi 0,6, 32
690 srdi 8,7, 64-32
691 or 0,0,8
692 ld 6,16(5)
693 std 0,0(4)
694 sldi 0,7, 32
695 srdi 8,6, 64-32
696 or 0,0,8
697 ld 7,24(5)
698 std 0,8(4)
699 addi 4,4,16
700 addi 5,5,32
701 blt cr6,L(du4_fini) /* if total DWs = 3, then bypass loop */
702 bf 31,L(du4_loop)
703 /* there is a third DW to copy */
704 sldi 0,6, 32
705 srdi 8,7, 64-32
706 or 0,0,8
707 std 0,0(4)
708 mr 6,7
709 ld 7,0(5)
710 addi 5,5,8
711 addi 4,4,8
712 beq cr6,L(du4_fini) /* if total DWs = 4, then bypass loop */
713 b L(du4_loop)
714 .align 4
715 L(du4_1dw):
716 sldi 0,6, 32
717 srdi 8,7, 64-32
718 addi 5,5,16
719 or 0,0,8
720 bf 31,L(du4_loop)
721 mr 6,7
722 ld 7,0(5)
723 addi 5,5,8
724 std 0,0(4)
725 addi 4,4,8
726 .align 4
727 /* copy 32 bytes at a time */
728 L(du4_loop):
729 sldi 0,6, 32
730 srdi 8,7, 64-32
731 or 0,0,8
732 ld 6,0(5)
733 std 0,0(4)
734 sldi 0,7, 32
735 srdi 8,6, 64-32
736 or 0,0,8
737 ld 7,8(5)
738 std 0,8(4)
739 sldi 0,6, 32
740 srdi 8,7, 64-32
741 or 0,0,8
742 ld 6,16(5)
743 std 0,16(4)
744 sldi 0,7, 32
745 srdi 8,6, 64-32
746 or 0,0,8
747 ld 7,24(5)
748 std 0,24(4)
749 addi 5,5,32
750 addi 4,4,32
751 bdnz+ L(du4_loop)
752 .align 4
753 L(du4_fini):
754 /* calculate and store the final DW */
755 sldi 0,6, 32
756 srdi 8,7, 64-32
757 or 0,0,8
758 std 0,0(4)
759 b L(du_done)
761 .align 4
762 L(du5_do):
763 bf 30,L(du5_1dw)
765 /* there are at least two DWs to copy */
766 sldi 0,6, 40
767 srdi 8,7, 64-40
768 or 0,0,8
769 ld 6,16(5)
770 std 0,0(4)
771 sldi 0,7, 40
772 srdi 8,6, 64-40
773 or 0,0,8
774 ld 7,24(5)
775 std 0,8(4)
776 addi 4,4,16
777 addi 5,5,32
778 blt cr6,L(du5_fini) /* if total DWs = 3, then bypass loop */
779 bf 31,L(du5_loop)
780 /* there is a third DW to copy */
781 sldi 0,6, 40
782 srdi 8,7, 64-40
783 or 0,0,8
784 std 0,0(4)
785 mr 6,7
786 ld 7,0(5)
787 addi 5,5,8
788 addi 4,4,8
789 beq cr6,L(du5_fini) /* if total DWs = 4, then bypass loop */
790 b L(du5_loop)
791 .align 4
792 L(du5_1dw):
793 sldi 0,6, 40
794 srdi 8,7, 64-40
795 addi 5,5,16
796 or 0,0,8
797 bf 31,L(du5_loop)
798 mr 6,7
799 ld 7,0(5)
800 addi 5,5,8
801 std 0,0(4)
802 addi 4,4,8
803 .align 4
804 /* copy 32 bytes at a time */
805 L(du5_loop):
806 sldi 0,6, 40
807 srdi 8,7, 64-40
808 or 0,0,8
809 ld 6,0(5)
810 std 0,0(4)
811 sldi 0,7, 40
812 srdi 8,6, 64-40
813 or 0,0,8
814 ld 7,8(5)
815 std 0,8(4)
816 sldi 0,6, 40
817 srdi 8,7, 64-40
818 or 0,0,8
819 ld 6,16(5)
820 std 0,16(4)
821 sldi 0,7, 40
822 srdi 8,6, 64-40
823 or 0,0,8
824 ld 7,24(5)
825 std 0,24(4)
826 addi 5,5,32
827 addi 4,4,32
828 bdnz+ L(du5_loop)
829 .align 4
830 L(du5_fini):
831 /* calculate and store the final DW */
832 sldi 0,6, 40
833 srdi 8,7, 64-40
834 or 0,0,8
835 std 0,0(4)
836 b L(du_done)
838 .align 4
839 L(du6_do):
840 bf 30,L(du6_1dw)
842 /* there are at least two DWs to copy */
843 sldi 0,6, 48
844 srdi 8,7, 64-48
845 or 0,0,8
846 ld 6,16(5)
847 std 0,0(4)
848 sldi 0,7, 48
849 srdi 8,6, 64-48
850 or 0,0,8
851 ld 7,24(5)
852 std 0,8(4)
853 addi 4,4,16
854 addi 5,5,32
855 blt cr6,L(du6_fini) /* if total DWs = 3, then bypass loop */
856 bf 31,L(du6_loop)
857 /* there is a third DW to copy */
858 sldi 0,6, 48
859 srdi 8,7, 64-48
860 or 0,0,8
861 std 0,0(4)
862 mr 6,7
863 ld 7,0(5)
864 addi 5,5,8
865 addi 4,4,8
866 beq cr6,L(du6_fini) /* if total DWs = 4, then bypass loop */
867 b L(du6_loop)
868 .align 4
869 L(du6_1dw):
870 sldi 0,6, 48
871 srdi 8,7, 64-48
872 addi 5,5,16
873 or 0,0,8
874 bf 31,L(du6_loop)
875 mr 6,7
876 ld 7,0(5)
877 addi 5,5,8
878 std 0,0(4)
879 addi 4,4,8
880 .align 4
881 /* copy 32 bytes at a time */
882 L(du6_loop):
883 sldi 0,6, 48
884 srdi 8,7, 64-48
885 or 0,0,8
886 ld 6,0(5)
887 std 0,0(4)
888 sldi 0,7, 48
889 srdi 8,6, 64-48
890 or 0,0,8
891 ld 7,8(5)
892 std 0,8(4)
893 sldi 0,6, 48
894 srdi 8,7, 64-48
895 or 0,0,8
896 ld 6,16(5)
897 std 0,16(4)
898 sldi 0,7, 48
899 srdi 8,6, 64-48
900 or 0,0,8
901 ld 7,24(5)
902 std 0,24(4)
903 addi 5,5,32
904 addi 4,4,32
905 bdnz+ L(du6_loop)
906 .align 4
907 L(du6_fini):
908 /* calculate and store the final DW */
909 sldi 0,6, 48
910 srdi 8,7, 64-48
911 or 0,0,8
912 std 0,0(4)
913 b L(du_done)
915 .align 4
916 L(du7_do):
917 bf 30,L(du7_1dw)
919 /* there are at least two DWs to copy */
920 sldi 0,6, 56
921 srdi 8,7, 64-56
922 or 0,0,8
923 ld 6,16(5)
924 std 0,0(4)
925 sldi 0,7, 56
926 srdi 8,6, 64-56
927 or 0,0,8
928 ld 7,24(5)
929 std 0,8(4)
930 addi 4,4,16
931 addi 5,5,32
932 blt cr6,L(du7_fini) /* if total DWs = 3, then bypass loop */
933 bf 31,L(du7_loop)
934 /* there is a third DW to copy */
935 sldi 0,6, 56
936 srdi 8,7, 64-56
937 or 0,0,8
938 std 0,0(4)
939 mr 6,7
940 ld 7,0(5)
941 addi 5,5,8
942 addi 4,4,8
943 beq cr6,L(du7_fini) /* if total DWs = 4, then bypass loop */
944 b L(du7_loop)
945 .align 4
946 L(du7_1dw):
947 sldi 0,6, 56
948 srdi 8,7, 64-56
949 addi 5,5,16
950 or 0,0,8
951 bf 31,L(du7_loop)
952 mr 6,7
953 ld 7,0(5)
954 addi 5,5,8
955 std 0,0(4)
956 addi 4,4,8
957 .align 4
958 /* copy 32 bytes at a time */
959 L(du7_loop):
960 sldi 0,6, 56
961 srdi 8,7, 64-56
962 or 0,0,8
963 ld 6,0(5)
964 std 0,0(4)
965 sldi 0,7, 56
966 srdi 8,6, 64-56
967 or 0,0,8
968 ld 7,8(5)
969 std 0,8(4)
970 sldi 0,6, 56
971 srdi 8,7, 64-56
972 or 0,0,8
973 ld 6,16(5)
974 std 0,16(4)
975 sldi 0,7, 56
976 srdi 8,6, 64-56
977 or 0,0,8
978 ld 7,24(5)
979 std 0,24(4)
980 addi 5,5,32
981 addi 4,4,32
982 bdnz+ L(du7_loop)
983 .align 4
984 L(du7_fini):
985 /* calculate and store the final DW */
986 sldi 0,6, 56
987 srdi 8,7, 64-56
988 or 0,0,8
989 std 0,0(4)
990 b L(du_done)
992 .align 4
993 L(du_done):
994 rldicr 0,31,0,60
995 mtcrf 0x01,31
996 bne cr1,.L9 /* If the tail is 0 bytes we are done! */
997 /* Return original dst pointer. */
998 ld 31,-8(1)
999 ld 3,-16(1)
1000 blr
1001 END_GEN_TB (BP_SYM (memcpy),TB_TOCLESS)
1002 libc_hidden_builtin_def (memcpy)