| blob | e8d56eb1358070182b8cc2c0bf61964f146b3963 |
1 /* Optimized memcpy implementation for PowerPC32 on POWER6.
2 Copyright (C) 2003, 2006 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA
18 02110-1301 USA. */
20 #include <sysdep.h>
21 #include <bp-sym.h>
22 #include <bp-asm.h>
24 /* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]);
25 Returns 'dst'.
27 Memcpy handles short copies (< 32-bytes) using a binary move blocks
28 (no loops) of lwz/stw. The tail (remaining 1-3) bytes is handled
29 with the appropriate combination of byte and halfword load/stores.
30 There is minimal effort to optimize the alignment of short moves.
32 Longer moves (>= 32-bytes) justify the effort to get at least the
33 destination word (4-byte) aligned. Further optimization is
34 possible when both source and destination are word aligned.
35 Each case has an optimized unrolled loop. */
37 EALIGN (BP_SYM (memcpy), 5, 0)
38 CALL_MCOUNT
40 stwu 1,-32(1)
41 cfi_adjust_cfa_offset(32)
42 cmplwi cr1,5,31 /* check for short move. */
43 neg 0,3
44 cmplwi cr1,5,31
45 clrlwi 10,4,30 /* check alignment of src. */
46 andi. 11,3,3 /* check alignment of dst. */
47 clrlwi 0,0,30 /* Number of bytes until the 1st word of dst. */
48 ble- cr1,L(word_unaligned_short) /* If move < 32 bytes. */
49 cmplw cr6,10,11
50 stw 31,24(1)
51 cfi_offset(31,(24-32))
52 stw 30,20(1)
53 cfi_offset(30,(20-32))
54 mr 30,3
55 beq .L0
56 mtcrf 0x01,0
57 subf 31,0,5 /* Length after alignment. */
58 add 12,4,0 /* Compute src addr after alignment. */
59 /* Move 0-3 bytes as needed to get the destination word aligned. */
60 1: bf 31,2f
61 lbz 6,0(4)
62 bf 30,3f
63 lhz 7,1(4)
64 stb 6,0(3)
65 sth 7,1(3)
66 addi 3,3,3
67 b 0f
68 3:
69 stb 6,0(3)
70 addi 3,3,1
71 b 0f
72 2: bf 30,0f
73 lhz 6,0(4)
74 sth 6,0(3)
75 addi 3,3,2
76 0:
77 clrlwi 10,12,30 /* check alignment of src again. */
78 srwi 9,31,2 /* Number of full words remaining. */
79 bne- cr6,L(wdu) /* If source is not word aligned. .L6 */
80 clrlwi 11,31,30 /* calculate the number of tail bytes */
81 b L(word_aligned)
82 /* Copy words from source to destination, assuming the destination is
83 aligned on a word boundary.
85 At this point we know there are at least 29 bytes left (32-3) to copy.
86 The next step is to determine if the source is also word aligned.
87 If not branch to the unaligned move code at .L6. which uses
88 a load, shift, store strategy.
90 Otherwise source and destination are word aligned, and we can use
91 the optimized word copy loop. */
92 .align 4
93 .L0:
94 mr 31,5
95 mr 12,4
96 bne- cr6,L(wdu) /* If source is not word aligned. .L6 */
97 srwi 9,5,2 /* Number of full words remaining. */
98 clrlwi 11,5,30 /* calculate the number of tail bytes */
100 /* Move words where destination and source are word aligned.
101 Use an unrolled loop to copy 4 words (16-bytes) per iteration.
102 If the the copy is not an exact multiple of 16 bytes, 1-3
103 words are copied as needed to set up the main loop. After
104 the main loop exits there may be a tail of 1-3 bytes. These bytes are
105 copied a halfword/byte at a time as needed to preserve alignment. */
106 L(word_aligned):
107 mtcrf 0x01,9
108 srwi 8,31,4 /* calculate the 16 byte loop count */
109 cmplwi cr1,9,4
110 cmplwi cr6,11,0
111 mr 11,12
113 bf 30,1f
114 lwz 6,0(12)
115 lwz 7,4(12)
116 addi 11,12,8
117 mtctr 8
118 stw 6,0(3)
119 stw 7,4(3)
120 addi 10,3,8
121 bf 31,4f
122 lwz 0,8(12)
123 stw 0,8(3)
124 blt cr1,3f
125 addi 11,12,12
126 addi 10,3,12
127 b 4f
128 .align 4
129 1:
130 mr 10,3
131 mtctr 8
132 bf 31,4f
133 lwz 6,0(12)
134 addi 11,12,4
135 stw 6,0(3)
136 addi 10,3,4
138 .align 4
139 4:
140 lwz 6,0(11)
141 lwz 7,4(11)
142 lwz 8,8(11)
143 lwz 0,12(11)
144 stw 6,0(10)
145 stw 7,4(10)
146 stw 8,8(10)
147 stw 0,12(10)
148 addi 11,11,16
149 addi 10,10,16
150 bdnz 4b
151 3:
152 clrrwi 0,31,2
153 mtcrf 0x01,31
154 beq cr6,0f
155 .L9:
156 add 3,3,0
157 add 12,12,0
159 /* At this point we have a tail of 0-3 bytes and we know that the
160 destination is word aligned. */
161 2: bf 30,1f
162 lhz 6,0(12)
163 addi 12,12,2
164 sth 6,0(3)
165 addi 3,3,2
166 1: bf 31,0f
167 lbz 6,0(12)
168 stb 6,0(3)
169 0:
170 /* Return original dst pointer. */
171 mr 3,30
172 lwz 30,20(1)
173 lwz 31,24(1)
174 addi 1,1,32
175 blr
177 /* Copy up to 31 bytes. This divided into two cases 0-8 bytes and 9-31
178 bytes. Each case is handled without loops, using binary (1,2,4,8)
179 tests.
181 In the short (0-8 byte) case no attempt is made to force alignment
182 of either source or destination. The hardware will handle the
183 unaligned load/stores with small delays for crossing 32- 128-byte,
184 and 4096-byte boundaries. Since these short moves are unlikely to be
185 unaligned or cross these boundaries, the overhead to force
186 alignment is not justified.
188 The longer (9-31 byte) move is more likely to cross 32- or 128-byte
189 boundaries. Since only loads are sensitive to the 32-/128-byte
190 boundaries it is more important to align the source then the
191 destination. If the source is not already word aligned, we first
192 move 1-3 bytes as needed. Since we are only word aligned we don't
193 use double word load/stores to insure that all loads are aligned.
194 While the destination and stores may still be unaligned, this
195 is only an issue for page (4096 byte boundary) crossing, which
196 should be rare for these short moves. The hardware handles this
197 case automatically with a small (~20 cycle) delay. */
198 .align 4
200 cfi_same_value (31)
201 cfi_same_value (30)
202 L(word_unaligned_short):
203 mtcrf 0x01,5
204 cmplwi cr6,5,8
205 neg 8,4
206 clrrwi 9,4,2
207 andi. 0,8,3
208 beq cr6,L(wus_8) /* Handle moves of 8 bytes. */
209 /* At least 9 bytes left. Get the source word aligned. */
210 cmpldi cr1,5,16
211 mr 12,4
212 ble cr6,L(wus_4) /* Handle moves of 0-8 bytes. */
213 mr 11,3
214 mr 10,5
215 cmplwi cr6,0,2
216 beq L(wus_tail) /* If the source is already word aligned skip this. */
217 /* Copy 1-3 bytes to get source address word aligned. */
218 lwz 6,0(9)
219 subf 10,0,5
220 add 12,4,0
221 blt cr6,5f
222 srdi 7,6,16
223 bgt cr6,3f
224 sth 6,0(3)
225 b 7f
226 .align 4
227 3:
228 stb 7,0(3)
229 sth 6,1(3)
230 b 7f
231 .align 4
232 5:
233 stb 6,0(3)
234 7:
235 cmplwi cr1,10,16
236 add 11,3,0
237 mtcrf 0x01,10
238 .align 4
239 L(wus_tail):
240 /* At least 6 bytes left and the source is word aligned. This allows
241 some speculative loads up front. */
242 /* We need to special case the fall-through because the biggest delays
243 are due to address computation not being ready in time for the
244 AGEN. */
245 lwz 6,0(12)
246 lwz 7,4(12)
247 blt cr1,L(wus_tail8)
248 cmplwi cr0,10,24
249 L(wus_tail16): /* Move 16 bytes. */
250 stw 6,0(11)
251 stw 7,4(11)
252 lwz 6,8(12)
253 lwz 7,12(12)
254 stw 6,8(11)
255 stw 7,12(11)
256 /* Move 8 bytes more. */
257 bf 28,L(wus_tail16p8)
258 cmplwi cr1,10,28
259 lwz 6,16(12)
260 lwz 7,20(12)
261 stw 6,16(11)
262 stw 7,20(11)
263 /* Move 4 bytes more. */
264 bf 29,L(wus_tail16p4)
265 lwz 6,24(12)
266 stw 6,24(11)
267 addi 12,12,28
268 addi 11,11,28
269 bgt cr1,L(wus_tail2)
270 /* exactly 28 bytes. Return original dst pointer and exit. */
271 addi 1,1,32
272 blr
273 .align 4
274 L(wus_tail16p8): /* less then 8 bytes left. */
275 beq cr1,L(wus_tailX) /* exactly 16 bytes, early exit. */
276 cmplwi cr1,10,20
277 bf 29,L(wus_tail16p2)
278 /* Move 4 bytes more. */
279 lwz 6,16(12)
280 stw 6,16(11)
281 addi 12,12,20
282 addi 11,11,20
283 bgt cr1,L(wus_tail2)
284 /* exactly 20 bytes. Return original dst pointer and exit. */
285 addi 1,1,32
286 blr
287 .align 4
288 L(wus_tail16p4): /* less then 4 bytes left. */
289 addi 12,12,24
290 addi 11,11,24
291 bgt cr0,L(wus_tail2)
292 /* exactly 24 bytes. Return original dst pointer and exit. */
293 addi 1,1,32
294 blr
295 .align 4
296 L(wus_tail16p2): /* 16 bytes moved, less then 4 bytes left. */
297 addi 12,12,16
298 addi 11,11,16
299 b L(wus_tail2)
301 .align 4
302 L(wus_tail8): /* Move 8 bytes. */
303 /* r6, r7 already loaded speculatively. */
304 cmplwi cr1,10,8
305 cmplwi cr0,10,12
306 bf 28,L(wus_tail4)
307 stw 6,0(11)
308 stw 7,4(11)
309 /* Move 4 bytes more. */
310 bf 29,L(wus_tail8p4)
311 lwz 6,8(12)
312 stw 6,8(11)
313 addi 12,12,12
314 addi 11,11,12
315 bgt cr0,L(wus_tail2)
316 /* exactly 12 bytes. Return original dst pointer and exit. */
317 addi 1,1,32
318 blr
319 .align 4
320 L(wus_tail8p4): /* less then 4 bytes left. */
321 addi 12,12,8
322 addi 11,11,8
323 bgt cr1,L(wus_tail2)
324 /* exactly 8 bytes. Return original dst pointer and exit. */
325 addi 1,1,32
326 blr
328 .align 4
329 L(wus_tail4): /* Move 4 bytes. */
330 /* r6 already loaded speculatively. If we are here we know there is
331 more then 4 bytes left. So there is no need to test. */
332 addi 12,12,4
333 stw 6,0(11)
334 addi 11,11,4
335 L(wus_tail2): /* Move 2-3 bytes. */
336 bf 30,L(wus_tail1)
337 lhz 6,0(12)
338 sth 6,0(11)
339 bf 31,L(wus_tailX)
340 lbz 7,2(12)
341 stb 7,2(11)
342 addi 1,1,32
343 blr
344 L(wus_tail1): /* Move 1 byte. */
345 bf 31,L(wus_tailX)
346 lbz 6,0(12)
347 stb 6,0(11)
348 L(wus_tailX):
349 /* Return original dst pointer. */
350 addi 1,1,32
351 blr
353 /* Special case to copy 0-8 bytes. */
354 .align 4
355 L(wus_8):
356 lwz 6,0(4)
357 lwz 7,4(4)
358 stw 6,0(3)
359 stw 7,4(3)
360 /* Return original dst pointer. */
361 addi 1,1,32
362 blr
363 .align 4
364 L(wus_4):
365 bf 29,L(wus_2)
366 lwz 6,0(4)
367 stw 6,0(3)
368 bf 30,L(wus_5)
369 lhz 7,4(4)
370 sth 7,4(3)
371 bf 31,L(wus_0)
372 lbz 8,6(4)
373 stb 8,6(3)
374 addi 1,1,32
375 blr
376 .align 4
377 L(wus_5):
378 bf 31,L(wus_0)
379 lbz 6,4(4)
380 stb 6,4(3)
381 /* Return original dst pointer. */
382 addi 1,1,32
383 blr
384 .align 4
385 L(wus_2): /* Move 2-3 bytes. */
386 bf 30,L(wus_1)
387 lhz 6,0(4)
388 sth 6,0(3)
389 bf 31,L(wus_0)
390 lbz 7,2(4)
391 stb 7,2(3)
392 addi 1,1,32
393 blr
394 .align 4
395 L(wus_1): /* Move 1 byte. */
396 bf 31,L(wus_0)
397 lbz 6,0(4)
398 stb 6,0(3)
399 .align 3
400 L(wus_0):
401 /* Return original dst pointer. */
402 addi 1,1,32
403 blr
405 .align 4
406 cfi_offset(31,(24-32))
407 cfi_offset(30,(20-32))
408 L(wdu):
410 /* Copy words where the destination is aligned but the source is
411 not. For power4, power5 and power6 machines there is penalty for
412 unaligned loads (src) that cross 32-byte, cacheline, or page
413 boundaries. So we want to use simple (unaligned) loads where
414 posible but avoid them where we know the load would span a 32-byte
415 boundary.
417 At this point we know we have at least 29 (32-3) bytes to copy
418 the src is unaligned. and we may cross at least one 32-byte
419 boundary. Also we have the following regester values:
420 r3 == adjusted dst, word aligned
421 r4 == unadjusted src
422 r5 == unadjusted len
423 r9 == adjusted Word length
424 r10 == src alignment (1-3)
425 r12 == adjuested src, not aligned
426 r31 == adjusted len
428 First we need to copy word upto but not crossing the next 32-byte
429 boundary. Then perform aligned loads just before and just after
430 the boundary and use shifts and or to gernerate the next aligned
431 word for dst. If more then 32 bytes remain we copy (unaligned src)
432 the next 7 words and repeat the loop until less then 32-bytes
433 remaim.
435 Then if more then 4 bytes remain we again use aligned loads,
436 shifts and or to generate the next dst word. We then process the
437 remaining words using unaligned loads as needed. Finally we check
438 if there more then 0 bytes (1-3) bytes remainting and use
439 halfword and or byte load/stores to complete the copy.
440 */
441 mr 4,12 /* restore unaligned adjusted src ptr */
442 clrlwi 0,12,27 /* Find dist from previous 32-byte boundary. */
443 slwi 10,10,3 /* calculate number of bits to shift 1st word left */
444 cmplwi cr5,0,16
445 subfic 8,0,32 /* Number of bytes to next 32-byte boundary. */
447 mtcrf 0x01,8
448 cmplwi cr1,10,16
449 subfic 9,10,32 /* number of bits to shift 2nd word right */
450 /* This test is reversed because the timing to compare the bytes to
451 32-byte boundary could not be meet. So we compare the bytes from
452 previous 32-byte boundary and invert the test. */
453 bge cr5,L(wdu_h32_8)
454 .align 4
455 lwz 6,0(4)
456 lwz 7,4(4)
457 addi 12,4,16 /* generate alternate pointers to avoid agen */
458 addi 11,3,16 /* timing issues downstream. */
459 stw 6,0(3)
460 stw 7,4(3)
461 subi 31,31,16
462 lwz 6,8(4)
463 lwz 7,12(4)
464 addi 4,4,16
465 stw 6,8(3)
466 stw 7,12(3)
467 addi 3,3,16
468 bf 28,L(wdu_h32_4)
469 lwz 6,0(12)
470 lwz 7,4(12)
471 subi 31,31,8
472 addi 4,4,8
473 stw 6,0(11)
474 stw 7,4(11)
475 addi 3,3,8
476 bf 29,L(wdu_h32_0)
477 lwz 6,8(12)
478 addi 4,4,4
479 subi 31,31,4
480 stw 6,8(11)
481 addi 3,3,4
482 b L(wdu_h32_0)
483 .align 4
484 L(wdu_h32_8):
485 bf 28,L(wdu_h32_4)
486 lwz 6,0(4)
487 lwz 7,4(4)
488 subi 31,31,8
489 bf 29,L(wdu_h32_8x)
490 stw 6,0(3)
491 stw 7,4(3)
492 lwz 6,8(4)
493 addi 4,4,12
494 subi 31,31,4
495 stw 6,8(3)
496 addi 3,3,12
497 b L(wdu_h32_0)
498 .align 4
499 L(wdu_h32_8x):
500 addi 4,4,8
501 stw 6,0(3)
502 stw 7,4(3)
503 addi 3,3,8
504 b L(wdu_h32_0)
505 .align 4
506 L(wdu_h32_4):
507 bf 29,L(wdu_h32_0)
508 lwz 6,0(4)
509 subi 31,31,4
510 addi 4,4,4
511 stw 6,0(3)
512 addi 3,3,4
513 .align 4
514 L(wdu_h32_0):
515 /* set up for 32-byte boundry crossing word move and possibly 32-byte
516 move loop. */
517 clrrwi 12,4,2
518 cmplwi cr5,31,32
519 bge cr1,L(wdu2_32)
520 #if 0
521 b L(wdu1_32)
522 /*
523 cmplwi cr1,10,8
524 beq cr1,L(wdu1_32)
525 cmplwi cr1,10,16
526 beq cr1,L(wdu2_32)
527 cmplwi cr1,10,24
528 beq cr1,L(wdu3_32)
529 */
530 L(wdu_32):
531 lwz 6,0(12)
532 cmplwi cr6,31,4
533 srwi 8,31,5 /* calculate the 32 byte loop count */
534 slw 0,6,10
535 clrlwi 31,31,27 /* The remaining bytes, < 32. */
536 blt cr5,L(wdu_32tail)
537 mtctr 8
538 cmplwi cr6,31,4
539 .align 4
540 L(wdu_loop32):
541 /* copy 32 bytes at a time */
542 lwz 8,4(12)
543 addi 12,12,32
544 lwz 7,4(4)
545 srw 8,8,9
546 or 0,0,8
547 stw 0,0(3)
548 stw 7,4(3)
549 lwz 6,8(4)
550 lwz 7,12(4)
551 stw 6,8(3)
552 stw 7,12(3)
553 lwz 6,16(4)
554 lwz 7,20(4)
555 stw 6,16(3)
556 stw 7,20(3)
557 lwz 6,24(4)
558 lwz 7,28(4)
559 lwz 8,0(12)
560 addi 4,4,32
561 stw 6,24(3)
562 stw 7,28(3)
563 addi 3,3,32
564 slw 0,8,10
565 bdnz+ L(wdu_loop32)
567 L(wdu_32tail):
568 mtcrf 0x01,31
569 cmplwi cr5,31,16
570 blt cr6,L(wdu_4tail)
571 /* calculate and store the final word */
572 lwz 8,4(12)
573 srw 8,8,9
574 or 6,0,8
575 b L(wdu_32tailx)
576 #endif
577 .align 4
578 L(wdu1_32):
579 lwz 6,-1(4)
580 cmplwi cr6,31,4
581 srwi 8,31,5 /* calculate the 32 byte loop count */
582 slwi 6,6,8
583 clrlwi 31,31,27 /* The remaining bytes, < 32. */
584 blt cr5,L(wdu1_32tail)
585 mtctr 8
586 cmplwi cr6,31,4
588 lwz 8,3(4)
589 lwz 7,4(4)
590 /* Equivalent to: srwi 8,8,32-8; or 6,6,8 */
591 rlwimi 6,8,8,(32-8),31
592 b L(wdu1_loop32x)
593 .align 4
594 L(wdu1_loop32):
595 /* copy 32 bytes at a time */
596 lwz 8,3(4)
597 lwz 7,4(4)
598 stw 10,-8(3)
599 stw 11,-4(3)
600 /* Equivalent to srwi 8,8,32-8; or 6,6,8 */
601 rlwimi 6,8,8,(32-8),31
602 L(wdu1_loop32x):
603 lwz 10,8(4)
604 lwz 11,12(4)
605 stw 6,0(3)
606 stw 7,4(3)
607 lwz 6,16(4)
608 lwz 7,20(4)
609 stw 10,8(3)
610 stw 11,12(3)
611 lwz 10,24(4)
612 lwz 11,28(4)
613 lwz 8,32-1(4)
614 addi 4,4,32
615 stw 6,16(3)
616 stw 7,20(3)
617 addi 3,3,32
618 slwi 6,8,8
619 bdnz+ L(wdu1_loop32)
620 stw 10,-8(3)
621 stw 11,-4(3)
623 L(wdu1_32tail):
624 mtcrf 0x01,31
625 cmplwi cr5,31,16
626 blt cr6,L(wdu_4tail)
627 /* calculate and store the final word */
628 lwz 8,3(4)
629 /* Equivalent to: srwi 8,8,32-9; or 6,6,8 */
630 rlwimi 6,8,8,(32-8),31
631 b L(wdu_32tailx)
633 L(wdu2_32):
634 bgt cr1,L(wdu3_32)
635 lwz 6,-2(4)
636 cmplwi cr6,31,4
637 srwi 8,31,5 /* calculate the 32 byte loop count */
638 slwi 6,6,16
639 clrlwi 31,31,27 /* The remaining bytes, < 32. */
640 blt cr5,L(wdu2_32tail)
641 mtctr 8
642 cmplwi cr6,31,4
644 lwz 8,2(4)
645 lwz 7,4(4)
646 /* Equivalent to: srwi 8,8,32-8; or 6,6,8 */
647 rlwimi 6,8,16,(32-16),31
648 b L(wdu2_loop32x)
649 .align 4
650 L(wdu2_loop32):
651 /* copy 32 bytes at a time */
652 lwz 8,2(4)
653 lwz 7,4(4)
654 stw 10,-8(3)
655 stw 11,-4(3)
656 /* Equivalent to srwi 8,8,32-8; or 6,6,8 */
657 rlwimi 6,8,16,(32-16),31
658 L(wdu2_loop32x):
659 lwz 10,8(4)
660 lwz 11,12(4)
661 stw 6,0(3)
662 stw 7,4(3)
663 lwz 6,16(4)
664 lwz 7,20(4)
665 stw 10,8(3)
666 stw 11,12(3)
667 lwz 10,24(4)
668 lwz 11,28(4)
669 /* lwz 8,0(12) */
670 lwz 8,32-2(4)
671 addi 4,4,32
672 stw 6,16(3)
673 stw 7,20(3)
674 addi 3,3,32
675 slwi 6,8,16
676 bdnz+ L(wdu2_loop32)
677 stw 10,-8(3)
678 stw 11,-4(3)
680 L(wdu2_32tail):
681 mtcrf 0x01,31
682 cmplwi cr5,31,16
683 blt cr6,L(wdu_4tail)
684 /* calculate and store the final word */
685 lwz 8,2(4)
686 /* Equivalent to: srwi 8,8,32-9; or 6,6,8 */
687 rlwimi 6,8,16,(32-16),31
688 b L(wdu_32tailx)
690 L(wdu3_32):
691 /* lwz 6,0(12) */
692 lwz 6,-3(4)
693 cmplwi cr6,31,4
694 srwi 8,31,5 /* calculate the 32 byte loop count */
695 slwi 6,6,24
696 clrlwi 31,31,27 /* The remaining bytes, < 32. */
697 blt cr5,L(wdu3_32tail)
698 mtctr 8
699 cmplwi cr6,31,4
701 lwz 8,1(4)
702 lwz 7,4(4)
703 /* Equivalent to: srwi 8,8,32-8; or 6,6,8 */
704 rlwimi 6,8,24,(32-24),31
705 b L(wdu3_loop32x)
706 .align 4
707 L(wdu3_loop32):
708 /* copy 32 bytes at a time */
709 lwz 8,1(4)
710 lwz 7,4(4)
711 stw 10,-8(3)
712 stw 11,-4(3)
713 /* Equivalent to srwi 8,8,32-8; or 6,6,8 */
714 rlwimi 6,8,24,(32-24),31
715 L(wdu3_loop32x):
716 lwz 10,8(4)
717 lwz 11,12(4)
718 stw 6,0(3)
719 stw 7,4(3)
720 lwz 6,16(4)
721 lwz 7,20(4)
722 stw 10,8(3)
723 stw 11,12(3)
724 lwz 10,24(4)
725 lwz 11,28(4)
726 lwz 8,32-3(4)
727 addi 4,4,32
728 stw 6,16(3)
729 stw 7,20(3)
730 addi 3,3,32
731 slwi 6,8,24
732 bdnz+ L(wdu3_loop32)
733 stw 10,-8(3)
734 stw 11,-4(3)
736 L(wdu3_32tail):
737 mtcrf 0x01,31
738 cmplwi cr5,31,16
739 blt cr6,L(wdu_4tail)
740 /* calculate and store the final word */
741 lwz 8,1(4)
742 /* Equivalent to: srwi 8,8,32-9; or 6,6,8 */
743 rlwimi 6,8,24,(32-24),31
744 b L(wdu_32tailx)
745 .align 4
746 L(wdu_32tailx):
747 blt cr5,L(wdu_t32_8)
748 lwz 7,4(4)
749 addi 12,4,16 /* generate alternate pointers to avoid agen */
750 addi 11,3,16 /* timing issues downstream. */
751 stw 6,0(3)
752 stw 7,4(3)
753 subi 31,31,16
754 lwz 6,8(4)
755 lwz 7,12(4)
756 addi 4,4,16
757 stw 6,8(3)
758 stw 7,12(3)
759 addi 3,3,16
760 bf 28,L(wdu_t32_4x)
761 lwz 6,0(12)
762 lwz 7,4(12)
763 addi 4,4,8
764 subi 31,31,8
765 stw 6,0(11)
766 stw 7,4(11)
767 addi 3,3,8
768 bf 29,L(wdu_t32_0)
769 lwz 6,8(12)
770 addi 4,4,4
771 subi 31,31,4
772 stw 6,8(11)
773 addi 3,3,4
774 b L(wdu_t32_0)
775 .align 4
776 L(wdu_t32_4x):
777 bf 29,L(wdu_t32_0)
778 lwz 6,0(4)
779 addi 4,4,4
780 subi 31,31,4
781 stw 6,0(3)
782 addi 3,3,4
783 b L(wdu_t32_0)
784 .align 4
785 L(wdu_t32_8):
786 bf 28,L(wdu_t32_4)
787 lwz 7,4(4)
788 subi 31,31,8
789 bf 29,L(wdu_t32_8x)
790 stw 6,0(3)
791 stw 7,4(3)
792 lwz 6,8(4)
793 subi 31,31,4
794 addi 4,4,12
795 stw 6,8(3)
796 addi 3,3,12
797 b L(wdu_t32_0)
798 .align 4
799 L(wdu_t32_8x):
800 addi 4,4,8
801 stw 6,0(3)
802 stw 7,4(3)
803 addi 3,3,8
804 b L(wdu_t32_0)
805 .align 4
806 L(wdu_t32_4):
807 subi 31,31,4
808 stw 6,0(3)
809 addi 4,4,4
810 addi 3,3,4
811 .align 4
812 L(wdu_t32_0):
813 L(wdu_4tail):
814 cmplwi cr6,31,0
815 beq cr6,L(wdus_0) /* If the tail is 0 bytes we are done! */
816 bf 30,L(wdus_3)
817 lhz 7,0(4)
818 sth 7,0(3)
819 bf 31,L(wdus_0)
820 lbz 8,2(4)
821 stb 8,2(3)
822 mr 3,30
823 lwz 30,20(1)
824 lwz 31,24(1)
825 addi 1,1,32
826 blr
827 .align 4
828 L(wdus_3):
829 bf 31,L(wus_0)
830 lbz 6,0(4)
831 stb 6,0(3)
832 .align 4
833 L(wdus_0):
834 /* Return original dst pointer. */
835 mr 3,30
836 lwz 30,20(1)
837 lwz 31,24(1)
838 addi 1,1,32
839 blr
840 END (BP_SYM (memcpy))
842 libc_hidden_builtin_def (memcpy)