| blob | c48db2f3df09465ab7255f4f411979c0729e2eff |
1 /* Optimized memcpy implementation for PowerPC32 on PowerPC64.
2 Copyright (C) 2003, 2006 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA
18 02110-1301 USA. */
20 #include <sysdep.h>
21 #include <bp-sym.h>
22 #include <bp-asm.h>
24 /* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]);
25 Returns 'dst'.
27 Memcpy handles short copies (< 32-bytes) using a binary move blocks
28 (no loops) of lwz/stw. The tail (remaining 1-3) bytes is handled
29 with the appropriate combination of byte and halfword load/stores.
30 There is minimal effort to optimize the alignment of short moves.
32 Longer moves (>= 32-bytes) justify the effort to get at least the
33 destination word (4-byte) aligned. Further optimization is
34 possible when both source and destination are word aligned.
35 Each case has an optimized unrolled loop. */
37 EALIGN (BP_SYM (memcpy), 5, 0)
38 CALL_MCOUNT
40 stwu 1,-32(1)
41 cfi_adjust_cfa_offset(32)
42 stw 30,20(1)
43 cfi_offset(30,(20-32))
44 mr 30,3
45 cmplwi cr1,5,31
46 stw 31,24(1)
47 cfi_offset(31,(24-32))
48 neg 0,3
49 andi. 11,3,3 /* check alignment of dst. */
50 clrlwi 0,0,30 /* Number of bytes until the 1st word of dst. */
51 clrlwi 10,4,30 /* check alignment of src. */
52 cmplwi cr6,5,8
53 ble- cr1,.L2 /* If move < 32 bytes use short move code. */
54 cmplw cr6,10,11
55 mr 12,4
56 srwi 9,5,2 /* Number of full words remaining. */
57 mtcrf 0x01,0
58 mr 31,5
59 beq .L0
61 subf 31,0,5
62 /* Move 0-3 bytes as needed to get the destination word aligned. */
63 1: bf 31,2f
64 lbz 6,0(12)
65 addi 12,12,1
66 stb 6,0(3)
67 addi 3,3,1
68 2: bf 30,0f
69 lhz 6,0(12)
70 addi 12,12,2
71 sth 6,0(3)
72 addi 3,3,2
73 0:
74 clrlwi 10,12,30 /* check alignment of src again. */
75 srwi 9,31,2 /* Number of full words remaining. */
77 /* Copy words from source to destination, assuming the destination is
78 aligned on a word boundary.
80 At this point we know there are at least 25 bytes left (32-7) to copy.
81 The next step is to determine if the source is also word aligned.
82 If not branch to the unaligned move code at .L6. which uses
83 a load, shift, store strategy.
85 Otherwise source and destination are word aligned, and we can use
86 the optimized word copy loop. */
87 .L0:
88 clrlwi 11,31,30 /* calculate the number of tail bytes */
89 mtcrf 0x01,9
90 bne- cr6,.L6 /* If source is not word aligned. */
92 /* Move words where destination and source are word aligned.
93 Use an unrolled loop to copy 4 words (16-bytes) per iteration.
94 If the the copy is not an exact multiple of 16 bytes, 1-3
95 words are copied as needed to set up the main loop. After
96 the main loop exits there may be a tail of 1-3 bytes. These bytes are
97 copied a halfword/byte at a time as needed to preserve alignment. */
99 srwi 8,31,4 /* calculate the 16 byte loop count */
100 cmplwi cr1,9,4
101 cmplwi cr6,11,0
102 mr 11,12
104 bf 30,1f
105 lwz 6,0(12)
106 lwz 7,4(12)
107 addi 11,12,8
108 mtctr 8
109 stw 6,0(3)
110 stw 7,4(3)
111 addi 10,3,8
112 bf 31,4f
113 lwz 0,8(12)
114 stw 0,8(3)
115 blt cr1,3f
116 addi 11,12,12
117 addi 10,3,12
118 b 4f
119 .align 4
120 1:
121 mr 10,3
122 mtctr 8
123 bf 31,4f
124 lwz 6,0(12)
125 addi 11,12,4
126 stw 6,0(3)
127 addi 10,3,4
129 .align 4
130 4:
131 lwz 6,0(11)
132 lwz 7,4(11)
133 lwz 8,8(11)
134 lwz 0,12(11)
135 stw 6,0(10)
136 stw 7,4(10)
137 stw 8,8(10)
138 stw 0,12(10)
139 addi 11,11,16
140 addi 10,10,16
141 bdnz 4b
142 3:
143 clrrwi 0,31,2
144 mtcrf 0x01,31
145 beq cr6,0f
146 .L9:
147 add 3,3,0
148 add 12,12,0
150 /* At this point we have a tail of 0-3 bytes and we know that the
151 destination is word aligned. */
152 2: bf 30,1f
153 lhz 6,0(12)
154 addi 12,12,2
155 sth 6,0(3)
156 addi 3,3,2
157 1: bf 31,0f
158 lbz 6,0(12)
159 stb 6,0(3)
160 0:
161 /* Return original dst pointer. */
162 mr 3,30
163 lwz 30,20(1)
164 lwz 31,24(1)
165 addi 1,1,32
166 blr
168 /* Copy up to 31 bytes. This is divided into two cases 0-8 bytes and
169 9-31 bytes. Each case is handled without loops, using binary
170 (1,2,4,8) tests.
172 In the short (0-8 byte) case no attempt is made to force alignment
173 of either source or destination. The hardware will handle the
174 unaligned load/stores with small delays for crossing 32- 64-byte, and
175 4096-byte boundaries. Since these short moves are unlikely to be
176 unaligned or cross these boundaries, the overhead to force
177 alignment is not justified.
179 The longer (9-31 byte) move is more likely to cross 32- or 64-byte
180 boundaries. Since only loads are sensitive to the 32-/64-byte
181 boundaries it is more important to align the source than the
182 destination. If the source is not already word aligned, we first
183 move 1-3 bytes as needed. While the destination and stores may
184 still be unaligned, this is only an issue for page (4096 byte
185 boundary) crossing, which should be rare for these short moves.
186 The hardware handles this case automatically with a small delay. */
188 .align 4
189 .L2:
190 mtcrf 0x01,5
191 neg 8,4
192 clrrwi 11,4,2
193 andi. 0,8,3
194 ble cr6,.LE8 /* Handle moves of 0-8 bytes. */
195 /* At least 9 bytes left. Get the source word aligned. */
196 cmplwi cr1,5,16
197 mr 10,5
198 mr 12,4
199 cmplwi cr6,0,2
200 beq .L3 /* If the source is already word aligned skip this. */
201 /* Copy 1-3 bytes to get source address word aligned. */
202 lwz 6,0(11)
203 subf 10,0,5
204 add 12,4,0
205 blt cr6,5f
206 srwi 7,6,16
207 bgt cr6,3f
208 sth 6,0(3)
209 b 7f
210 .align 4
211 3:
212 stb 7,0(3)
213 sth 6,1(3)
214 b 7f
215 .align 4
216 5:
217 stb 6,0(3)
218 7:
219 cmplwi cr1,10,16
220 add 3,3,0
221 mtcrf 0x01,10
222 .align 4
223 .L3:
224 /* At least 6 bytes left and the source is word aligned. */
225 blt cr1,8f
226 16: /* Move 16 bytes. */
227 lwz 6,0(12)
228 lwz 7,4(12)
229 stw 6,0(3)
230 lwz 6,8(12)
231 stw 7,4(3)
232 lwz 7,12(12)
233 addi 12,12,16
234 stw 6,8(3)
235 stw 7,12(3)
236 addi 3,3,16
237 8: /* Move 8 bytes. */
238 bf 28,4f
239 lwz 6,0(12)
240 lwz 7,4(12)
241 addi 12,12,8
242 stw 6,0(3)
243 stw 7,4(3)
244 addi 3,3,8
245 4: /* Move 4 bytes. */
246 bf 29,2f
247 lwz 6,0(12)
248 addi 12,12,4
249 stw 6,0(3)
250 addi 3,3,4
251 2: /* Move 2-3 bytes. */
252 bf 30,1f
253 lhz 6,0(12)
254 sth 6,0(3)
255 bf 31,0f
256 lbz 7,2(12)
257 stb 7,2(3)
258 mr 3,30
259 lwz 30,20(1)
260 addi 1,1,32
261 blr
262 1: /* Move 1 byte. */
263 bf 31,0f
264 lbz 6,0(12)
265 stb 6,0(3)
266 0:
267 /* Return original dst pointer. */
268 mr 3,30
269 lwz 30,20(1)
270 addi 1,1,32
271 blr
273 /* Special case to copy 0-8 bytes. */
274 .align 4
275 .LE8:
276 mr 12,4
277 bne cr6,4f
278 lwz 6,0(4)
279 lwz 7,4(4)
280 stw 6,0(3)
281 stw 7,4(3)
282 /* Return original dst pointer. */
283 mr 3,30
284 lwz 30,20(1)
285 addi 1,1,32
286 blr
287 .align 4
288 4: bf 29,2b
289 lwz 6,0(4)
290 stw 6,0(3)
291 6:
292 bf 30,5f
293 lhz 7,4(4)
294 sth 7,4(3)
295 bf 31,0f
296 lbz 8,6(4)
297 stb 8,6(3)
298 mr 3,30
299 lwz 30,20(1)
300 addi 1,1,32
301 blr
302 .align 4
303 5:
304 bf 31,0f
305 lbz 6,4(4)
306 stb 6,4(3)
307 .align 4
308 0:
309 /* Return original dst pointer. */
310 mr 3,30
311 lwz 30,20(1)
312 addi 1,1,32
313 blr
315 .align 4
316 .L6:
318 /* Copy words where the destination is aligned but the source is
319 not. Use aligned word loads from the source, shifted to realign
320 the data, to allow aligned destination stores.
321 Use an unrolled loop to copy 4 words (16-bytes) per iteration.
322 A single word is retained for storing at loop exit to avoid walking
323 off the end of a page within the loop.
324 If the copy is not an exact multiple of 16 bytes, 1-3
325 words are copied as needed to set up the main loop. After
326 the main loop exits there may be a tail of 1-3 bytes. These bytes are
327 copied a halfword/byte at a time as needed to preserve alignment. */
330 cmplwi cr6,11,0 /* are there tail bytes left ? */
331 subf 5,10,12 /* back up src pointer to prev word alignment */
332 slwi 10,10,3 /* calculate number of bits to shift 1st word left */
333 addi 11,9,-1 /* we move one word after the loop */
334 srwi 8,11,2 /* calculate the 16 byte loop count */
335 lwz 6,0(5) /* load 1st src word into R6 */
336 mr 4,3
337 lwz 7,4(5) /* load 2nd src word into R7 */
338 mtcrf 0x01,11
339 subfic 9,10,32 /* number of bits to shift 2nd word right */
340 mtctr 8
341 bf 30,1f
343 /* there are at least two words to copy, so copy them */
344 slw 0,6,10 /* shift 1st src word to left align it in R0 */
345 srw 8,7,9 /* shift 2nd src word to right align it in R8 */
346 or 0,0,8 /* or them to get word to store */
347 lwz 6,8(5) /* load the 3rd src word */
348 stw 0,0(4) /* store the 1st dst word */
349 slw 0,7,10 /* now left align 2nd src word into R0 */
350 srw 8,6,9 /* shift 3rd src word to right align it in R8 */
351 or 0,0,8 /* or them to get word to store */
352 lwz 7,12(5)
353 stw 0,4(4) /* store the 2nd dst word */
354 addi 4,4,8
355 addi 5,5,16
356 bf 31,4f
357 /* there is a third word to copy, so copy it */
358 slw 0,6,10 /* shift 3rd src word to left align it in R0 */
359 srw 8,7,9 /* shift 4th src word to right align it in R8 */
360 or 0,0,8 /* or them to get word to store */
361 stw 0,0(4) /* store 3rd dst word */
362 mr 6,7
363 lwz 7,0(5)
364 addi 5,5,4
365 addi 4,4,4
366 b 4f
367 .align 4
368 1:
369 slw 0,6,10 /* shift 1st src word to left align it in R0 */
370 srw 8,7,9 /* shift 2nd src word to right align it in R8 */
371 addi 5,5,8
372 or 0,0,8 /* or them to get word to store */
373 bf 31,4f
374 mr 6,7
375 lwz 7,0(5)
376 addi 5,5,4
377 stw 0,0(4) /* store the 1st dst word */
378 addi 4,4,4
380 .align 4
381 4:
382 /* copy 16 bytes at a time */
383 slw 0,6,10
384 srw 8,7,9
385 or 0,0,8
386 lwz 6,0(5)
387 stw 0,0(4)
388 slw 0,7,10
389 srw 8,6,9
390 or 0,0,8
391 lwz 7,4(5)
392 stw 0,4(4)
393 slw 0,6,10
394 srw 8,7,9
395 or 0,0,8
396 lwz 6,8(5)
397 stw 0,8(4)
398 slw 0,7,10
399 srw 8,6,9
400 or 0,0,8
401 lwz 7,12(5)
402 stw 0,12(4)
403 addi 5,5,16
404 addi 4,4,16
405 bdnz+ 4b
406 8:
407 /* calculate and store the final word */
408 slw 0,6,10
409 srw 8,7,9
410 or 0,0,8
411 stw 0,0(4)
412 3:
413 clrrwi 0,31,2
414 mtcrf 0x01,31
415 bne cr6,.L9 /* If the tail is 0 bytes we are done! */
417 /* Return original dst pointer. */
418 mr 3,30
419 lwz 30,20(1)
420 lwz 31,24(1)
421 addi 1,1,32
422 blr
423 END (BP_SYM (memcpy))
425 libc_hidden_builtin_def (memcpy)