| blob | cb908ce9d5d041c571428d917230f152ef7f095a |
1 /* Optimized memcpy implementation for PowerPC64.
2 Copyright (C) 2003-2014 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
19 #include <sysdep.h>
21 /* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]);
22 Returns 'dst'.
24 Memcpy handles short copies (< 32-bytes) using a binary move blocks
25 (no loops) of lwz/stw. The tail (remaining 1-3) bytes is handled
26 with the appropriate combination of byte and halfword load/stores.
27 There is minimal effort to optimize the alignment of short moves.
28 The 64-bit implementations of POWER3 and POWER4 do a reasonable job
29 of handling unaligned load/stores that do not cross 32-byte boundaries.
31 Longer moves (>= 32-bytes) justify the effort to get at least the
32 destination doubleword (8-byte) aligned. Further optimization is
33 possible when both source and destination are doubleword aligned.
34 Each case has a optimized unrolled loop. */
36 .machine power4
37 EALIGN (memcpy, 5, 0)
38 CALL_MCOUNT 3
40 cmpldi cr1,5,31
41 neg 0,3
42 std 3,-16(1)
43 std 31,-8(1)
44 cfi_offset(31,-8)
45 andi. 11,3,7 /* check alignment of dst. */
46 clrldi 0,0,61 /* Number of bytes until the 1st doubleword of dst. */
47 clrldi 10,4,61 /* check alignment of src. */
48 cmpldi cr6,5,8
49 ble- cr1,.L2 /* If move < 32 bytes use short move code. */
50 cmpld cr6,10,11
51 mr 12,4
52 srdi 9,5,3 /* Number of full double words remaining. */
53 mtcrf 0x01,0
54 mr 31,5
55 beq .L0
57 subf 31,0,5
58 /* Move 0-7 bytes as needed to get the destination doubleword aligned. */
59 1: bf 31,2f
60 lbz 6,0(12)
61 addi 12,12,1
62 stb 6,0(3)
63 addi 3,3,1
64 2: bf 30,4f
65 lhz 6,0(12)
66 addi 12,12,2
67 sth 6,0(3)
68 addi 3,3,2
69 4: bf 29,0f
70 lwz 6,0(12)
71 addi 12,12,4
72 stw 6,0(3)
73 addi 3,3,4
74 0:
75 clrldi 10,12,61 /* check alignment of src again. */
76 srdi 9,31,3 /* Number of full double words remaining. */
78 /* Copy doublewords from source to destination, assuming the
79 destination is aligned on a doubleword boundary.
81 At this point we know there are at least 25 bytes left (32-7) to copy.
82 The next step is to determine if the source is also doubleword aligned.
83 If not branch to the unaligned move code at .L6. which uses
84 a load, shift, store strategy.
86 Otherwise source and destination are doubleword aligned, and we can
87 the optimized doubleword copy loop. */
88 .L0:
89 clrldi 11,31,61
90 mtcrf 0x01,9
91 cmpldi cr1,11,0
92 bne- cr6,.L6 /* If source is not DW aligned. */
94 /* Move doublewords where destination and source are DW aligned.
95 Use a unrolled loop to copy 4 doubleword (32-bytes) per iteration.
96 If the copy is not an exact multiple of 32 bytes, 1-3
97 doublewords are copied as needed to set up the main loop. After
98 the main loop exits there may be a tail of 1-7 bytes. These byte are
99 copied a word/halfword/byte at a time as needed to preserve alignment. */
101 srdi 8,31,5
102 cmpldi cr1,9,4
103 cmpldi cr6,11,0
104 mr 11,12
106 bf 30,1f
107 ld 6,0(12)
108 ld 7,8(12)
109 addi 11,12,16
110 mtctr 8
111 std 6,0(3)
112 std 7,8(3)
113 addi 10,3,16
114 bf 31,4f
115 ld 0,16(12)
116 std 0,16(3)
117 blt cr1,3f
118 addi 11,12,24
119 addi 10,3,24
120 b 4f
121 .align 4
122 1:
123 mr 10,3
124 mtctr 8
125 bf 31,4f
126 ld 6,0(12)
127 addi 11,12,8
128 std 6,0(3)
129 addi 10,3,8
131 .align 4
132 4:
133 ld 6,0(11)
134 ld 7,8(11)
135 ld 8,16(11)
136 ld 0,24(11)
137 addi 11,11,32
138 2:
139 std 6,0(10)
140 std 7,8(10)
141 std 8,16(10)
142 std 0,24(10)
143 addi 10,10,32
144 bdnz 4b
145 3:
147 rldicr 0,31,0,60
148 mtcrf 0x01,31
149 beq cr6,0f
150 .L9:
151 add 3,3,0
152 add 12,12,0
154 /* At this point we have a tail of 0-7 bytes and we know that the
155 destination is double word aligned. */
156 4: bf 29,2f
157 lwz 6,0(12)
158 addi 12,12,4
159 stw 6,0(3)
160 addi 3,3,4
161 2: bf 30,1f
162 lhz 6,0(12)
163 addi 12,12,2
164 sth 6,0(3)
165 addi 3,3,2
166 1: bf 31,0f
167 lbz 6,0(12)
168 stb 6,0(3)
169 0:
170 /* Return original dst pointer. */
171 ld 31,-8(1)
172 ld 3,-16(1)
173 blr
175 /* Copy up to 31 bytes. This divided into two cases 0-8 bytes and 9-31
176 bytes. Each case is handled without loops, using binary (1,2,4,8)
177 tests.
179 In the short (0-8 byte) case no attempt is made to force alignment
180 of either source or destination. The hardware will handle the
181 unaligned load/stores with small delays for crossing 32- 64-byte, and
182 4096-byte boundaries. Since these short moves are unlikely to be
183 unaligned or cross these boundaries, the overhead to force
184 alignment is not justified.
186 The longer (9-31 byte) move is more likely to cross 32- or 64-byte
187 boundaries. Since only loads are sensitive to the 32-/64-byte
188 boundaries it is more important to align the source then the
189 destination. If the source is not already word aligned, we first
190 move 1-3 bytes as needed. Since we are only word aligned we don't
191 use double word load/stores to insure that all loads are aligned.
192 While the destination and stores may still be unaligned, this
193 is only an issue for page (4096 byte boundary) crossing, which
194 should be rare for these short moves. The hardware handles this
195 case automatically with a small delay. */
197 .align 4
198 .L2:
199 mtcrf 0x01,5
200 neg 8,4
201 clrrdi 11,4,2
202 andi. 0,8,3
203 ble cr6,.LE8 /* Handle moves of 0-8 bytes. */
204 /* At least 9 bytes left. Get the source word aligned. */
205 cmpldi cr1,5,16
206 mr 10,5
207 mr 12,4
208 cmpldi cr6,0,2
209 beq .L3 /* If the source is already word aligned skip this. */
210 /* Copy 1-3 bytes to get source address word aligned. */
211 lwz 6,0(11)
212 subf 10,0,5
213 add 12,4,0
214 blt cr6,5f
215 srdi 7,6,16
216 bgt cr6,3f
217 #ifdef __LITTLE_ENDIAN__
218 sth 7,0(3)
219 #else
220 sth 6,0(3)
221 #endif
222 b 7f
223 .align 4
224 3:
225 #ifdef __LITTLE_ENDIAN__
226 rotlwi 6,6,24
227 stb 6,0(3)
228 sth 7,1(3)
229 #else
230 stb 7,0(3)
231 sth 6,1(3)
232 #endif
233 b 7f
234 .align 4
235 5:
236 #ifdef __LITTLE_ENDIAN__
237 rotlwi 6,6,8
238 #endif
239 stb 6,0(3)
240 7:
241 cmpldi cr1,10,16
242 add 3,3,0
243 mtcrf 0x01,10
244 .align 4
245 .L3:
246 /* At least 6 bytes left and the source is word aligned. */
247 blt cr1,8f
248 16: /* Move 16 bytes. */
249 lwz 6,0(12)
250 lwz 7,4(12)
251 stw 6,0(3)
252 lwz 6,8(12)
253 stw 7,4(3)
254 lwz 7,12(12)
255 addi 12,12,16
256 stw 6,8(3)
257 stw 7,12(3)
258 addi 3,3,16
259 8: /* Move 8 bytes. */
260 bf 28,4f
261 lwz 6,0(12)
262 lwz 7,4(12)
263 addi 12,12,8
264 stw 6,0(3)
265 stw 7,4(3)
266 addi 3,3,8
267 4: /* Move 4 bytes. */
268 bf 29,2f
269 lwz 6,0(12)
270 addi 12,12,4
271 stw 6,0(3)
272 addi 3,3,4
273 2: /* Move 2-3 bytes. */
274 bf 30,1f
275 lhz 6,0(12)
276 sth 6,0(3)
277 bf 31,0f
278 lbz 7,2(12)
279 stb 7,2(3)
280 ld 3,-16(1)
281 blr
282 1: /* Move 1 byte. */
283 bf 31,0f
284 lbz 6,0(12)
285 stb 6,0(3)
286 0:
287 /* Return original dst pointer. */
288 ld 3,-16(1)
289 blr
291 /* Special case to copy 0-8 bytes. */
292 .align 4
293 .LE8:
294 mr 12,4
295 bne cr6,4f
296 /* Would have liked to use use ld/std here but the 630 processors are
297 slow for load/store doubles that are not at least word aligned.
298 Unaligned Load/Store word execute with only a 1 cycle penalty. */
299 lwz 6,0(4)
300 lwz 7,4(4)
301 stw 6,0(3)
302 stw 7,4(3)
303 /* Return original dst pointer. */
304 ld 3,-16(1)
305 blr
306 .align 4
307 4: bf 29,2b
308 lwz 6,0(4)
309 stw 6,0(3)
310 6:
311 bf 30,5f
312 lhz 7,4(4)
313 sth 7,4(3)
314 bf 31,0f
315 lbz 8,6(4)
316 stb 8,6(3)
317 ld 3,-16(1)
318 blr
319 .align 4
320 5:
321 bf 31,0f
322 lbz 6,4(4)
323 stb 6,4(3)
324 .align 4
325 0:
326 /* Return original dst pointer. */
327 ld 3,-16(1)
328 blr
330 .align 4
331 .L6:
333 /* Copy doublewords where the destination is aligned but the source is
334 not. Use aligned doubleword loads from the source, shifted to realign
335 the data, to allow aligned destination stores. */
336 addi 11,9,-1 /* loop DW count is one less than total */
337 subf 5,10,12
338 sldi 10,10,3
339 mr 4,3
340 srdi 8,11,2 /* calculate the 32 byte loop count */
341 ld 6,0(5)
342 mtcrf 0x01,11
343 cmpldi cr6,9,4
344 mtctr 8
345 ld 7,8(5)
346 subfic 9,10,64
347 bf 30,1f
349 /* there are at least two DWs to copy */
350 #ifdef __LITTLE_ENDIAN__
351 srd 0,6,10
352 sld 8,7,9
353 #else
354 sld 0,6,10
355 srd 8,7,9
356 #endif
357 or 0,0,8
358 ld 6,16(5)
359 std 0,0(4)
360 #ifdef __LITTLE_ENDIAN__
361 srd 0,7,10
362 sld 8,6,9
363 #else
364 sld 0,7,10
365 srd 8,6,9
366 #endif
367 or 0,0,8
368 ld 7,24(5)
369 std 0,8(4)
370 addi 4,4,16
371 addi 5,5,32
372 blt cr6,8f /* if total DWs = 3, then bypass loop */
373 bf 31,4f
374 /* there is a third DW to copy */
375 #ifdef __LITTLE_ENDIAN__
376 srd 0,6,10
377 sld 8,7,9
378 #else
379 sld 0,6,10
380 srd 8,7,9
381 #endif
382 or 0,0,8
383 std 0,0(4)
384 mr 6,7
385 ld 7,0(5)
386 addi 5,5,8
387 addi 4,4,8
388 beq cr6,8f /* if total DWs = 4, then bypass loop */
389 b 4f
390 .align 4
391 1:
392 #ifdef __LITTLE_ENDIAN__
393 srd 0,6,10
394 sld 8,7,9
395 #else
396 sld 0,6,10
397 srd 8,7,9
398 #endif
399 addi 5,5,16
400 or 0,0,8
401 bf 31,4f
402 mr 6,7
403 ld 7,0(5)
404 addi 5,5,8
405 std 0,0(4)
406 addi 4,4,8
407 .align 4
408 /* copy 32 bytes at a time */
409 4:
410 #ifdef __LITTLE_ENDIAN__
411 srd 0,6,10
412 sld 8,7,9
413 #else
414 sld 0,6,10
415 srd 8,7,9
416 #endif
417 or 0,0,8
418 ld 6,0(5)
419 std 0,0(4)
420 #ifdef __LITTLE_ENDIAN__
421 srd 0,7,10
422 sld 8,6,9
423 #else
424 sld 0,7,10
425 srd 8,6,9
426 #endif
427 or 0,0,8
428 ld 7,8(5)
429 std 0,8(4)
430 #ifdef __LITTLE_ENDIAN__
431 srd 0,6,10
432 sld 8,7,9
433 #else
434 sld 0,6,10
435 srd 8,7,9
436 #endif
437 or 0,0,8
438 ld 6,16(5)
439 std 0,16(4)
440 #ifdef __LITTLE_ENDIAN__
441 srd 0,7,10
442 sld 8,6,9
443 #else
444 sld 0,7,10
445 srd 8,6,9
446 #endif
447 or 0,0,8
448 ld 7,24(5)
449 std 0,24(4)
450 addi 5,5,32
451 addi 4,4,32
452 bdnz+ 4b
453 .align 4
454 8:
455 /* calculate and store the final DW */
456 #ifdef __LITTLE_ENDIAN__
457 srd 0,6,10
458 sld 8,7,9
459 #else
460 sld 0,6,10
461 srd 8,7,9
462 #endif
463 or 0,0,8
464 std 0,0(4)
465 3:
466 rldicr 0,31,0,60
467 mtcrf 0x01,31
468 bne cr1,.L9 /* If the tail is 0 bytes we are done! */
469 /* Return original dst pointer. */
470 ld 31,-8(1)
471 ld 3,-16(1)
472 blr
473 END_GEN_TB (memcpy,TB_TOCLESS)
474 libc_hidden_builtin_def (memcpy)