| blob | 9910ebda829d1736e996bdefa898db0bd08efbb2 |
1 /* Optimized memcpy implementation for PowerPC64.
2 Copyright (C) 2003, 2006 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA
18 02110-1301 USA. */
20 #include <sysdep.h>
21 #include <bp-sym.h>
22 #include <bp-asm.h>
24 /* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]);
25 Returns 'dst'.
27 Memcpy handles short copies (< 32-bytes) using a binary move blocks
28 (no loops) of lwz/stw. The tail (remaining 1-3) bytes is handled
29 with the appropriate combination of byte and halfword load/stores.
30 There is minimal effort to optimize the alignment of short moves.
31 The 64-bit implementations of POWER3 and POWER4 do a reasonable job
32 of handling unligned load/stores that do not cross 32-byte boundries.
34 Longer moves (>= 32-bytes) justify the effort to get at least the
35 destination doubleword (8-byte) aligned. Further optimization is
36 posible when both source and destination are doubleword aligned.
37 Each case has a optimized unrolled loop. */
39 EALIGN (BP_SYM (memcpy), 5, 0)
40 CALL_MCOUNT 3
42 cmpldi cr1,5,31
43 neg 0,3
44 std 3,-16(1)
45 std 31,-8(1)
46 cfi_offset(31,-8)
47 andi. 11,3,7 /* check alignement of dst. */
48 clrldi 0,0,61 /* Number of bytes until the 1st doubleword of dst. */
49 clrldi 10,4,61 /* check alignement of src. */
50 cmpldi cr6,5,8
51 ble- cr1,.L2 /* If move < 32 bytes use short move code. */
52 cmpld cr6,10,11
53 mr 12,4
54 srdi 9,5,3 /* Number of full double words remaining. */
55 mtcrf 0x01,0
56 mr 31,5
57 beq .L0
59 subf 31,0,5
60 /* Move 0-7 bytes as needed to get the destination doubleword alligned. */
61 1: bf 31,2f
62 lbz 6,0(12)
63 addi 12,12,1
64 stb 6,0(3)
65 addi 3,3,1
66 2: bf 30,4f
67 lhz 6,0(12)
68 addi 12,12,2
69 sth 6,0(3)
70 addi 3,3,2
71 4: bf 29,0f
72 lwz 6,0(12)
73 addi 12,12,4
74 stw 6,0(3)
75 addi 3,3,4
76 0:
77 clrldi 10,12,61 /* check alignement of src again. */
78 srdi 9,31,3 /* Number of full double words remaining. */
80 /* Copy doublewords from source to destination, assumpting the
81 destination is aligned on a doubleword boundary.
83 At this point we know there are at least 25 bytes left (32-7) to copy.
84 The next step is to determine if the source is also doubleword aligned.
85 If not branch to the unaligned move code at .L6. which uses
86 a load, shift, store strategy.
88 Otherwise source and destination are doubleword aligned, and we can
89 the optimized doubleword copy loop. */
90 .L0:
91 clrldi 11,31,61
92 mtcrf 0x01,9
93 cmpldi cr1,11,0
94 bne- cr6,.L6 /* If source is not DW aligned. */
96 /* Move doublewords where destination and source are DW aligned.
97 Use a unrolled loop to copy 4 doubleword (32-bytes) per iteration.
98 If the the copy is not an exact multiple of 32 bytes, 1-3
99 doublewords are copied as needed to set up the main loop. After
100 the main loop exits there may be a tail of 1-7 bytes. These byte are
101 copied a word/halfword/byte at a time as needed to preserve alignment. */
103 srdi 8,31,5
104 cmpldi cr1,9,4
105 cmpldi cr6,11,0
106 mr 11,12
108 bf 30,1f
109 ld 6,0(12)
110 ld 7,8(12)
111 addi 11,12,16
112 mtctr 8
113 std 6,0(3)
114 std 7,8(3)
115 addi 10,3,16
116 bf 31,4f
117 ld 0,16(12)
118 std 0,16(3)
119 blt cr1,3f
120 addi 11,12,24
121 addi 10,3,24
122 b 4f
123 .align 4
124 1:
125 mr 10,3
126 mtctr 8
127 bf 31,4f
128 ld 6,0(12)
129 addi 11,12,8
130 std 6,0(3)
131 addi 10,3,8
133 .align 4
134 4:
135 ld 6,0(11)
136 ld 7,8(11)
137 ld 8,16(11)
138 ld 0,24(11)
139 addi 11,11,32
140 2:
141 std 6,0(10)
142 std 7,8(10)
143 std 8,16(10)
144 std 0,24(10)
145 addi 10,10,32
146 bdnz 4b
147 3:
149 rldicr 0,31,0,60
150 mtcrf 0x01,31
151 beq cr6,0f
152 .L9:
153 add 3,3,0
154 add 12,12,0
156 /* At this point we have a tail of 0-7 bytes and we know that the
157 destiniation is double word aligned. */
158 4: bf 29,2f
159 lwz 6,0(12)
160 addi 12,12,4
161 stw 6,0(3)
162 addi 3,3,4
163 2: bf 30,1f
164 lhz 6,0(12)
165 addi 12,12,2
166 sth 6,0(3)
167 addi 3,3,2
168 1: bf 31,0f
169 lbz 6,0(12)
170 stb 6,0(3)
171 0:
172 /* Return original dst pointer. */
173 ld 31,-8(1)
174 ld 3,-16(1)
175 blr
177 /* Copy up to 31 bytes. This divided into two cases 0-8 bytes and 9-31
178 bytes. Each case is handled without loops, using binary (1,2,4,8)
179 tests.
181 In the short (0-8 byte) case no attempt is made to force alignment
182 of either source or destination. The hardware will handle the
183 unaligned load/stores with small delays for crossing 32- 64-byte, and
184 4096-byte boundaries. Since these short moves are unlikely to be
185 unaligned or cross these boundaries, the overhead to force
186 alignment is not justified.
188 The longer (9-31 byte) move is more likely to cross 32- or 64-byte
189 boundaries. Since only loads are sensitive to the 32-/64-byte
190 boundaries it is more important to align the source then the
191 destination. If the source is not already word aligned, we first
192 move 1-3 bytes as needed. Since we are only word aligned we don't
193 use double word load/stores to insure that all loads are aligned.
194 While the destination and stores may still be unaligned, this
195 is only an issue for page (4096 byte boundary) crossing, which
196 should be rare for these short moves. The hardware handles this
197 case automatically with a small delay. */
199 .align 4
200 .L2:
201 mtcrf 0x01,5
202 neg 8,4
203 clrrdi 11,4,2
204 andi. 0,8,3
205 ble cr6,.LE8 /* Handle moves of 0-8 bytes. */
206 /* At least 9 bytes left. Get the source word aligned. */
207 cmpldi cr1,5,16
208 mr 10,5
209 mr 12,4
210 cmpldi cr6,0,2
211 beq .L3 /* If the source is already word aligned skip this. */
212 /* Copy 1-3 bytes to get source address word aligned. */
213 lwz 6,0(11)
214 subf 10,0,5
215 add 12,4,0
216 blt cr6,5f
217 srdi 7,6,16
218 bgt cr6,3f
219 sth 6,0(3)
220 b 7f
221 .align 4
222 3:
223 stb 7,0(3)
224 sth 6,1(3)
225 b 7f
226 .align 4
227 5:
228 stb 6,0(3)
229 7:
230 cmpldi cr1,10,16
231 add 3,3,0
232 mtcrf 0x01,10
233 .align 4
234 .L3:
235 /* At least 6 bytes left and the source is word aligned. */
236 blt cr1,8f
237 16: /* Move 16 bytes. */
238 lwz 6,0(12)
239 lwz 7,4(12)
240 stw 6,0(3)
241 lwz 6,8(12)
242 stw 7,4(3)
243 lwz 7,12(12)
244 addi 12,12,16
245 stw 6,8(3)
246 stw 7,12(3)
247 addi 3,3,16
248 8: /* Move 8 bytes. */
249 bf 28,4f
250 lwz 6,0(12)
251 lwz 7,4(12)
252 addi 12,12,8
253 stw 6,0(3)
254 stw 7,4(3)
255 addi 3,3,8
256 4: /* Move 4 bytes. */
257 bf 29,2f
258 lwz 6,0(12)
259 addi 12,12,4
260 stw 6,0(3)
261 addi 3,3,4
262 2: /* Move 2-3 bytes. */
263 bf 30,1f
264 lhz 6,0(12)
265 sth 6,0(3)
266 bf 31,0f
267 lbz 7,2(12)
268 stb 7,2(3)
269 ld 3,-16(1)
270 blr
271 1: /* Move 1 byte. */
272 bf 31,0f
273 lbz 6,0(12)
274 stb 6,0(3)
275 0:
276 /* Return original dst pointer. */
277 ld 3,-16(1)
278 blr
280 /* Special case to copy 0-8 bytes. */
281 .align 4
282 .LE8:
283 mr 12,4
284 bne cr6,4f
285 /* Would have liked to use use ld/std here but the 630 processors are
286 slow for load/store doubles that are not at least word aligned.
287 Unaligned Load/Store word execute with only a 1 cycle penaltity. */
288 lwz 6,0(4)
289 lwz 7,4(4)
290 stw 6,0(3)
291 stw 7,4(3)
292 /* Return original dst pointer. */
293 ld 3,-16(1)
294 blr
295 .align 4
296 4: bf 29,2b
297 lwz 6,0(4)
298 stw 6,0(3)
299 6:
300 bf 30,5f
301 lhz 7,4(4)
302 sth 7,4(3)
303 bf 31,0f
304 lbz 8,6(4)
305 stb 8,6(3)
306 ld 3,-16(1)
307 blr
308 .align 4
309 5:
310 bf 31,0f
311 lbz 6,4(4)
312 stb 6,4(3)
313 .align 4
314 0:
315 /* Return original dst pointer. */
316 ld 3,-16(1)
317 blr
319 .align 4
320 .L6:
322 /* Copy doublewords where the destination is aligned but the source is
323 not. Use aligned doubleword loads from the source, shifted to realign
324 the data, to allow aligned destination stores. */
325 addi 11,9,-1 /* loop DW count is one less than total */
326 subf 5,10,12
327 sldi 10,10,3
328 mr 4,3
329 srdi 8,11,2 /* calculate the 32 byte loop count */
330 ld 6,0(5)
331 mtcrf 0x01,11
332 cmpldi cr6,9,4
333 mtctr 8
334 ld 7,8(5)
335 subfic 9,10,64
336 bf 30,1f
338 /* there are at least two DWs to copy */
339 sld 0,6,10
340 srd 8,7,9
341 or 0,0,8
342 ld 6,16(5)
343 std 0,0(4)
344 sld 0,7,10
345 srd 8,6,9
346 or 0,0,8
347 ld 7,24(5)
348 std 0,8(4)
349 addi 4,4,16
350 addi 5,5,32
351 blt cr6,8f /* if total DWs = 3, then bypass loop */
352 bf 31,4f
353 /* there is a third DW to copy */
354 sld 0,6,10
355 srd 8,7,9
356 or 0,0,8
357 std 0,0(4)
358 mr 6,7
359 ld 7,0(5)
360 addi 5,5,8
361 addi 4,4,8
362 beq cr6,8f /* if total DWs = 4, then bypass loop */
363 b 4f
364 .align 4
365 1:
366 sld 0,6,10
367 srd 8,7,9
368 addi 5,5,16
369 or 0,0,8
370 bf 31,4f
371 mr 6,7
372 ld 7,0(5)
373 addi 5,5,8
374 std 0,0(4)
375 addi 4,4,8
376 .align 4
377 /* copy 32 bytes at a time */
378 4: sld 0,6,10
379 srd 8,7,9
380 or 0,0,8
381 ld 6,0(5)
382 std 0,0(4)
383 sld 0,7,10
384 srd 8,6,9
385 or 0,0,8
386 ld 7,8(5)
387 std 0,8(4)
388 sld 0,6,10
389 srd 8,7,9
390 or 0,0,8
391 ld 6,16(5)
392 std 0,16(4)
393 sld 0,7,10
394 srd 8,6,9
395 or 0,0,8
396 ld 7,24(5)
397 std 0,24(4)
398 addi 5,5,32
399 addi 4,4,32
400 bdnz+ 4b
401 .align 4
402 8:
403 /* calculate and store the final DW */
404 sld 0,6,10
405 srd 8,7,9
406 or 0,0,8
407 std 0,0(4)
408 3:
409 rldicr 0,31,0,60
410 mtcrf 0x01,31
411 bne cr1,.L9 /* If the tail is 0 bytes we are done! */
412 /* Return original dst pointer. */
413 ld 31,-8(1)
414 ld 3,-16(1)
415 blr
416 END_GEN_TB (BP_SYM (memcpy),TB_TOCLESS)
417 libc_hidden_builtin_def (memcpy)