| blob | 9df5bb42b6b4f9cdd6f02e4756eb59db0bc6d4f9 |
1 /* Optimized memcpy implementation for PowerPC64.
2 Copyright (C) 2003 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
18 02111-1307 USA. */
20 #include <sysdep.h>
21 #include <bp-sym.h>
22 #include <bp-asm.h>
24 /* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]);
25 Returns 'dst'.
27 Memcpy handles short copies (< 32-bytes) using a binary move blocks
28 (no loops) of lwz/stw. The tail (remaining 1-3) bytes is handled
29 with the appropriate combination of byte and halfword load/stores.
30 There is minimal effort to optimize the alignment of short moves.
31 The 64-bit implementations of POWER3 and POWER4 do a reasonable job
32 of handling unligned load/stores that do not cross 32-byte boundries.
34 Longer moves (>= 32-bytes) justify the effort to get at least the
35 destination doubleword (8-byte) aligned. Further optimization is
36 posible when both source and destination are doubleword aligned.
37 Each case has a optimized unrolled loop. */
39 EALIGN (BP_SYM (memcpy), 5, 0)
40 CALL_MCOUNT 3
42 cmpldi cr1,5,31
43 neg 0,3
44 std 3,-16(1)
45 std 31,-8(1)
46 andi. 11,3,7 /* check alignement of dst. */
47 clrldi 0,0,61 /* Number of bytes until the 1st doubleword of dst. */
48 clrldi 10,4,61 /* check alignement of src. */
49 cmpldi cr6,5,8
50 ble- cr1,.L2 /* If move < 32 bytes use short move code. */
51 cmpld cr6,10,11
52 mr 12,4
53 srdi 9,5,3 /* Number of full double words remaining. */
54 mtcrf 0x01,0
55 mr 31,5
56 beq .L0
58 subf 31,0,5
59 /* Move 0-7 bytes as needed to get the destination doubleword alligned. */
60 1: bf 31,2f
61 lbz 6,0(12)
62 addi 12,12,1
63 stb 6,0(3)
64 addi 3,3,1
65 2: bf 30,4f
66 lhz 6,0(12)
67 addi 12,12,2
68 sth 6,0(3)
69 addi 3,3,2
70 4: bf 29,0f
71 lwz 6,0(12)
72 addi 12,12,4
73 stw 6,0(3)
74 addi 3,3,4
75 0:
76 clrldi 10,12,61 /* check alignement of src again. */
77 srdi 9,31,3 /* Number of full double words remaining. */
79 /* Copy doublewords from source to destination, assumpting the
80 destination is aligned on a doubleword boundary.
82 At this point we know there are at least 25 bytes left (32-7) to copy.
83 The next step is to determine if the source is also doubleword aligned.
84 If not branch to the unaligned move code at .L6. which uses
85 a load, shift, store strategy.
87 Otherwise source and destination are doubleword aligned, and we can
88 the optimized doubleword copy loop. */
89 .L0:
90 clrldi 11,31,61
91 mtcrf 0x01,9
92 bne- cr6,.L6 /* If source is not DW aligned. */
94 /* Move doublewords where destination and source are DW aligned.
95 Use a unrolled loop to copy 4 doubleword (32-bytes) per iteration.
96 If the the copy is not an exact multiple of 32 bytes, 1-3
97 doublewords are copied as needed to set up the main loop. After
98 the main loop exits there may be a tail of 1-7 bytes. These byte are
99 copied a word/halfword/byte at a time as needed to preserve alignment. */
101 srdi 8,31,5
102 cmpldi cr1,9,4
103 cmpldi cr6,11,0
104 mr 11,12
106 bf 30,1f
107 ld 6,0(12)
108 ld 7,8(12)
109 addi 11,12,16
110 mtctr 8
111 std 6,0(3)
112 std 7,8(3)
113 addi 10,3,16
114 bf 31,4f
115 ld 0,16(12)
116 std 0,16(3)
117 blt cr1,3f
118 addi 11,12,24
119 addi 10,3,24
120 b 4f
121 .align 4
122 1:
123 mr 10,3
124 mtctr 8
125 bf 31,4f
126 ld 6,0(12)
127 addi 11,12,8
128 std 6,0(3)
129 addi 10,3,8
131 .align 4
132 4:
133 ld 6,0(11)
134 ld 7,8(11)
135 ld 8,16(11)
136 ld 0,24(11)
137 addi 11,11,32
138 2:
139 std 6,0(10)
140 std 7,8(10)
141 std 8,16(10)
142 std 0,24(10)
143 addi 10,10,32
144 bdnz 4b
145 3:
147 rldicr 0,31,0,60
148 mtcrf 0x01,31
149 beq cr6,0f
150 .L9:
151 add 3,3,0
152 add 12,12,0
154 /* At this point we have a tail of 0-7 bytes and we know that the
155 destiniation is double word aligned. */
156 4: bf 29,2f
157 lwz 6,0(12)
158 addi 12,12,4
159 stw 6,0(3)
160 addi 3,3,4
161 2: bf 30,1f
162 lhz 6,0(12)
163 addi 12,12,2
164 sth 6,0(3)
165 addi 3,3,2
166 1: bf 31,0f
167 lbz 6,0(12)
168 stb 6,0(3)
169 0:
170 /* Return original dst pointer. */
171 ld 31,-8(1)
172 ld 3,-16(1)
173 blr
175 /* Copy up to 31 bytes. This divided into two cases 0-8 bytes and 9-31
176 bytes. Each case is handled without loops, using binary (1,2,4,8)
177 tests.
179 In the short (0-8 byte) case no attempt is made to force alignment
180 of either source or destination. The hardware will handle the
181 unaligned load/stores with small delays for crossing 32- 64-byte, and
182 4096-byte boundaries. Since these short moves are unlikely to be
183 unaligned or cross these boundaries, the overhead to force
184 alignment is not justified.
186 The longer (9-31 byte) move is more likely to cross 32- or 64-byte
187 boundaries. Since only loads are sensitive to the 32-/64-byte
188 boundaries it is more important to align the source then the
189 destination. If the source is not already word aligned, we first
190 move 1-3 bytes as needed. Since we are only word aligned we don't
191 use double word load/stores to insure that all loads are aligned.
192 While the destination and stores may still be unaligned, this
193 is only an issue for page (4096 byte boundary) crossing, which
194 should be rare for these short moves. The hardware handles this
195 case automatically with a small delay. */
197 .align 4
198 .L2:
199 mtcrf 0x01,5
200 neg 8,4
201 clrrdi 11,4,2
202 andi. 0,8,3
203 ble cr6,.LE8 /* Handle moves of 0-8 bytes. */
204 /* At least 9 bytes left. Get the source word aligned. */
205 cmpldi cr1,5,16
206 mr 10,5
207 mr 12,4
208 cmpldi cr6,0,2
209 beq .L3 /* If the source is already word aligned skip this. */
210 /* Copy 1-3 bytes to get source address word aligned. */
211 lwz 6,0(11)
212 subf 10,0,5
213 add 12,4,0
214 blt cr6,5f
215 srdi 7,6,16
216 bgt cr6,3f
217 sth 6,0(3)
218 b 7f
219 .align 4
220 3:
221 stb 7,0(3)
222 sth 6,1(3)
223 b 7f
224 .align 4
225 5:
226 stb 6,0(3)
227 7:
228 cmpldi cr1,10,16
229 add 3,3,0
230 mtcrf 0x01,10
231 .align 4
232 .L3:
233 /* At least 6 bytes left and the source is word aligned. */
234 blt cr1,8f
235 16: /* Move 16 bytes. */
236 lwz 6,0(12)
237 lwz 7,4(12)
238 stw 6,0(3)
239 lwz 6,8(12)
240 stw 7,4(3)
241 lwz 7,12(12)
242 addi 12,12,16
243 stw 6,8(3)
244 stw 7,12(3)
245 addi 3,3,16
246 8: /* Move 8 bytes. */
247 bf 28,4f
248 lwz 6,0(12)
249 lwz 7,4(12)
250 addi 12,12,8
251 stw 6,0(3)
252 stw 7,4(3)
253 addi 3,3,8
254 4: /* Move 4 bytes. */
255 bf 29,2f
256 lwz 6,0(12)
257 addi 12,12,4
258 stw 6,0(3)
259 addi 3,3,4
260 2: /* Move 2-3 bytes. */
261 bf 30,1f
262 lhz 6,0(12)
263 sth 6,0(3)
264 bf 31,0f
265 lbz 7,2(12)
266 stb 7,2(3)
267 ld 3,-16(1)
268 blr
269 1: /* Move 1 byte. */
270 bf 31,0f
271 lbz 6,0(12)
272 stb 6,0(3)
273 0:
274 /* Return original dst pointer. */
275 ld 3,-16(1)
276 blr
278 /* Special case to copy 0-8 bytes. */
279 .align 4
280 .LE8:
281 mr 12,4
282 bne cr6,4f
283 /* Would have liked to use use ld/std here but the 630 processors are
284 slow for load/store doubles that are not at least word aligned.
285 Unaligned Load/Store word execute with only a 1 cycle penaltity. */
286 lwz 6,0(4)
287 lwz 7,4(4)
288 stw 6,0(3)
289 stw 7,4(3)
290 /* Return original dst pointer. */
291 ld 3,-16(1)
292 blr
293 .align 4
294 4: bf 29,2b
295 lwz 6,0(4)
296 stw 6,0(3)
297 6:
298 bf 30,5f
299 lhz 7,4(4)
300 sth 7,4(3)
301 bf 31,0f
302 lbz 8,6(4)
303 stb 8,6(3)
304 ld 3,-16(1)
305 blr
306 .align 4
307 5:
308 bf 31,0f
309 lbz 6,4(4)
310 stb 6,4(3)
311 .align 4
312 0:
313 /* Return original dst pointer. */
314 ld 3,-16(1)
315 blr
317 .align 4
318 .L6:
320 /* Copy doublewords where the destination is aligned but the source is
321 not. Use aligned doubleword loads from the source, shifted to realign
322 the data, to allow aligned destination stores. */
323 subf 5,10,12
324 andi. 0,9,1
325 cmpldi cr6,11,0
326 sldi 10,10,3
327 mr 11,9
328 mr 4,3
329 ld 6,0(5)
330 ld 7,8(5)
331 subfic 9,10,64
332 beq 2f
333 sld 0,6,10
334 cmpldi 11,1
335 mr 6,7
336 addi 4,4,-8
337 addi 11,11,-1
338 b 1f
339 2: addi 5,5,8
340 .align 4
341 0: sld 0,6,10
342 srd 8,7,9
343 cmpldi 11,2
344 ld 6,8(5)
345 or 0,0,8
346 addi 11,11,-2
347 std 0,0(4)
348 sld 0,7,10
349 1: srd 8,6,9
350 or 0,0,8
351 beq 8f
352 ld 7,16(5)
353 std 0,8(4)
354 addi 5,5,16
355 addi 4,4,16
356 b 0b
357 .align 4
358 8:
359 std 0,8(4)
360 rldicr 0,31,0,60
361 mtcrf 0x01,31
362 bne cr6,.L9 /* If the tail is 0 bytes we are done! */
363 /* Return original dst pointer. */
364 ld 31,-8(1)
365 ld 3,-16(1)
366 blr
367 END_GEN_TB (BP_SYM (memcpy),TB_TOCLESS)
368 libc_hidden_builtin_def (memcpy)