| blob | ea74c117dd1588664946f407f52956feb0dca98b |
1 /* Optimized 64-bit memset implementation for POWER6.
2 Copyright (C) 1997, 1999, 2000, 2002, 2003, 2007
3 Free Software Foundation, Inc.
4 This file is part of the GNU C Library.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, write to the Free
18 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
19 02111-1307 USA. */
21 #include <sysdep.h>
22 #include <bp-sym.h>
23 #include <bp-asm.h>
25 /* __ptr_t [r3] memset (__ptr_t s [r3], int c [r4], size_t n [r5]));
26 Returns 's'.
28 The memset is done in three sizes: byte (8 bits), word (32 bits),
29 cache line (256 bits). There is a special case for setting cache lines
30 to 0, to take advantage of the dcbz instruction. */
32 .machine power6
33 EALIGN (BP_SYM (memset), 7, 0)
34 CALL_MCOUNT 3
36 #define rTMP r0
37 #define rRTN r3 /* Initial value of 1st argument. */
38 #if __BOUNDED_POINTERS__
39 # define rMEMP0 r4 /* Original value of 1st arg. */
40 # define rCHR r5 /* Char to set in each byte. */
41 # define rLEN r6 /* Length of region to set. */
42 # define rMEMP r10 /* Address at which we are storing. */
43 #else
44 # define rMEMP0 r3 /* Original value of 1st arg. */
45 # define rCHR r4 /* Char to set in each byte. */
46 # define rLEN r5 /* Length of region to set. */
47 # define rMEMP r6 /* Address at which we are storing. */
48 #endif
49 #define rALIGN r7 /* Number of bytes we are setting now (when aligning). */
50 #define rMEMP2 r8
51 #define rMEMP3 r9 /* Alt mem pointer. */
52 L(_memset):
53 #if __BOUNDED_POINTERS__
54 cmpldi cr1, rRTN, 0
55 CHECK_BOUNDS_BOTH_WIDE (rMEMP0, rTMP, rTMP2, rLEN)
56 beq cr1, L(b0)
57 STORE_RETURN_VALUE (rMEMP0)
58 STORE_RETURN_BOUNDS (rTMP, rTMP2)
59 L(b0):
60 #endif
61 /* Take care of case for size <= 4. */
62 cmpldi cr1, rLEN, 8
63 andi. rALIGN, rMEMP0, 7
64 mr rMEMP, rMEMP0
65 ble cr1, L(small)
67 /* Align to doubleword boundary. */
68 cmpldi cr5, rLEN, 31
69 rlwimi rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword. */
70 beq+ L(aligned2)
71 mtcrf 0x01, rMEMP0
72 subfic rALIGN, rALIGN, 8
73 cror 28,30,31 /* Detect odd word aligned. */
74 add rMEMP, rMEMP, rALIGN
75 sub rLEN, rLEN, rALIGN
76 rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
77 bt 29, L(g4)
78 /* Process the even word of doubleword. */
79 bf+ 31, L(g2)
80 stb rCHR, 0(rMEMP0)
81 bt 30, L(g4x)
82 L(g2):
83 sth rCHR, -6(rMEMP)
84 L(g4x):
85 stw rCHR, -4(rMEMP)
86 b L(aligned)
87 /* Process the odd word of doubleword. */
88 L(g4):
89 bf 28, L(g4x) /* If false, word aligned on odd word. */
90 bf+ 31, L(g0)
91 stb rCHR, 0(rMEMP0)
92 bt 30, L(aligned)
93 L(g0):
94 sth rCHR, -2(rMEMP)
96 /* Handle the case of size < 31. */
97 L(aligned2):
98 rlwimi rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word. */
99 L(aligned):
100 mtcrf 0x01, rLEN
101 ble cr5, L(medium)
102 /* Align to 32-byte boundary. */
103 andi. rALIGN, rMEMP, 0x18
104 subfic rALIGN, rALIGN, 0x20
105 insrdi rCHR,rCHR,32,0 /* Replicate word to double word. */
106 beq L(caligned)
107 mtcrf 0x01, rALIGN
108 add rMEMP, rMEMP, rALIGN
109 sub rLEN, rLEN, rALIGN
110 cmplwi cr1, rALIGN, 0x10
111 mr rMEMP2, rMEMP
112 bf 28, L(a1)
113 stdu rCHR, -8(rMEMP2)
114 L(a1): blt cr1, L(a2)
115 std rCHR, -8(rMEMP2)
116 stdu rCHR, -16(rMEMP2)
117 L(a2):
119 /* Now aligned to a 32 byte boundary. */
120 .align 4
121 L(caligned):
122 cmpldi cr1, rCHR, 0
123 clrrdi. rALIGN, rLEN, 5
124 mtcrf 0x01, rLEN
125 beq cr1, L(zloopstart) /* Special case for clearing memory using dcbz. */
126 beq L(medium) /* We may not actually get to do a full line. */
127 .align 4
128 /* Storing a non-zero "c" value. We are aligned at a sector (32-byte)
129 boundary may not be at cache line (128-byte) boundary. */
130 L(nzloopstart):
131 /* memset in 32-byte chunks until we get to a cache line boundary.
132 If rLEN is less then the distance to the next cache-line boundary use
133 cacheAligned1 code to finish the tail. */
134 cmpldi cr1,rLEN,128
136 andi. rTMP,rMEMP,127
137 blt cr1,L(cacheAligned1)
138 addi rMEMP3,rMEMP,32
139 beq L(nzCacheAligned)
140 addi rLEN,rLEN,-32
141 std rCHR,0(rMEMP)
142 std rCHR,8(rMEMP)
143 std rCHR,16(rMEMP)
144 addi rMEMP,rMEMP,32
145 andi. rTMP,rMEMP3,127
146 std rCHR,-8(rMEMP3)
148 beq L(nzCacheAligned)
149 addi rLEN,rLEN,-32
150 std rCHR,0(rMEMP3)
151 addi rMEMP,rMEMP,32
152 std rCHR,8(rMEMP3)
153 andi. rTMP,rMEMP,127
154 std rCHR,16(rMEMP3)
155 std rCHR,24(rMEMP3)
157 beq L(nzCacheAligned)
158 addi rLEN,rLEN,-32
159 std rCHR,32(rMEMP3)
160 addi rMEMP,rMEMP,32
161 cmpldi cr1,rLEN,128
162 std rCHR,40(rMEMP3)
163 cmpldi cr6,rLEN,256
164 li rMEMP2,128
165 std rCHR,48(rMEMP3)
166 std rCHR,56(rMEMP3)
167 blt cr1,L(cacheAligned1)
168 b L(nzCacheAligned128)
170 /* Now we are aligned to the cache line and can use dcbtst. */
171 .align 4
172 L(nzCacheAligned):
173 cmpldi cr1,rLEN,128
174 blt cr1,L(cacheAligned1)
175 b L(nzCacheAligned128)
176 .align 5
177 L(nzCacheAligned128):
178 cmpldi cr1,rLEN,256
179 addi rMEMP3,rMEMP,64
180 std rCHR,0(rMEMP)
181 std rCHR,8(rMEMP)
182 std rCHR,16(rMEMP)
183 std rCHR,24(rMEMP)
184 std rCHR,32(rMEMP)
185 std rCHR,40(rMEMP)
186 std rCHR,48(rMEMP)
187 std rCHR,56(rMEMP)
188 addi rMEMP,rMEMP3,64
189 addi rLEN,rLEN,-128
190 std rCHR,0(rMEMP3)
191 std rCHR,8(rMEMP3)
192 std rCHR,16(rMEMP3)
193 std rCHR,24(rMEMP3)
194 std rCHR,32(rMEMP3)
195 std rCHR,40(rMEMP3)
196 std rCHR,48(rMEMP3)
197 std rCHR,56(rMEMP3)
198 bge cr1,L(nzCacheAligned128)
199 dcbtst 0,rMEMP
200 b L(cacheAligned1)
201 .align 5
202 /* Storing a zero "c" value. We are aligned at a sector (32-byte)
203 boundary but may not be at cache line (128-byte) boundary. If the
204 remaining length spans a full cache line we can use the Data cache
205 block zero instruction. */
206 L(zloopstart):
207 /* memset in 32-byte chunks until we get to a cache line boundary.
208 If rLEN is less then the distance to the next cache-line boundary use
209 cacheAligned1 code to finish the tail. */
210 cmpldi cr1,rLEN,128
211 beq L(medium)
212 L(getCacheAligned):
213 andi. rTMP,rMEMP,127
214 nop
215 blt cr1,L(cacheAligned1)
216 addi rMEMP3,rMEMP,32
217 beq L(cacheAligned)
218 addi rLEN,rLEN,-32
219 std rCHR,0(rMEMP)
220 std rCHR,8(rMEMP)
221 std rCHR,16(rMEMP)
222 addi rMEMP,rMEMP,32
223 andi. rTMP,rMEMP3,127
224 std rCHR,-8(rMEMP3)
225 L(getCacheAligned2):
226 beq L(cacheAligned)
227 addi rLEN,rLEN,-32
228 std rCHR,0(rMEMP3)
229 std rCHR,8(rMEMP3)
230 addi rMEMP,rMEMP,32
231 andi. rTMP,rMEMP,127
232 std rCHR,16(rMEMP3)
233 std rCHR,24(rMEMP3)
234 L(getCacheAligned3):
235 beq L(cacheAligned)
236 addi rLEN,rLEN,-32
237 std rCHR,32(rMEMP3)
238 addi rMEMP,rMEMP,32
239 cmpldi cr1,rLEN,128
240 std rCHR,40(rMEMP3)
241 cmpldi cr6,rLEN,256
242 li rMEMP2,128
243 std rCHR,48(rMEMP3)
244 std rCHR,56(rMEMP3)
245 blt cr1,L(cacheAligned1)
246 blt cr6,L(cacheAligned128)
247 b L(cacheAlignedx)
249 /* Now we are aligned to the cache line and can use dcbz. */
250 .align 5
251 L(cacheAligned):
252 cmpldi cr1,rLEN,128
253 cmpldi cr6,rLEN,256
254 blt cr1,L(cacheAligned1)
255 li rMEMP2,128
256 L(cacheAlignedx):
257 cmpldi cr5,rLEN,640
258 blt cr6,L(cacheAligned128)
259 bgt cr5,L(cacheAligned512)
260 cmpldi cr6,rLEN,512
261 dcbz 0,rMEMP
262 cmpldi cr1,rLEN,384
263 dcbz rMEMP2,rMEMP
264 addi rMEMP,rMEMP,256
265 addi rLEN,rLEN,-256
266 blt cr1,L(cacheAligned1)
267 blt cr6,L(cacheAligned128)
268 b L(cacheAligned256)
269 .align 5
270 /* A simple loop for the longer (>640 bytes) lengths. This form limits
271 the branch miss-predicted to exactly 1 at loop exit.*/
272 L(cacheAligned512):
273 cmpli cr1,rLEN,128
274 blt cr1,L(cacheAligned1)
275 dcbz 0,rMEMP
276 addi rLEN,rLEN,-128
277 addi rMEMP,rMEMP,128
278 b L(cacheAligned512)
279 .align 5
280 L(cacheAligned256):
282 cmpldi cr6,rLEN,512
284 dcbz 0,rMEMP
285 cmpldi cr1,rLEN,384
286 dcbz rMEMP2,rMEMP
287 addi rMEMP,rMEMP,256
288 addi rLEN,rLEN,-256
290 bge cr6,L(cacheAligned256)
292 blt cr1,L(cacheAligned1)
293 .align 4
294 L(cacheAligned128):
295 dcbz 0,rMEMP
296 addi rMEMP,rMEMP,128
297 addi rLEN,rLEN,-128
298 nop
299 L(cacheAligned1):
300 cmpldi cr1,rLEN,32
301 blt cr1,L(handletail32)
302 addi rMEMP3,rMEMP,32
303 addi rLEN,rLEN,-32
304 std rCHR,0(rMEMP)
305 std rCHR,8(rMEMP)
306 std rCHR,16(rMEMP)
307 addi rMEMP,rMEMP,32
308 cmpldi cr1,rLEN,32
309 std rCHR,-8(rMEMP3)
310 L(cacheAligned2):
311 blt cr1,L(handletail32)
312 addi rLEN,rLEN,-32
313 std rCHR,0(rMEMP3)
314 std rCHR,8(rMEMP3)
315 addi rMEMP,rMEMP,32
316 cmpldi cr1,rLEN,32
317 std rCHR,16(rMEMP3)
318 std rCHR,24(rMEMP3)
319 nop
320 L(cacheAligned3):
321 blt cr1,L(handletail32)
322 addi rMEMP,rMEMP,32
323 addi rLEN,rLEN,-32
324 std rCHR,32(rMEMP3)
325 std rCHR,40(rMEMP3)
326 std rCHR,48(rMEMP3)
327 std rCHR,56(rMEMP3)
329 /* We are here because the length or remainder (rLEN) is less than the
330 cache line/sector size and does not justify aggressive loop unrolling.
331 So set up the preconditions for L(medium) and go there. */
332 .align 3
333 L(handletail32):
334 cmpldi cr1,rLEN,0
335 beqlr cr1
336 b L(medium)
338 .align 5
339 L(small):
340 /* Memset of 8 bytes or less. */
341 cmpldi cr6, rLEN, 4
342 cmpldi cr5, rLEN, 1
343 ble cr6,L(le4)
344 subi rLEN, rLEN, 4
345 stb rCHR,0(rMEMP)
346 stb rCHR,1(rMEMP)
347 stb rCHR,2(rMEMP)
348 stb rCHR,3(rMEMP)
349 addi rMEMP,rMEMP, 4
350 cmpldi cr5, rLEN, 1
351 L(le4):
352 cmpldi cr1, rLEN, 3
353 bltlr cr5
354 stb rCHR, 0(rMEMP)
355 beqlr cr5
356 stb rCHR, 1(rMEMP)
357 bltlr cr1
358 stb rCHR, 2(rMEMP)
359 beqlr cr1
360 stb rCHR, 3(rMEMP)
361 blr
363 /* Memset of 0-31 bytes. */
364 .align 5
365 L(medium):
366 insrdi rCHR,rCHR,32,0 /* Replicate word to double word. */
367 cmpldi cr1, rLEN, 16
368 L(medium_tail2):
369 add rMEMP, rMEMP, rLEN
370 L(medium_tail):
371 bt- 31, L(medium_31t)
372 bt- 30, L(medium_30t)
373 L(medium_30f):
374 bt 29, L(medium_29t)
375 L(medium_29f):
376 bge cr1, L(medium_27t)
377 bflr 28
378 std rCHR, -8(rMEMP)
379 blr
381 L(medium_31t):
382 stbu rCHR, -1(rMEMP)
383 bf- 30, L(medium_30f)
384 L(medium_30t):
385 sthu rCHR, -2(rMEMP)
386 bf- 29, L(medium_29f)
387 L(medium_29t):
388 stwu rCHR, -4(rMEMP)
389 blt cr1, L(medium_27f)
390 L(medium_27t):
391 std rCHR, -8(rMEMP)
392 stdu rCHR, -16(rMEMP)
393 L(medium_27f):
394 bflr 28
395 L(medium_28t):
396 std rCHR, -8(rMEMP)
397 blr
398 END_GEN_TB (BP_SYM (memset),TB_TOCLESS)
399 libc_hidden_builtin_def (memset)
401 /* Copied from bzero.S to prevent the linker from inserting a stub
402 between bzero and memset. */
403 ENTRY (BP_SYM (__bzero))
404 CALL_MCOUNT 3
405 #if __BOUNDED_POINTERS__
406 mr r6,r4
407 li r5,0
408 mr r4,r3
409 /* Tell memset that we don't want a return value. */
410 li r3,0
411 b L(_memset)
412 #else
413 mr r5,r4
414 li r4,0
415 b L(_memset)
416 #endif
417 END_GEN_TB (BP_SYM (__bzero),TB_TOCLESS)
419 weak_alias (BP_SYM (__bzero), BP_SYM (bzero))