| blob | 3353000186e9d5baa0430616dff841ddf355a328 |
1 /* Optimized version of the standard memset() function.
2 This file is part of the GNU C Library.
3 Copyright (C) 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
4 Contributed by Dan Pop for Itanium <Dan.Pop@cern.ch>.
5 Rewritten for McKinley by Sverre Jarp, HP Labs/CERN <Sverre.Jarp@cern.ch>
7 The GNU C Library is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public
9 License as published by the Free Software Foundation; either
10 version 2.1 of the License, or (at your option) any later version.
12 The GNU C Library is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
17 You should have received a copy of the GNU Lesser General Public
18 License along with the GNU C Library; if not, write to the Free
19 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
20 02111-1307 USA. */
22 /* Return: dest
24 Inputs:
25 in0: dest
26 in1: value
27 in2: count
29 The algorithm is fairly straightforward: set byte by byte until we
30 we get to a 16B-aligned address, then loop on 128 B chunks using an
31 early store as prefetching, then loop on 32B chucks, then clear remaining
32 words, finally clear remaining bytes.
33 Since a stf.spill f0 can store 16B in one go, we use this instruction
34 to get peak speed when value = 0. */
36 #include <sysdep.h>
37 #undef ret
39 #define dest in0
40 #define value in1
41 #define cnt in2
43 #define tmp r31
44 #define save_lc r30
45 #define ptr0 r29
46 #define ptr1 r28
47 #define ptr2 r27
48 #define ptr3 r26
49 #define ptr9 r24
50 #define loopcnt r23
51 #define linecnt r22
52 #define bytecnt r21
54 #define fvalue f6
56 // This routine uses only scratch predicate registers (p6 - p15)
57 #define p_scr p6 // default register for same-cycle branches
58 #define p_nz p7
59 #define p_zr p8
60 #define p_unalgn p9
61 #define p_y p11
62 #define p_n p12
63 #define p_yy p13
64 #define p_nn p14
66 #define movi0 mov
68 #define MIN1 15
69 #define MIN1P1HALF 8
70 #define LINE_SIZE 128
71 #define LSIZE_SH 7 // shift amount
72 #define PREF_AHEAD 8
74 #define USE_FLP
75 #if defined(USE_INT)
76 #define store st8
77 #define myval value
78 #elif defined(USE_FLP)
79 #define store stf8
80 #define myval fvalue
81 #endif
83 .align 64
84 ENTRY(memset)
85 { .mmi
86 .prologue
87 alloc tmp = ar.pfs, 3, 0, 0, 0
88 lfetch.nt1 [dest]
89 .save ar.lc, save_lc
90 movi0 save_lc = ar.lc
91 } { .mmi
92 .body
93 mov ret0 = dest // return value
94 cmp.ne p_nz, p_zr = value, r0 // use stf.spill if value is zero
95 cmp.eq p_scr, p0 = cnt, r0
96 ;; }
97 { .mmi
98 and ptr2 = -(MIN1+1), dest // aligned address
99 and tmp = MIN1, dest // prepare to check for alignment
100 tbit.nz p_y, p_n = dest, 0 // Do we have an odd address? (M_B_U)
101 } { .mib
102 mov ptr1 = dest
103 mux1 value = value, @brcst // create 8 identical bytes in word
104 (p_scr) br.ret.dpnt.many rp // return immediately if count = 0
105 ;; }
106 { .mib
107 cmp.ne p_unalgn, p0 = tmp, r0
108 } { .mib // NB: # of bytes to move is 1 higher
109 sub bytecnt = (MIN1+1), tmp // than loopcnt
110 cmp.gt p_scr, p0 = 16, cnt // is it a minimalistic task?
111 (p_scr) br.cond.dptk.many .move_bytes_unaligned // go move just a few (M_B_U)
112 ;; }
113 { .mmi
114 (p_unalgn) add ptr1 = (MIN1+1), ptr2 // after alignment
115 (p_unalgn) add ptr2 = MIN1P1HALF, ptr2 // after alignment
116 (p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 3 // should we do a st8 ?
117 ;; }
118 { .mib
119 (p_y) add cnt = -8, cnt
120 (p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 2 // should we do a st4 ?
121 } { .mib
122 (p_y) st8 [ptr2] = value, -4
123 (p_n) add ptr2 = 4, ptr2
124 ;; }
125 { .mib
126 (p_yy) add cnt = -4, cnt
127 (p_unalgn) tbit.nz.unc p_y, p_n = bytecnt, 1 // should we do a st2 ?
128 } { .mib
129 (p_yy) st4 [ptr2] = value, -2
130 (p_nn) add ptr2 = 2, ptr2
131 ;; }
132 { .mmi
133 mov tmp = LINE_SIZE+1 // for compare
134 (p_y) add cnt = -2, cnt
135 (p_unalgn) tbit.nz.unc p_yy, p_nn = bytecnt, 0 // should we do a st1 ?
136 } { .mmi
137 setf.sig fvalue=value // transfer value to FLP side
138 (p_y) st2 [ptr2] = value, -1
139 (p_n) add ptr2 = 1, ptr2
140 ;; }
142 { .mmi
143 (p_yy) st1 [ptr2] = value
144 cmp.gt p_scr, p0 = tmp, cnt // is it a minimalistic task?
145 } { .mbb
146 (p_yy) add cnt = -1, cnt
147 (p_scr) br.cond.dpnt.many .fraction_of_line // go move just a few
148 ;; }
150 { .mib
151 nop.m 0
152 shr.u linecnt = cnt, LSIZE_SH
153 (p_zr) br.cond.dptk.many .l1b // Jump to use stf.spill
154 ;; }
156 .align 32 // -------- // L1A: store ahead into cache lines; fill later
157 { .mmi
158 and tmp = -(LINE_SIZE), cnt // compute end of range
159 mov ptr9 = ptr1 // used for prefetching
160 and cnt = (LINE_SIZE-1), cnt // remainder
161 } { .mmi
162 mov loopcnt = PREF_AHEAD-1 // default prefetch loop
163 cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value
164 ;; }
165 { .mmi
166 (p_scr) add loopcnt = -1, linecnt // start of stores
167 add ptr2 = 8, ptr1 // (beyond prefetch stores)
168 add ptr1 = tmp, ptr1 // first address beyond total
169 ;; } // range
170 { .mmi
171 add tmp = -1, linecnt // next loop count
172 movi0 ar.lc = loopcnt
173 ;; }
174 .pref_l1a:
175 { .mib
176 store [ptr9] = myval, 128 // Do stores one cache line apart
177 nop.i 0
178 br.cloop.dptk.few .pref_l1a
179 ;; }
180 { .mmi
181 add ptr0 = 16, ptr2 // Two stores in parallel
182 movi0 ar.lc = tmp
183 ;; }
184 .l1ax:
185 { .mmi
186 store [ptr2] = myval, 8
187 store [ptr0] = myval, 8
188 ;; }
189 { .mmi
190 store [ptr2] = myval, 24
191 store [ptr0] = myval, 24
192 ;; }
193 { .mmi
194 store [ptr2] = myval, 8
195 store [ptr0] = myval, 8
196 ;; }
197 { .mmi
198 store [ptr2] = myval, 24
199 store [ptr0] = myval, 24
200 ;; }
201 { .mmi
202 store [ptr2] = myval, 8
203 store [ptr0] = myval, 8
204 ;; }
205 { .mmi
206 store [ptr2] = myval, 24
207 store [ptr0] = myval, 24
208 ;; }
209 { .mmi
210 store [ptr2] = myval, 8
211 store [ptr0] = myval, 32
212 cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching?
213 ;; }
214 { .mmb
215 store [ptr2] = myval, 24
216 (p_scr) store [ptr9] = myval, 128
217 br.cloop.dptk.few .l1ax
218 ;; }
219 { .mbb
220 cmp.le p_scr, p0 = 8, cnt // just a few bytes left ?
221 (p_scr) br.cond.dpnt.many .fraction_of_line // Branch no. 2
222 br.cond.dpnt.many .move_bytes_from_alignment // Branch no. 3
223 ;; }
225 .align 32
226 .l1b: // ------------------ // L1B: store ahead into cache lines; fill later
227 { .mmi
228 and tmp = -(LINE_SIZE), cnt // compute end of range
229 mov ptr9 = ptr1 // used for prefetching
230 and cnt = (LINE_SIZE-1), cnt // remainder
231 } { .mmi
232 mov loopcnt = PREF_AHEAD-1 // default prefetch loop
233 cmp.gt p_scr, p0 = PREF_AHEAD, linecnt // check against actual value
234 ;; }
235 { .mmi
236 (p_scr) add loopcnt = -1, linecnt
237 add ptr2 = 16, ptr1 // start of stores (beyond prefetch stores)
238 add ptr1 = tmp, ptr1 // first address beyond total range
239 ;; }
240 { .mmi
241 add tmp = -1, linecnt // next loop count
242 movi0 ar.lc = loopcnt
243 ;; }
244 .pref_l1b:
245 { .mib
246 stf.spill [ptr9] = f0, 128 // Do stores one cache line apart
247 nop.i 0
248 br.cloop.dptk.few .pref_l1b
249 ;; }
250 { .mmi
251 add ptr0 = 16, ptr2 // Two stores in parallel
252 movi0 ar.lc = tmp
253 ;; }
254 .l1bx:
255 { .mmi
256 stf.spill [ptr2] = f0, 32
257 stf.spill [ptr0] = f0, 32
258 ;; }
259 { .mmi
260 stf.spill [ptr2] = f0, 32
261 stf.spill [ptr0] = f0, 32
262 ;; }
263 { .mmi
264 stf.spill [ptr2] = f0, 32
265 stf.spill [ptr0] = f0, 64
266 cmp.lt p_scr, p0 = ptr9, ptr1 // do we need more prefetching?
267 ;; }
268 { .mmb
269 stf.spill [ptr2] = f0, 32
270 (p_scr) stf.spill [ptr9] = f0, 128
271 br.cloop.dptk.few .l1bx
272 ;; }
273 { .mib
274 cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
275 (p_scr) br.cond.dpnt.many .move_bytes_from_alignment
276 ;; }
278 .fraction_of_line:
279 { .mib
280 add ptr2 = 16, ptr1
281 shr.u loopcnt = cnt, 5 // loopcnt = cnt / 32
282 ;; }
283 { .mib
284 cmp.eq p_scr, p0 = loopcnt, r0
285 add loopcnt = -1, loopcnt
286 (p_scr) br.cond.dpnt.many .store_words
287 ;; }
288 { .mib
289 and cnt = 0x1f, cnt // compute the remaining cnt
290 movi0 ar.lc = loopcnt
291 ;; }
292 .align 32
293 .l2: // ---------------------------- // L2A: store 32B in 2 cycles
294 { .mmb
295 store [ptr1] = myval, 8
296 store [ptr2] = myval, 8
297 ;; } { .mmb
298 store [ptr1] = myval, 24
299 store [ptr2] = myval, 24
300 br.cloop.dptk.many .l2
301 ;; }
302 .store_words:
303 { .mib
304 cmp.gt p_scr, p0 = 8, cnt // just a few bytes left ?
305 (p_scr) br.cond.dpnt.many .move_bytes_from_alignment // Branch
306 ;; }
308 { .mmi
309 store [ptr1] = myval, 8 // store
310 cmp.le p_y, p_n = 16, cnt //
311 add cnt = -8, cnt // subtract
312 ;; }
313 { .mmi
314 (p_y) store [ptr1] = myval, 8 // store
315 (p_y) cmp.le.unc p_yy, p_nn = 16, cnt //
316 (p_y) add cnt = -8, cnt // subtract
317 ;; }
318 { .mmi // store
319 (p_yy) store [ptr1] = myval, 8 //
320 (p_yy) add cnt = -8, cnt // subtract
321 ;; }
323 .move_bytes_from_alignment:
324 { .mib
325 cmp.eq p_scr, p0 = cnt, r0
326 tbit.nz.unc p_y, p0 = cnt, 2 // should we terminate with a st4 ?
327 (p_scr) br.cond.dpnt.few .restore_and_exit
328 ;; }
329 { .mib
330 (p_y) st4 [ptr1] = value, 4
331 tbit.nz.unc p_yy, p0 = cnt, 1 // should we terminate with a st2 ?
332 ;; }
333 { .mib
334 (p_yy) st2 [ptr1] = value, 2
335 tbit.nz.unc p_y, p0 = cnt, 0
336 ;; }
338 { .mib
339 (p_y) st1 [ptr1] = value
340 ;; }
341 .restore_and_exit:
342 { .mib
343 nop.m 0
344 movi0 ar.lc = save_lc
345 br.ret.sptk.many rp
346 ;; }
348 .move_bytes_unaligned:
349 { .mmi
350 .pred.rel "mutex",p_y, p_n
351 .pred.rel "mutex",p_yy, p_nn
352 (p_n) cmp.le p_yy, p_nn = 4, cnt
353 (p_y) cmp.le p_yy, p_nn = 5, cnt
354 (p_n) add ptr2 = 2, ptr1
355 } { .mmi
356 (p_y) add ptr2 = 3, ptr1
357 (p_y) st1 [ptr1] = value, 1 // fill 1 (odd-aligned) byte
358 (p_y) add cnt = -1, cnt // [15, 14 (or less) left]
359 ;; }
360 { .mmi
361 (p_yy) cmp.le.unc p_y, p0 = 8, cnt
362 add ptr3 = ptr1, cnt // prepare last store
363 movi0 ar.lc = save_lc
364 } { .mmi
365 (p_yy) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
366 (p_yy) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
367 (p_yy) add cnt = -4, cnt // [11, 10 (o less) left]
368 ;; }
369 { .mmi
370 (p_y) cmp.le.unc p_yy, p0 = 8, cnt
371 add ptr3 = -1, ptr3 // last store
372 tbit.nz p_scr, p0 = cnt, 1 // will there be a st2 at the end ?
373 } { .mmi
374 (p_y) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
375 (p_y) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
376 (p_y) add cnt = -4, cnt // [7, 6 (or less) left]
377 ;; }
378 { .mmi
379 (p_yy) st2 [ptr1] = value, 4 // fill 2 (aligned) bytes
380 (p_yy) st2 [ptr2] = value, 4 // fill 2 (aligned) bytes
381 // [3, 2 (or less) left]
382 tbit.nz p_y, p0 = cnt, 0 // will there be a st1 at the end ?
383 } { .mmi
384 (p_yy) add cnt = -4, cnt
385 ;; }
386 { .mmb
387 (p_scr) st2 [ptr1] = value // fill 2 (aligned) bytes
388 (p_y) st1 [ptr3] = value // fill last byte (using ptr3)
389 br.ret.sptk.many rp
390 ;; }
391 END(memset)
392 libc_hidden_builtin_def (memset)