1 /* $Id: VIScsum.S,v 1.7 2002/02/09 19:49:30 davem Exp $
2 * VIScsum.S: High bandwidth IP checksumming utilizing the UltraSparc
3 * Visual Instruction Set.
5 * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
6 * Copyright (C) 2000 David S. Miller (davem@redhat.com)
8 * Based on older sparc32/sparc64 checksum.S, which is:
10 * Copyright(C) 1995 Linus Torvalds
11 * Copyright(C) 1995 Miguel de Icaza
12 * Copyright(C) 1996, 1997 David S. Miller
14 * Linux/Alpha checksum c-code
15 * Linux/ix86 inline checksum assembly
16 * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code)
17 * David Mosberger-Tang for optimized reference c-code
18 * BSD4.4 portable checksum routine
30 #include <asm/visasm.h>
31 #include <asm/thread_info.h>
33 #define ASI_BLK_P 0xf0
37 /* Dobrou noc, SunSoft engineers. Spete sladce.
38 * This has a couple of tricks in and those
39 * tricks are UltraLinux trade secrets :))
42 #define START_THE_TRICK(fz,f0,f2,f4,f6,f8,f10) \
43 fcmpgt32 %fz, %f0, %g1 /* FPM Group */; \
44 fcmpgt32 %fz, %f2, %g2 /* FPM Group */; \
45 fcmpgt32 %fz, %f4, %g3 /* FPM Group */; \
46 inc %g1 /* IEU0 Group */; \
47 fcmpgt32 %fz, %f6, %g5 /* FPM */; \
48 srl %g1, 1, %g1 /* IEU0 Group */; \
49 fcmpgt32 %fz, %f8, %g7 /* FPM */; \
50 inc %g2 /* IEU0 Group */; \
51 fcmpgt32 %fz, %f10, %o3 /* FPM */; \
52 srl %g2, 1, %g2 /* IEU0 Group */; \
54 srl %g3, 1, %g3 /* IEU0 Group */; \
55 add %o2, %g1, %o2 /* IEU1 */; \
56 add %o2, %g2, %o2 /* IEU0 Group */; \
58 add %o2, %g3, %o2 /* IEU0 Group */;
60 #define DO_THE_TRICK(O12,O14,f0,f2,f4,f6,f8,f10,f12,f14,F0,F2,F4,F6,F8,F10,F12,F14) \
61 srl %g5, 1, %g5 /* IEU0 Group */; \
62 fpadd32 %F0, %f0, %F0 /* FPA */; \
63 fcmpgt32 %O12, %f12, %o4 /* FPM */; \
64 inc %g7 /* IEU0 Group */; \
65 fpadd32 %F2, %f2, %F2 /* FPA */; \
66 fcmpgt32 %O14, %f14, %o5 /* FPM */; \
67 add %o2, %g5, %o2 /* IEU1 Group */; \
68 fpadd32 %F4, %f4, %F4 /* FPA */; \
69 fcmpgt32 %f0, %F0, %g1 /* FPM */; \
70 srl %g7, 1, %g7 /* IEU0 Group */; \
71 fpadd32 %F6, %f6, %F6 /* FPA */; \
72 fcmpgt32 %f2, %F2, %g2 /* FPM */; \
73 add %o2, %g7, %o2 /* IEU0 Group */; \
74 fpadd32 %F8, %f8, %F8 /* FPA */; \
75 fcmpgt32 %f4, %F4, %g3 /* FPM */; \
76 inc %o3 /* IEU0 Group */; \
77 fpadd32 %F10, %f10, %F10 /* FPA */; \
78 fcmpgt32 %f6, %F6, %g5 /* FPM */; \
79 srl %o3, 1, %o3 /* IEU0 Group */; \
80 fpadd32 %F12, %f12, %F12 /* FPA */; \
81 fcmpgt32 %f8, %F8, %g7 /* FPM */; \
82 add %o2, %o3, %o2 /* IEU0 Group */; \
83 fpadd32 %F14, %f14, %F14 /* FPA */; \
84 fcmpgt32 %f10, %F10, %o3 /* FPM */; \
85 inc %o4 /* IEU0 Group */; \
87 srl %o4, 1, %o4 /* IEU0 Group */; \
89 srl %o5, 1, %o5 /* IEU0 Group */; \
90 add %o2, %o4, %o2 /* IEU1 */; \
91 srl %g1, 1, %g1 /* IEU0 Group */; \
92 add %o2, %o5, %o2 /* IEU1 */; \
93 inc %g2 /* IEU0 Group */; \
94 add %o2, %g1, %o2 /* IEU1 */; \
95 srl %g2, 1, %g2 /* IEU0 Group */; \
97 srl %g3, 1, %g3 /* IEU0 Group */; \
98 add %o2, %g2, %o2 /* IEU1 */; \
99 inc %g5 /* IEU0 Group */; \
100 add %o2, %g3, %o2 /* IEU0 */;
102 #define END_THE_TRICK(O12,O14,f0,f2,f4,f6,f8,f10,f12,f14,S0,S1,S2,S3,T0,T1,U0,fz) \
103 srl %g5, 1, %g5 /* IEU0 Group */; \
104 fpadd32 %f2, %f0, %S0 /* FPA */; \
105 fcmpgt32 %O12, %f12, %o4 /* FPM */; \
106 inc %g7 /* IEU0 Group */; \
107 fpadd32 %f6, %f4, %S1 /* FPA */; \
108 fcmpgt32 %O14, %f14, %o5 /* FPM */; \
109 srl %g7, 1, %g7 /* IEU0 Group */; \
110 fpadd32 %f10, %f8, %S2 /* FPA */; \
111 fcmpgt32 %f0, %S0, %g1 /* FPM */; \
112 inc %o3 /* IEU0 Group */; \
113 fpadd32 %f14, %f12, %S3 /* FPA */; \
114 fcmpgt32 %f4, %S1, %g2 /* FPM */; \
115 add %o2, %g5, %o2 /* IEU0 Group */; \
116 fpadd32 %S0, %S1, %T0 /* FPA */; \
117 fcmpgt32 %f8, %S2, %g3 /* FPM */; \
118 add %o2, %g7, %o2 /* IEU0 Group */; \
119 fzero %fz /* FPA */; \
120 fcmpgt32 %f12, %S3, %g5 /* FPM */; \
121 srl %o3, 1, %o3 /* IEU0 Group */; \
122 fpadd32 %S2, %S3, %T1 /* FPA */; \
123 fcmpgt32 %S0, %T0, %g7 /* FPM */; \
124 add %o2, %o3, %o2 /* IEU0 Group */; \
125 fpadd32 %T0, %T1, %U0 /* FPA */; \
126 fcmpgt32 %S2, %T1, %o3 /* FPM */; \
127 inc %o4 /* IEU0 Group */; \
128 inc %o5 /* IEU1 */; \
129 srl %o4, 1, %o4 /* IEU0 Group */; \
130 inc %g1 /* IEU1 */; \
131 add %o2, %o4, %o2 /* IEU0 Group */; \
132 fcmpgt32 %fz, %f2, %o4 /* FPM */; \
133 srl %o5, 1, %o5 /* IEU0 Group */; \
134 inc %g2 /* IEU1 */; \
135 add %o2, %o5, %o2 /* IEU0 Group */; \
136 fcmpgt32 %fz, %f6, %o5 /* FPM */; \
137 srl %g1, 1, %g1 /* IEU0 Group */; \
138 inc %g3 /* IEU1 */; \
139 add %o2, %g1, %o2 /* IEU0 Group */; \
140 fcmpgt32 %fz, %f10, %g1 /* FPM */; \
141 srl %g2, 1, %g2 /* IEU0 Group */; \
142 inc %g5 /* IEU1 */; \
143 add %o2, %g2, %o2 /* IEU0 Group */; \
144 fcmpgt32 %fz, %f14, %g2 /* FPM */; \
145 srl %g3, 1, %g3 /* IEU0 Group */; \
146 inc %g7 /* IEU1 */; \
147 add %o2, %g3, %o2 /* IEU0 Group */; \
148 fcmpgt32 %fz, %S1, %g3 /* FPM */; \
149 srl %g5, 1, %g5 /* IEU0 Group */; \
150 inc %o3 /* IEU1 */; \
151 add %o2, %g5, %o2 /* IEU0 Group */; \
152 fcmpgt32 %fz, %S3, %g5 /* FPM */; \
153 srl %g7, 1, %g7 /* IEU0 Group */; \
154 inc %o4 /* IEU1 */; \
155 add %o2, %g7, %o2 /* IEU0 Group */; \
156 fcmpgt32 %fz, %T1, %g7 /* FPM */; \
157 srl %o3, 1, %o3 /* IEU0 Group */; \
158 inc %o5 /* IEU1 */; \
159 add %o2, %o3, %o2 /* IEU0 Group */; \
160 fcmpgt32 %T0, %U0, %o3 /* FPM */; \
161 srl %o4, 1, %o4 /* IEU0 Group */; \
162 inc %g1 /* IEU1 */; \
163 sub %o2, %o4, %o2 /* IEU0 Group */; \
164 fcmpgt32 %fz, %U0, %o4 /* FPM */; \
165 srl %o5, 1, %o5 /* IEU0 Group */; \
166 inc %g2 /* IEU1 */; \
167 srl %g1, 1, %g1 /* IEU0 Group */; \
168 sub %o2, %o5, %o2 /* IEU1 */; \
169 std %U0, [%sp + STACKOFF] /* Store */; \
170 srl %g2, 1, %g2 /* IEU0 Group */; \
171 sub %o2, %g1, %o2 /* IEU1 */; \
172 inc %g3 /* IEU0 Group */; \
173 sub %o2, %g2, %o2 /* IEU1 */; \
174 srl %g3, 1, %g3 /* IEU0 Group */; \
175 inc %g5 /* IEU1 */; \
176 srl %g5, 1, %g5 /* IEU0 Group */; \
177 sub %o2, %g3, %o2 /* IEU1 */; \
178 ldx [%sp + STACKOFF], %o5 /* Load Group */; \
179 inc %g7 /* IEU0 */; \
180 sub %o2, %g5, %o2 /* IEU1 */; \
181 srl %g7, 1, %g7 /* IEU0 Group */; \
182 inc %o3 /* IEU1 */; \
183 srl %o3, 1, %o3 /* IEU0 Group */; \
184 sub %o2, %g7, %o2 /* IEU1 */; \
185 inc %o4 /* IEU0 Group */; \
186 add %o2, %o3, %o2 /* IEU1 */; \
187 srl %o4, 1, %o4 /* IEU0 Group */; \
188 sub %o2, %o4, %o2 /* IEU0 Group */; \
189 addcc %o2, %o5, %o2 /* IEU1 Group */; \
190 bcs,a,pn %xcc, 33f /* CTI */; \
191 add %o2, 1, %o2 /* IEU0 */; \
194 #define CSUM_LASTCHUNK(offset) \
195 ldx [%o0 - offset - 0x10], %g2; \
196 ldx [%o0 - offset - 0x08], %g3; \
197 addcc %g2, %o2, %o2; \
198 bcs,a,pn %xcc, 31f; \
200 31: addcc %g3, %o2, %o2; \
201 bcs,a,pn %xcc, 32f; \
209 andcc %o0, 7, %g0 /* IEU1 Group */
210 be,pt %icc, 4f /* CTI */
211 andcc %o0, 0x38, %g3 /* IEU1 */
212 mov 1, %g5 /* IEU0 Group */
213 cmp %o1, 6 /* IEU1 */
214 bl,pn %icc, 21f /* CTI */
215 andcc %o0, 1, %g0 /* IEU1 Group */
216 bne,pn %icc, csump_really_slow /* CTI */
217 andcc %o0, 2, %g0 /* IEU1 Group */
218 be,pt %icc, 1f /* CTI */
219 and %o0, 4, %g7 /* IEU0 */
220 lduh [%o0], %g2 /* Load */
221 sub %o1, 2, %o1 /* IEU0 Group */
222 add %o0, 2, %o0 /* IEU1 */
223 andcc %o0, 4, %g7 /* IEU1 Group */
224 sll %g5, 16, %g5 /* IEU0 */
225 sll %g2, 16, %g2 /* IEU0 Group */
226 addcc %g2, %o2, %o2 /* IEU1 Group (regdep) */
227 bcs,a,pn %icc, 1f /* CTI */
228 add %o2, %g5, %o2 /* IEU0 */
229 1: ld [%o0], %g2 /* Load */
230 brz,a,pn %g7, 4f /* CTI+IEU1 Group */
231 and %o0, 0x38, %g3 /* IEU0 */
232 add %o0, 4, %o0 /* IEU0 Group */
233 sub %o1, 4, %o1 /* IEU1 */
234 addcc %g2, %o2, %o2 /* IEU1 Group */
235 bcs,a,pn %icc, 1f /* CTI */
236 add %o2, 1, %o2 /* IEU0 */
237 1: and %o0, 0x38, %g3 /* IEU1 Group */
238 4: srl %o2, 0, %o2 /* IEU0 Group */
239 mov 0x40, %g1 /* IEU1 */
240 brz,pn %g3, 3f /* CTI+IEU1 Group */
241 sub %g1, %g3, %g1 /* IEU0 */
242 cmp %o1, 56 /* IEU1 Group */
243 blu,pn %icc, 20f /* CTI */
244 andcc %o0, 8, %g0 /* IEU1 Group */
245 be,pn %icc, 1f /* CTI */
246 ldx [%o0], %g2 /* Load */
247 add %o0, 8, %o0 /* IEU0 Group */
248 sub %o1, 8, %o1 /* IEU1 */
249 addcc %g2, %o2, %o2 /* IEU1 Group */
250 bcs,a,pn %xcc, 1f /* CTI */
251 add %o2, 1, %o2 /* IEU0 */
252 1: andcc %g1, 0x10, %g0 /* IEU1 Group */
253 be,pn %icc, 2f /* CTI */
254 and %g1, 0x20, %g1 /* IEU0 */
255 ldx [%o0], %g2 /* Load */
256 ldx [%o0+8], %g3 /* Load Group */
257 add %o0, 16, %o0 /* IEU0 */
258 sub %o1, 16, %o1 /* IEU1 */
259 addcc %g2, %o2, %o2 /* IEU1 Group */
260 bcs,a,pn %xcc, 1f /* CTI */
261 add %o2, 1, %o2 /* IEU0 */
262 1: addcc %g3, %o2, %o2 /* IEU1 Group */
263 bcs,a,pn %xcc, 2f /* CTI */
264 add %o2, 1, %o2 /* IEU0 */
265 2: brz,pn %g1, 3f /* CTI+IEU1 Group */
266 ldx [%o0], %g2 /* Load */
267 ldx [%o0+8], %g3 /* Load Group */
268 ldx [%o0+16], %g5 /* Load Group */
269 ldx [%o0+24], %g7 /* Load Group */
270 add %o0, 32, %o0 /* IEU0 */
271 sub %o1, 32, %o1 /* IEU1 */
272 addcc %g2, %o2, %o2 /* IEU1 Group */
273 bcs,a,pn %xcc, 1f /* CTI */
274 add %o2, 1, %o2 /* IEU0 */
275 1: addcc %g3, %o2, %o2 /* IEU1 Group */
276 bcs,a,pn %xcc, 1f /* CTI */
277 add %o2, 1, %o2 /* IEU0 */
278 1: addcc %g5, %o2, %o2 /* IEU1 Group */
279 bcs,a,pn %xcc, 1f /* CTI */
280 add %o2, 1, %o2 /* IEU0 */
281 1: addcc %g7, %o2, %o2 /* IEU1 Group */
282 bcs,a,pn %xcc, 3f /* CTI */
283 add %o2, 1, %o2 /* IEU0 */
284 3: cmp %o1, 0xc0 /* IEU1 Group */
285 blu,pn %icc, 20f /* CTI */
286 sllx %o2, 32, %g5 /* IEU0 */
290 addcc %o2, %g5, %o2 /* IEU1 Group */
291 sub %o1, 0xc0, %o1 /* IEU0 */
292 wr %g0, ASI_BLK_P, %asi /* LSU Group */
293 membar #StoreLoad /* LSU Group */
294 srlx %o2, 32, %o2 /* IEU0 Group */
295 bcs,a,pn %xcc, 1f /* CTI */
296 add %o2, 1, %o2 /* IEU1 */
297 1: andcc %o1, 0x80, %g0 /* IEU1 Group */
298 bne,pn %icc, 7f /* CTI */
299 andcc %o1, 0x40, %g0 /* IEU1 Group */
300 be,pn %icc, 6f /* CTI */
302 fzero %f14 /* FPA Group */
303 ldda [%o0 + 0x000] %asi, %f16
304 ldda [%o0 + 0x040] %asi, %f32
305 ldda [%o0 + 0x080] %asi, %f48
306 START_THE_TRICK(f12,f16,f18,f20,f22,f24,f26)
308 6: sub %o0, 0x40, %o0 /* IEU0 Group */
310 fzero %f30 /* FPA Group */
311 ldda [%o0 + 0x040] %asi, %f32
312 ldda [%o0 + 0x080] %asi, %f48
313 ldda [%o0 + 0x0c0] %asi, %f0
314 START_THE_TRICK(f28,f32,f34,f36,f38,f40,f42)
316 7: bne,pt %icc, 8f /* CTI */
318 add %o0, 0x40, %o0 /* IEU0 Group */
320 fzero %f62 /* FPA Group */
321 ldda [%o0 - 0x040] %asi, %f0
322 ldda [%o0 + 0x000] %asi, %f16
323 ldda [%o0 + 0x040] %asi, %f32
324 START_THE_TRICK(f60,f0,f2,f4,f6,f8,f10)
326 8: add %o0, 0x80, %o0 /* IEU0 Group */
328 ldda [%o0 - 0x080] %asi, %f48
329 ldda [%o0 - 0x040] %asi, %f0
330 ldda [%o0 + 0x000] %asi, %f16
331 START_THE_TRICK(f44,f48,f50,f52,f54,f56,f58)
332 1: DO_THE_TRICK(f44,f46,f48,f50,f52,f54,f56,f58,f60,f62,f0,f2,f4,f6,f8,f10,f12,f14)
333 ldda [%o0 + 0x040] %asi, %f32
334 2: DO_THE_TRICK(f60,f62,f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30)
335 ldda [%o0 + 0x080] %asi, %f48
336 3: DO_THE_TRICK(f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46)
337 ldda [%o0 + 0x0c0] %asi, %f0
338 4: DO_THE_TRICK(f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,f48,f50,f52,f54,f56,f58,f60,f62)
339 add %o0, 0x100, %o0 /* IEU0 Group */
340 subcc %o1, 0x100, %o1 /* IEU1 */
341 bgeu,a,pt %icc, 1b /* CTI */
342 ldda [%o0 + 0x000] %asi, %f16
343 membar #Sync /* LSU Group */
344 DO_THE_TRICK(f44,f46,f48,f50,f52,f54,f56,f58,f60,f62,f0,f2,f4,f6,f8,f10,f12,f14)
345 END_THE_TRICK(f60,f62,f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30)
347 ldub [%g6 + TI_CURRENT_DS], %g7
349 and %o1, 0x3f, %o1 /* IEU0 Group */
354 20: andcc %o1, 0xf0, %g1 /* IEU1 Group */
355 be,pn %icc, 23f /* CTI */
356 and %o1, 0xf, %o3 /* IEU0 */
358 22: sll %g1, 1, %o4 /* IEU0 Group */
359 sethi %hi(23f), %g7 /* IEU1 */
360 sub %g7, %o4, %g7 /* IEU0 Group */
361 jmpl %g7 + %lo(23f), %g0 /* CTI Group brk forced*/
362 add %o0, %g1, %o0 /* IEU0 */
364 22: rd %pc, %g7 /* LSU Group+4bubbles */
365 sll %g1, 1, %o4 /* IEU0 Group */
366 sub %g7, %o4, %g7 /* IEU0 Group (regdep) */
367 jmpl %g7 + (23f - 22b), %g0 /* CTI Group brk forced*/
368 add %o0, %g1, %o0 /* IEU0 */
385 23: brnz,pn %o3, 26f /* CTI+IEU1 Group */
386 24: sllx %o2, 32, %g1 /* IEU0 */
387 25: addcc %o2, %g1, %o0 /* IEU1 Group */
388 srlx %o0, 32, %o0 /* IEU0 Group (regdep) */
389 bcs,a,pn %xcc, 1f /* CTI */
390 add %o0, 1, %o0 /* IEU1 */
391 1: retl /* CTI Group brk forced*/
392 srl %o0, 0, %o0 /* IEU0 */
393 26: andcc %o1, 8, %g0 /* IEU1 Group */
394 be,pn %icc, 1f /* CTI */
395 ldx [%o0], %g3 /* Load */
396 add %o0, 8, %o0 /* IEU0 Group */
397 addcc %g3, %o2, %o2 /* IEU1 Group */
398 bcs,a,pn %xcc, 1f /* CTI */
399 add %o2, 1, %o2 /* IEU0 */
400 1: andcc %o1, 4, %g0 /* IEU1 Group */
401 be,a,pn %icc, 1f /* CTI */
403 ld [%o0], %g2 /* Load */
404 add %o0, 4, %o0 /* IEU0 Group */
405 sllx %g2, 32, %g2 /* IEU0 Group */
406 1: andcc %o1, 2, %g0 /* IEU1 */
407 be,a,pn %icc, 1f /* CTI */
408 clr %o4 /* IEU0 Group */
409 lduh [%o0], %o4 /* Load */
410 add %o0, 2, %o0 /* IEU1 */
411 sll %o4, 16, %o4 /* IEU0 Group */
412 1: andcc %o1, 1, %g0 /* IEU1 */
413 be,a,pn %icc, 1f /* CTI */
414 clr %o5 /* IEU0 Group */
415 ldub [%o0], %o5 /* Load */
416 sll %o5, 8, %o5 /* IEU0 Group */
417 1: or %g2, %o4, %o4 /* IEU1 */
418 or %o5, %o4, %o4 /* IEU0 Group (regdep) */
419 addcc %o4, %o2, %o2 /* IEU1 Group (regdep) */
420 bcs,a,pn %xcc, 1f /* CTI */
421 add %o2, 1, %o2 /* IEU0 */
422 1: ba,pt %xcc, 25b /* CTI Group */
423 sllx %o2, 32, %g1 /* IEU0 */
424 21: srl %o2, 0, %o2 /* IEU0 Group */
425 cmp %o1, 0 /* IEU1 */
426 be,pn %icc, 24b /* CTI */
427 andcc %o1, 4, %g0 /* IEU1 Group */
428 be,a,pn %icc, 1f /* CTI */
430 lduh [%o0], %g3 /* Load */
431 lduh [%o0+2], %g2 /* Load Group */
432 add %o0, 4, %o0 /* IEU0 Group */
433 sllx %g3, 48, %g3 /* IEU0 Group */
434 sllx %g2, 32, %g2 /* IEU0 Group */
435 or %g3, %g2, %g2 /* IEU0 Group */
436 1: andcc %o1, 2, %g0 /* IEU1 */
437 be,a,pn %icc, 1f /* CTI */
438 clr %o4 /* IEU0 Group */
439 lduh [%o0], %o4 /* Load */
440 add %o0, 2, %o0 /* IEU1 */
441 sll %o4, 16, %o4 /* IEU0 Group */
442 1: andcc %o1, 1, %g0 /* IEU1 */
443 be,a,pn %icc, 1f /* CTI */
444 clr %o5 /* IEU0 Group */
445 ldub [%o0], %o5 /* Load */
446 sll %o5, 8, %o5 /* IEU0 Group */
447 1: or %g2, %o4, %o4 /* IEU1 */
448 or %o5, %o4, %o4 /* IEU0 Group (regdep) */
449 addcc %o4, %o2, %o2 /* IEU1 Group (regdep) */
450 bcs,a,pn %xcc, 1f /* CTI */
451 add %o2, 1, %o2 /* IEU0 */
452 1: ba,pt %xcc, 25b /* CTI Group */
453 sllx %o2, 32, %g1 /* IEU0 */
455 /* When buff is byte aligned and len is large, we backoff to
456 * this really slow handling. The issue is that we cannot do
457 * the VIS stuff when buff is byte aligned as unaligned.c will
498 sethi %hi(64512), %g2
525 sethi %hi(64512), %g2