2 * csum_partial_copy - do IP checksumming and copy
4 * (C) Copyright 1996 Linus Torvalds
6 * Don't look at this too closely - you'll go mad. The things
7 * we do for performance..
10 #include <linux/types.h>
11 #include <linux/string.h>
12 #include <asm/uaccess.h>
16 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
19 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
21 #define extql(x,y,z) \
22 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
24 #define extqh(x,y,z) \
25 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
27 #define mskql(x,y,z) \
28 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
30 #define mskqh(x,y,z) \
31 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
33 #define insql(x,y,z) \
34 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
36 #define insqh(x,y,z) \
37 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
40 #define __get_user_u(x,ptr) \
43 __asm__ __volatile__( \
46 ".section __ex_table,\"a\"\n" \
48 " lda %0,2b-1b(%1)\n" \
50 : "=r"(x), "=r"(__guu_err) \
51 : "m"(__m(ptr)), "1"(0)); \
55 #define __put_user_u(x,ptr) \
58 __asm__ __volatile__( \
61 ".section __ex_table,\"a\"\n" \
63 " lda $31,2b-1b(%0)\n" \
66 : "m"(__m(addr)), "rJ"(x), "0"(0)); \
72 * Ok. This isn't fun, but this is the EASY case.
74 static inline unsigned long
75 csum_partial_cfu_aligned(const unsigned long *src
, unsigned long *dst
,
76 long len
, unsigned long checksum
,
79 unsigned long carry
= 0;
84 err
|= __get_user(word
, src
);
89 carry
= checksum
< word
;
96 unsigned long word
, tmp
;
97 err
|= __get_user(word
, src
);
99 mskql(word
, len
, word
);
101 mskqh(tmp
, len
, tmp
);
102 carry
= checksum
< word
;
106 if (err
) *errp
= err
;
111 * This is even less fun, but this is still reasonably
114 static inline unsigned long
115 csum_partial_cfu_dest_aligned(const unsigned long *src
, unsigned long *dst
,
117 long len
, unsigned long checksum
,
121 unsigned long word
, carry
;
122 unsigned long lastsrc
= 7+len
+(unsigned long)src
;
125 err
|= __get_user_u(first
,src
);
128 unsigned long second
;
130 err
|= __get_user_u(second
, src
+1);
131 extql(first
, soff
, word
);
134 extqh(second
, soff
, first
);
141 carry
= checksum
< word
;
147 unsigned long second
;
148 err
|= __get_user_u(second
, lastsrc
);
150 extql(first
, soff
, word
);
151 extqh(second
, soff
, first
);
153 mskql(word
, len
, word
);
155 mskqh(tmp
, len
, tmp
);
156 carry
= checksum
< word
;
160 if (err
) *errp
= err
;
165 * This is slightly less fun than the above..
167 static inline unsigned long
168 csum_partial_cfu_src_aligned(const unsigned long *src
, unsigned long *dst
,
170 long len
, unsigned long checksum
,
171 unsigned long partial_dest
,
174 unsigned long carry
= 0;
176 unsigned long second_dest
;
179 mskql(partial_dest
, doff
, partial_dest
);
181 err
|= __get_user(word
, src
);
183 insql(word
, doff
, second_dest
);
185 stq_u(partial_dest
| second_dest
, dst
);
188 insqh(word
, doff
, partial_dest
);
189 carry
= checksum
< word
;
195 err
|= __get_user(word
, src
);
196 mskql(word
, len
, word
);
199 insql(word
, doff
, second_dest
);
201 carry
= checksum
< word
;
202 partial_dest
|= second_dest
;
204 stq_u(partial_dest
, dst
);
207 insqh(word
, doff
, partial_dest
);
211 ldq_u(second_dest
, dst
);
212 mskqh(second_dest
, doff
, second_dest
);
213 stq_u(partial_dest
| second_dest
, dst
);
216 if (err
) *errp
= err
;
221 * This is so totally un-fun that it's frightening. Don't
222 * look at this too closely, you'll go blind.
224 static inline unsigned long
225 csum_partial_cfu_unaligned(const unsigned long * src
, unsigned long * dst
,
226 unsigned long soff
, unsigned long doff
,
227 long len
, unsigned long checksum
,
228 unsigned long partial_dest
,
231 unsigned long carry
= 0;
233 unsigned long lastsrc
;
236 err
|= __get_user_u(first
, src
);
237 lastsrc
= 7+len
+(unsigned long)src
;
238 mskql(partial_dest
, doff
, partial_dest
);
240 unsigned long second
, word
;
241 unsigned long second_dest
;
243 err
|= __get_user_u(second
, src
+1);
244 extql(first
, soff
, word
);
247 extqh(second
, soff
, first
);
251 insql(word
, doff
, second_dest
);
253 stq_u(partial_dest
| second_dest
, dst
);
254 carry
= checksum
< word
;
255 insqh(word
, doff
, partial_dest
);
261 unsigned long second
, word
;
262 unsigned long second_dest
;
264 err
|= __get_user_u(second
, lastsrc
);
265 extql(first
, soff
, word
);
266 extqh(second
, soff
, first
);
269 mskql(word
, len
-doff
, word
);
271 insql(word
, doff
, second_dest
);
272 carry
= checksum
< word
;
273 stq_u(partial_dest
| second_dest
, dst
);
275 ldq_u(second_dest
, dst
+1);
276 insqh(word
, doff
, partial_dest
);
277 mskqh(second_dest
, len
, second_dest
);
278 stq_u(partial_dest
| second_dest
, dst
+1);
282 unsigned long second
, word
;
283 unsigned long second_dest
;
285 err
|= __get_user_u(second
, lastsrc
);
286 extql(first
, soff
, word
);
287 extqh(second
, soff
, first
);
289 ldq_u(second_dest
, dst
);
290 mskql(word
, len
-doff
, word
);
292 mskqh(second_dest
, len
, second_dest
);
293 carry
= checksum
< word
;
294 insql(word
, doff
, word
);
295 stq_u(partial_dest
| word
| second_dest
, dst
);
298 if (err
) *errp
= err
;
303 do_csum_partial_copy_from_user(const char *src
, char *dst
, int len
,
304 unsigned int sum
, int *errp
)
306 unsigned long checksum
= (unsigned) sum
;
307 unsigned long soff
= 7 & (unsigned long) src
;
308 unsigned long doff
= 7 & (unsigned long) dst
;
313 checksum
= csum_partial_cfu_aligned(
314 (const unsigned long *) src
,
315 (unsigned long *) dst
,
316 len
-8, checksum
, errp
);
318 checksum
= csum_partial_cfu_dest_aligned(
319 (const unsigned long *) src
,
320 (unsigned long *) dst
,
321 soff
, len
-8, checksum
, errp
);
323 unsigned long partial_dest
;
324 ldq_u(partial_dest
, dst
);
326 checksum
= csum_partial_cfu_src_aligned(
327 (const unsigned long *) src
,
328 (unsigned long *) dst
,
329 doff
, len
-8, checksum
,
332 checksum
= csum_partial_cfu_unaligned(
333 (const unsigned long *) src
,
334 (unsigned long *) dst
,
335 soff
, doff
, len
-8, checksum
,
339 checksum
= (checksum
& 0xffffffff) + (checksum
>> 32);
340 /* 33 -> < 32 bits */
341 checksum
= (checksum
& 0xffff) + (checksum
>> 16);
343 checksum
= (checksum
& 0xffff) + (checksum
>> 16);
344 checksum
= (checksum
& 0xffff) + (checksum
>> 16);
350 csum_partial_copy_from_user(const char *src
, char *dst
, int len
,
351 unsigned int sum
, int *errp
)
353 if (!access_ok(src
, len
, VERIFY_READ
)) {
359 return do_csum_partial_copy_from_user(src
, dst
, len
, sum
, errp
);
363 csum_partial_copy_nocheck(const char *src
, char *dst
, int len
, unsigned int sum
)
365 return do_csum_partial_copy_from_user(src
, dst
, len
, sum
, NULL
);
369 csum_partial_copy (const char *src
, char *dst
, int len
, unsigned int sum
)
374 ret
= do_csum_partial_copy_from_user(src
, dst
, len
, sum
, &error
);
376 printk("csum_partial_copy_old(): tell mingo to convert me!\n");