arch/alpha/lib/csum_partial_copy.c

   1 /*
   2  * csum_partial_copy - do IP checksumming and copy
   3  *
   4  * (C) Copyright 1996 Linus Torvalds
   5  *
   6  * Don't look at this too closely - you'll go mad. The things
   7  * we do for performance..
   8  */
   9
  10 #include <linux/types.h>
  11 #include <linux/string.h>
  12 #include <asm/uaccess.h>
  13
  14
  15 #define ldq_u(x,y) \
  16 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
  17
  18 #define stq_u(x,y) \
  19 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
  20
  21 #define extql(x,y,z) \
  22 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  23
  24 #define extqh(x,y,z) \
  25 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  26
  27 #define mskql(x,y,z) \
  28 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  29
  30 #define mskqh(x,y,z) \
  31 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  32
  33 #define insql(x,y,z) \
  34 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  35
  36 #define insqh(x,y,z) \
  37 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  38
  39
  40 #define __get_user_u(x,ptr)                             \
  41 ({                                                      \
  42         long __guu_err;                                 \
  43         __asm__ __volatile__(                           \
  44         "1:     ldq_u %0,%2\n"                          \
  45         "2:\n"                                          \
  46         ".section __ex_table,\"a\"\n"                   \
  47         "       .gprel32 1b\n"                          \
  48         "       lda %0,2b-1b(%1)\n"                     \
  49         ".previous"                                     \
  50                 : "=r"(x), "=r"(__guu_err)              \
  51                 : "m"(__m(ptr)), "1"(0));               \
  52         __guu_err;                                      \
  53 })
  54
  55 #define __put_user_u(x,ptr)                             \
  56 ({                                                      \
  57         long __puu_err;                                 \
  58         __asm__ __volatile__(                           \
  59         "1:     stq_u %2,%1\n"                          \
  60         "2:\n"                                          \
  61         ".section __ex_table,\"a\"\n"                   \
  62         "       .gprel32 1b"                            \
  63         "       lda $31,2b-1b(%0)\n"                    \
  64         ".previous"                                     \
  65                 : "=r"(__puu_err)                       \
  66                 : "m"(__m(addr)), "rJ"(x), "0"(0));     \
  67         __puu_err;                                      \
  68 })
  69
  70
  71 /*
  72  * Ok. This isn't fun, but this is the EASY case.
  73  */
  74 static inline unsigned long
  75 csum_partial_cfu_aligned(const unsigned long *src, unsigned long *dst,
  76                          long len, unsigned long checksum,
  77                          int *errp)
  78 {
  79         unsigned long carry = 0;
  80         int err = 0;
  81
  82         while (len >= 0) {
  83                 unsigned long word;
  84                 err |= __get_user(word, src);
  85                 checksum += carry;
  86                 src++;
  87                 checksum += word;
  88                 len -= 8;
  89                 carry = checksum < word;
  90                 *dst = word;
  91                 dst++;
  92         }
  93         len += 8;
  94         checksum += carry;
  95         if (len) {
  96                 unsigned long word, tmp;
  97                 err |= __get_user(word, src);
  98                 tmp = *dst;
  99                 mskql(word, len, word);
 100                 checksum += word;
 101                 mskqh(tmp, len, tmp);
 102                 carry = checksum < word;
 103                 *dst = word | tmp;
 104                 checksum += carry;
 105         }
 106         if (err) *errp = err;
 107         return checksum;
 108 }
 109
 110 /*
 111  * This is even less fun, but this is still reasonably
 112  * easy.
 113  */
 114 static inline unsigned long
 115 csum_partial_cfu_dest_aligned(const unsigned long *src, unsigned long *dst,
 116                               unsigned long soff,
 117                               long len, unsigned long checksum,
 118                               int *errp)
 119 {
 120         unsigned long first;
 121         unsigned long word, carry;
 122         unsigned long lastsrc = 7+len+(unsigned long)src;
 123         int err = 0;
 124
 125         err |= __get_user_u(first,src);
 126         carry = 0;
 127         while (len >= 0) {
 128                 unsigned long second;
 129
 130                 err |= __get_user_u(second, src+1);
 131                 extql(first, soff, word);
 132                 len -= 8;
 133                 src++;
 134                 extqh(second, soff, first);
 135                 checksum += carry;
 136                 word |= first;
 137                 first = second;
 138                 checksum += word;
 139                 *dst = word;
 140                 dst++;
 141                 carry = checksum < word;
 142         }
 143         len += 8;
 144         checksum += carry;
 145         if (len) {
 146                 unsigned long tmp;
 147                 unsigned long second;
 148                 err |= __get_user_u(second, lastsrc);
 149                 tmp = *dst;
 150                 extql(first, soff, word);
 151                 extqh(second, soff, first);
 152                 word |= first;
 153                 mskql(word, len, word);
 154                 checksum += word;
 155                 mskqh(tmp, len, tmp);
 156                 carry = checksum < word;
 157                 *dst = word | tmp;
 158                 checksum += carry;
 159         }
 160         if (err) *errp = err;
 161         return checksum;
 162 }
 163
 164 /*
 165  * This is slightly less fun than the above..
 166  */
 167 static inline unsigned long
 168 csum_partial_cfu_src_aligned(const unsigned long *src, unsigned long *dst,
 169                              unsigned long doff,
 170                              long len, unsigned long checksum,
 171                              unsigned long partial_dest,
 172                              int *errp)
 173 {
 174         unsigned long carry = 0;
 175         unsigned long word;
 176         unsigned long second_dest;
 177         int err = 0;
 178
 179         mskql(partial_dest, doff, partial_dest);
 180         while (len >= 0) {
 181                 err |= __get_user(word, src);
 182                 len -= 8;
 183                 insql(word, doff, second_dest);
 184                 checksum += carry;
 185                 stq_u(partial_dest | second_dest, dst);
 186                 src++;
 187                 checksum += word;
 188                 insqh(word, doff, partial_dest);
 189                 carry = checksum < word;
 190                 dst++;
 191         }
 192         len += 8;
 193         if (len) {
 194                 checksum += carry;
 195                 err |= __get_user(word, src);
 196                 mskql(word, len, word);
 197                 len -= 8;
 198                 checksum += word;
 199                 insql(word, doff, second_dest);
 200                 len += doff;
 201                 carry = checksum < word;
 202                 partial_dest |= second_dest;
 203                 if (len >= 0) {
 204                         stq_u(partial_dest, dst);
 205                         if (!len) goto out;
 206                         dst++;
 207                         insqh(word, doff, partial_dest);
 208                 }
 209                 doff = len;
 210         }
 211         ldq_u(second_dest, dst);
 212         mskqh(second_dest, doff, second_dest);
 213         stq_u(partial_dest | second_dest, dst);
 214 out:
 215         checksum += carry;
 216         if (err) *errp = err;
 217         return checksum;
 218 }
 219
 220 /*
 221  * This is so totally un-fun that it's frightening. Don't
 222  * look at this too closely, you'll go blind.
 223  */
 224 static inline unsigned long
 225 csum_partial_cfu_unaligned(const unsigned long * src, unsigned long * dst,
 226                            unsigned long soff, unsigned long doff,
 227                            long len, unsigned long checksum,
 228                            unsigned long partial_dest,
 229                            int *errp)
 230 {
 231         unsigned long carry = 0;
 232         unsigned long first;
 233         unsigned long lastsrc;
 234         int err = 0;
 235
 236         err |= __get_user_u(first, src);
 237         lastsrc = 7+len+(unsigned long)src;
 238         mskql(partial_dest, doff, partial_dest);
 239         while (len >= 0) {
 240                 unsigned long second, word;
 241                 unsigned long second_dest;
 242
 243                 err |= __get_user_u(second, src+1);
 244                 extql(first, soff, word);
 245                 checksum += carry;
 246                 len -= 8;
 247                 extqh(second, soff, first);
 248                 src++;
 249                 word |= first;
 250                 first = second;
 251                 insql(word, doff, second_dest);
 252                 checksum += word;
 253                 stq_u(partial_dest | second_dest, dst);
 254                 carry = checksum < word;
 255                 insqh(word, doff, partial_dest);
 256                 dst++;
 257         }
 258         len += doff;
 259         checksum += carry;
 260         if (len >= 0) {
 261                 unsigned long second, word;
 262                 unsigned long second_dest;
 263
 264                 err |= __get_user_u(second, lastsrc);
 265                 extql(first, soff, word);
 266                 extqh(second, soff, first);
 267                 word |= first;
 268                 first = second;
 269                 mskql(word, len-doff, word);
 270                 checksum += word;
 271                 insql(word, doff, second_dest);
 272                 carry = checksum < word;
 273                 stq_u(partial_dest | second_dest, dst);
 274                 if (len) {
 275                         ldq_u(second_dest, dst+1);
 276                         insqh(word, doff, partial_dest);
 277                         mskqh(second_dest, len, second_dest);
 278                         stq_u(partial_dest | second_dest, dst+1);
 279                 }
 280                 checksum += carry;
 281         } else {
 282                 unsigned long second, word;
 283                 unsigned long second_dest;
 284
 285                 err |= __get_user_u(second, lastsrc);
 286                 extql(first, soff, word);
 287                 extqh(second, soff, first);
 288                 word |= first;
 289                 ldq_u(second_dest, dst);
 290                 mskql(word, len-doff, word);
 291                 checksum += word;
 292                 mskqh(second_dest, len, second_dest);
 293                 carry = checksum < word;
 294                 insql(word, doff, word);
 295                 stq_u(partial_dest | word | second_dest, dst);
 296                 checksum += carry;
 297         }
 298         if (err) *errp = err;
 299         return checksum;
 300 }
 301
 302 static unsigned int
 303 do_csum_partial_copy_from_user(const char *src, char *dst, int len,
 304                                unsigned int sum, int *errp)
 305 {
 306         unsigned long checksum = (unsigned) sum;
 307         unsigned long soff = 7 & (unsigned long) src;
 308         unsigned long doff = 7 & (unsigned long) dst;
 309
 310         if (len) {
 311                 if (!doff) {
 312                         if (!soff)
 313                                 checksum = csum_partial_cfu_aligned(
 314                                         (const unsigned long *) src,
 315                                         (unsigned long *) dst,
 316                                         len-8, checksum, errp);
 317                         else
 318                                 checksum = csum_partial_cfu_dest_aligned(
 319                                         (const unsigned long *) src,
 320                                         (unsigned long *) dst,
 321                                         soff, len-8, checksum, errp);
 322                 } else {
 323                         unsigned long partial_dest;
 324                         ldq_u(partial_dest, dst);
 325                         if (!soff)
 326                                 checksum = csum_partial_cfu_src_aligned(
 327                                         (const unsigned long *) src,
 328                                         (unsigned long *) dst,
 329                                         doff, len-8, checksum,
 330                                         partial_dest, errp);
 331                         else
 332                                 checksum = csum_partial_cfu_unaligned(
 333                                         (const unsigned long *) src,
 334                                         (unsigned long *) dst,
 335                                         soff, doff, len-8, checksum,
 336                                         partial_dest, errp);
 337                 }
 338                 /* 64 -> 33 bits */
 339                 checksum = (checksum & 0xffffffff) + (checksum >> 32);
 340                 /* 33 -> < 32 bits */
 341                 checksum = (checksum & 0xffff) + (checksum >> 16);
 342                 /* 32 -> 16 bits */
 343                 checksum = (checksum & 0xffff) + (checksum >> 16);
 344                 checksum = (checksum & 0xffff) + (checksum >> 16);
 345         }
 346         return checksum;
 347 }
 348
 349 unsigned int
 350 csum_partial_copy_from_user(const char *src, char *dst, int len,
 351                             unsigned int sum, int *errp)
 352 {
 353         if (!access_ok(src, len, VERIFY_READ)) {
 354                 *errp = -EFAULT;
 355                 memset(dst, 0, len);
 356                 return sum;
 357         }
 358
 359         return do_csum_partial_copy_from_user(src, dst, len, sum, errp);
 360 }
 361
 362 unsigned int
 363 csum_partial_copy_nocheck(const char *src, char *dst, int len, unsigned int sum)
 364 {
 365         return do_csum_partial_copy_from_user(src, dst, len, sum, NULL);
 366 }
 367
 368 unsigned int
 369 csum_partial_copy (const char *src, char *dst, int len, unsigned int sum)
 370 {
 371         unsigned int ret;
 372         int error = 0;
 373
 374         ret = do_csum_partial_copy_from_user(src, dst, len, sum, &error);
 375         if (error)
 376                 printk("csum_partial_copy_old(): tell mingo to convert me!\n");
 377
 378         return ret;
 379 }