arch/alpha/lib/csum_partial_copy.c

   1 /*
   2  * csum_partial_copy - do IP checksumming and copy
   3  *
   4  * (C) Copyright 1996 Linus Torvalds
   5  *
   6  * Don't look at this too closely - you'll go mad. The things
   7  * we do for performance..
   8  */
   9
  10 #include <linux/types.h>
  11 #include <linux/string.h>
  12 #include <asm/uaccess.h>
  13
  14
  15 #define ldq_u(x,y) \
  16 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
  17
  18 #define stq_u(x,y) \
  19 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
  20
  21 #define extql(x,y,z) \
  22 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  23
  24 #define extqh(x,y,z) \
  25 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  26
  27 #define mskql(x,y,z) \
  28 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  29
  30 #define mskqh(x,y,z) \
  31 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  32
  33 #define insql(x,y,z) \
  34 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  35
  36 #define insqh(x,y,z) \
  37 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  38
  39
  40 #define __get_user_u(x,ptr)                             \
  41 ({                                                      \
  42         long __guu_err;                                 \
  43         __asm__ __volatile__(                           \
  44         "1:     ldq_u %0,%2\n"                          \
  45         "2:\n"                                          \
  46         ".section __ex_table,\"a\"\n"                   \
  47         "       .gprel32 1b\n"                          \
  48         "       lda %0,2b-1b(%1)\n"                     \
  49         ".previous"                                     \
  50                 : "=r"(x), "=r"(__guu_err)              \
  51                 : "m"(__m(ptr)), "1"(0));               \
  52         __guu_err;                                      \
  53 })
  54
  55 #define __put_user_u(x,ptr)                             \
  56 ({                                                      \
  57         long __puu_err;                                 \
  58         __asm__ __volatile__(                           \
  59         "1:     stq_u %2,%1\n"                          \
  60         "2:\n"                                          \
  61         ".section __ex_table,\"a\"\n"                   \
  62         "       .gprel32 1b"                            \
  63         "       lda $31,2b-1b(%0)\n"                    \
  64         ".previous"                                     \
  65                 : "=r"(__puu_err)                       \
  66                 : "m"(__m(addr)), "rJ"(x), "0"(0));     \
  67         __puu_err;                                      \
  68 })
  69
  70
  71 /*
  72  * Ok. This isn't fun, but this is the EASY case.
  73  */
  74 static inline unsigned long
  75 csum_partial_cfu_aligned(const unsigned long *src, unsigned long *dst,
  76                          long len, unsigned long checksum,
  77                          int *errp)
  78 {
  79         unsigned long carry = 0;
  80         int err = 0;
  81
  82         while (len >= 0) {
  83                 unsigned long word;
  84                 err |= __get_user(word, src);
  85                 checksum += carry;
  86                 src++;
  87                 checksum += word;
  88                 len -= 8;
  89                 carry = checksum < word;
  90                 *dst = word;
  91                 dst++;
  92         }
  93         len += 8;
  94         checksum += carry;
  95         if (len) {
  96                 unsigned long word, tmp;
  97                 err |= __get_user(word, src);
  98                 tmp = *dst;
  99                 mskql(word, len, word);
 100                 checksum += word;
 101                 mskqh(tmp, len, tmp);
 102                 carry = checksum < word;
 103                 *dst = word | tmp;
 104                 checksum += carry;
 105         }
 106         if (err) *errp = err;
 107         return checksum;
 108 }
 109
 110 /*
 111  * This is even less fun, but this is still reasonably
 112  * easy.
 113  */
 114 static inline unsigned long
 115 csum_partial_cfu_dest_aligned(const unsigned long *src, unsigned long *dst,
 116                               unsigned long soff,
 117                               long len, unsigned long checksum,
 118                               int *errp)
 119 {
 120         unsigned long first;
 121         unsigned long word, carry;
 122         unsigned long lastsrc = 7+len+(unsigned long)src;
 123         int err = 0;
 124
 125         err |= __get_user_u(first,src);
 126         carry = 0;
 127         while (len >= 0) {
 128                 unsigned long second;
 129
 130                 err |= __get_user_u(second, src+1);
 131                 extql(first, soff, word);
 132                 len -= 8;
 133                 src++;
 134                 extqh(second, soff, first);
 135                 checksum += carry;
 136                 word |= first;
 137                 first = second;
 138                 checksum += word;
 139                 *dst = word;
 140                 dst++;
 141                 carry = checksum < word;
 142         }
 143         len += 8;
 144         checksum += carry;
 145         if (len) {
 146                 unsigned long tmp;
 147                 unsigned long second;
 148                 err |= __get_user_u(second, lastsrc);
 149                 tmp = *dst;
 150                 extql(first, soff, word);
 151                 extqh(second, soff, first);
 152                 word |= first;
 153                 mskql(word, len, word);
 154                 checksum += word;
 155                 mskqh(tmp, len, tmp);
 156                 carry = checksum < word;
 157                 *dst = word | tmp;
 158                 checksum += carry;
 159         }
 160         if (err) *errp = err;
 161         return checksum;
 162 }
 163
 164 /*
 165  * This is slightly less fun than the above..
 166  */
 167 static inline unsigned long
 168 csum_partial_cfu_src_aligned(const unsigned long *src, unsigned long *dst,
 169                              unsigned long doff,
 170                              long len, unsigned long checksum,
 171                              unsigned long partial_dest,
 172                              int *errp)
 173 {
 174         unsigned long carry = 0;
 175         unsigned long word;
 176         int err = 0;
 177
 178         mskql(partial_dest, doff, partial_dest);
 179         while (len >= 0) {
 180                 unsigned long second_dest;
 181                 err |= __get_user(word, src);
 182                 len -= 8;
 183                 insql(word, doff, second_dest);
 184                 checksum += carry;
 185                 stq_u(partial_dest | second_dest, dst);
 186                 src++;
 187                 checksum += word;
 188                 insqh(word, doff, partial_dest);
 189                 carry = checksum < word;
 190                 dst++;
 191         }
 192         len += doff;
 193         checksum += carry;
 194         if (len >= 0) {
 195                 unsigned long second_dest;
 196                 err |= __get_user(word, src);
 197                 mskql(word, len-doff, word);
 198                 checksum += word;
 199                 insql(word, doff, second_dest);
 200                 stq_u(partial_dest | second_dest, dst);
 201                 carry = checksum < word;
 202                 if (len) {
 203                         ldq_u(second_dest, dst+1);
 204                         insqh(word, doff, partial_dest);
 205                         mskqh(second_dest, len, second_dest);
 206                         stq_u(partial_dest | second_dest, dst+1);
 207                 }
 208                 checksum += carry;
 209         } else if (len & 7) {
 210                 unsigned long second_dest;
 211                 err |= __get_user(word, src);
 212                 ldq_u(second_dest, dst);
 213                 mskql(word, len-doff, word);
 214                 checksum += word;
 215                 mskqh(second_dest, len, second_dest);
 216                 carry = checksum < word;
 217                 insql(word, doff, word);
 218                 stq_u(partial_dest | word | second_dest, dst);
 219                 checksum += carry;
 220         }
 221         if (err) *errp = err;
 222         return checksum;
 223 }
 224
 225 /*
 226  * This is so totally un-fun that it's frightening. Don't
 227  * look at this too closely, you'll go blind.
 228  */
 229 static inline unsigned long
 230 csum_partial_cfu_unaligned(const unsigned long * src, unsigned long * dst,
 231                            unsigned long soff, unsigned long doff,
 232                            long len, unsigned long checksum,
 233                            unsigned long partial_dest,
 234                            int *errp)
 235 {
 236         unsigned long carry = 0;
 237         unsigned long first;
 238         unsigned long lastsrc;
 239         int err = 0;
 240
 241         err |= __get_user_u(first, src);
 242         lastsrc = 7+len+(unsigned long)src;
 243         mskql(partial_dest, doff, partial_dest);
 244         while (len >= 0) {
 245                 unsigned long second, word;
 246                 unsigned long second_dest;
 247
 248                 err |= __get_user_u(second, src+1);
 249                 extql(first, soff, word);
 250                 checksum += carry;
 251                 len -= 8;
 252                 extqh(second, soff, first);
 253                 src++;
 254                 word |= first;
 255                 first = second;
 256                 insql(word, doff, second_dest);
 257                 checksum += word;
 258                 stq_u(partial_dest | second_dest, dst);
 259                 carry = checksum < word;
 260                 insqh(word, doff, partial_dest);
 261                 dst++;
 262         }
 263         len += doff;
 264         checksum += carry;
 265         if (len >= 0) {
 266                 unsigned long second, word;
 267                 unsigned long second_dest;
 268
 269                 err |= __get_user_u(second, lastsrc);
 270                 extql(first, soff, word);
 271                 extqh(second, soff, first);
 272                 word |= first;
 273                 first = second;
 274                 mskql(word, len-doff, word);
 275                 checksum += word;
 276                 insql(word, doff, second_dest);
 277                 carry = checksum < word;
 278                 stq_u(partial_dest | second_dest, dst);
 279                 if (len) {
 280                         ldq_u(second_dest, dst+1);
 281                         insqh(word, doff, partial_dest);
 282                         mskqh(second_dest, len, second_dest);
 283                         stq_u(partial_dest | second_dest, dst+1);
 284                 }
 285                 checksum += carry;
 286         } else if (len & 7) {
 287                 unsigned long second, word;
 288                 unsigned long second_dest;
 289
 290                 err |= __get_user_u(second, lastsrc);
 291                 extql(first, soff, word);
 292                 extqh(second, soff, first);
 293                 word |= first;
 294                 ldq_u(second_dest, dst);
 295                 mskql(word, len-doff, word);
 296                 checksum += word;
 297                 mskqh(second_dest, len, second_dest);
 298                 carry = checksum < word;
 299                 insql(word, doff, word);
 300                 stq_u(partial_dest | word | second_dest, dst);
 301                 checksum += carry;
 302         }
 303         if (err) *errp = err;
 304         return checksum;
 305 }
 306
 307 static unsigned int
 308 do_csum_partial_copy_from_user(const char *src, char *dst, int len,
 309                                unsigned int sum, int *errp)
 310 {
 311         unsigned long checksum = (unsigned) sum;
 312         unsigned long soff = 7 & (unsigned long) src;
 313         unsigned long doff = 7 & (unsigned long) dst;
 314
 315         if (len) {
 316                 if (!doff) {
 317                         if (!soff)
 318                                 checksum = csum_partial_cfu_aligned(
 319                                         (const unsigned long *) src,
 320                                         (unsigned long *) dst,
 321                                         len-8, checksum, errp);
 322                         else
 323                                 checksum = csum_partial_cfu_dest_aligned(
 324                                         (const unsigned long *) src,
 325                                         (unsigned long *) dst,
 326                                         soff, len-8, checksum, errp);
 327                 } else {
 328                         unsigned long partial_dest;
 329                         ldq_u(partial_dest, dst);
 330                         if (!soff)
 331                                 checksum = csum_partial_cfu_src_aligned(
 332                                         (const unsigned long *) src,
 333                                         (unsigned long *) dst,
 334                                         doff, len-8, checksum,
 335                                         partial_dest, errp);
 336                         else
 337                                 checksum = csum_partial_cfu_unaligned(
 338                                         (const unsigned long *) src,
 339                                         (unsigned long *) dst,
 340                                         soff, doff, len-8, checksum,
 341                                         partial_dest, errp);
 342                 }
 343                 /* 64 -> 33 bits */
 344                 checksum = (checksum & 0xffffffff) + (checksum >> 32);
 345                 /* 33 -> < 32 bits */
 346                 checksum = (checksum & 0xffff) + (checksum >> 16);
 347                 /* 32 -> 16 bits */
 348                 checksum = (checksum & 0xffff) + (checksum >> 16);
 349                 checksum = (checksum & 0xffff) + (checksum >> 16);
 350         }
 351         return checksum;
 352 }
 353
 354 unsigned int
 355 csum_partial_copy_from_user(const char *src, char *dst, int len,
 356                             unsigned int sum, int *errp)
 357 {
 358         if (!access_ok(src, len, VERIFY_READ)) {
 359                 *errp = -EFAULT;
 360                 memset(dst, 0, len);
 361                 return sum;
 362         }
 363
 364         return do_csum_partial_copy_from_user(src, dst, len, sum, errp);
 365 }
 366
 367 unsigned int
 368 csum_partial_copy (const char *src, char *dst, int len, unsigned int sum)
 369 {
 370         unsigned int ret;
 371         int error = 0;
 372
 373         ret = do_csum_partial_copy_from_user(src, dst, len, sum, &error);
 374         if (error)
 375                 printk("csum_partial_copy_old(): tell mingo to convert me!\n");
 376
 377         return ret;
 378 }