arch/alpha/lib/csum_partial_copy.c

   1 /*
   2  * csum_partial_copy - do IP checksumming and copy
   3  *
   4  * (C) Copyright 1996 Linus Torvalds
   5  * accelerated versions (and 21264 assembly versions ) contributed by
   6  *      Rick Gorton     <rick.gorton@alpha-processor.com>
   7  *
   8  * Don't look at this too closely - you'll go mad. The things
   9  * we do for performance..
  10  */
  11
  12 #include <linux/types.h>
  13 #include <linux/string.h>
  14 #include <linux/uaccess.h>
  15
  16
  17 #define ldq_u(x,y) \
  18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
  19
  20 #define stq_u(x,y) \
  21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
  22
  23 #define extql(x,y,z) \
  24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  25
  26 #define extqh(x,y,z) \
  27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  28
  29 #define mskql(x,y,z) \
  30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  31
  32 #define mskqh(x,y,z) \
  33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  34
  35 #define insql(x,y,z) \
  36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  37
  38 #define insqh(x,y,z) \
  39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  40
  41
  42 #define __get_user_u(x,ptr)                             \
  43 ({                                                      \
  44         long __guu_err;                                 \
  45         __asm__ __volatile__(                           \
  46         "1:     ldq_u %0,%2\n"                          \
  47         "2:\n"                                          \
  48         EXC(1b,2b,%0,%1)                                \
  49                 : "=r"(x), "=r"(__guu_err)              \
  50                 : "m"(__m(ptr)), "1"(0));               \
  51         __guu_err;                                      \
  52 })
  53
  54 #define __put_user_u(x,ptr)                             \
  55 ({                                                      \
  56         long __puu_err;                                 \
  57         __asm__ __volatile__(                           \
  58         "1:     stq_u %2,%1\n"                          \
  59         "2:\n"                                          \
  60         EXC(1b,2b,$31,%0)                               \
  61                 : "=r"(__puu_err)                       \
  62                 : "m"(__m(addr)), "rJ"(x), "0"(0));     \
  63         __puu_err;                                      \
  64 })
  65
  66
  67 static inline unsigned short from64to16(unsigned long x)
  68 {
  69         /* Using extract instructions is a bit more efficient
  70            than the original shift/bitmask version.  */
  71
  72         union {
  73                 unsigned long   ul;
  74                 unsigned int    ui[2];
  75                 unsigned short  us[4];
  76         } in_v, tmp_v, out_v;
  77
  78         in_v.ul = x;
  79         tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
  80
  81         /* Since the bits of tmp_v.sh[3] are going to always be zero,
  82            we don't have to bother to add that in.  */
  83         out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
  84                         + (unsigned long) tmp_v.us[2];
  85
  86         /* Similarly, out_v.us[2] is always zero for the final add.  */
  87         return out_v.us[0] + out_v.us[1];
  88 }
  89
  90
  91
  92 /*
  93  * Ok. This isn't fun, but this is the EASY case.
  94  */
  95 static inline unsigned long
  96 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
  97                          long len, unsigned long checksum,
  98                          int *errp)
  99 {
 100         unsigned long carry = 0;
 101         int err = 0;
 102
 103         while (len >= 0) {
 104                 unsigned long word;
 105                 err |= __get_user(word, src);
 106                 checksum += carry;
 107                 src++;
 108                 checksum += word;
 109                 len -= 8;
 110                 carry = checksum < word;
 111                 *dst = word;
 112                 dst++;
 113         }
 114         len += 8;
 115         checksum += carry;
 116         if (len) {
 117                 unsigned long word, tmp;
 118                 err |= __get_user(word, src);
 119                 tmp = *dst;
 120                 mskql(word, len, word);
 121                 checksum += word;
 122                 mskqh(tmp, len, tmp);
 123                 carry = checksum < word;
 124                 *dst = word | tmp;
 125                 checksum += carry;
 126         }
 127         if (err && errp) *errp = err;
 128         return checksum;
 129 }
 130
 131 /*
 132  * This is even less fun, but this is still reasonably
 133  * easy.
 134  */
 135 static inline unsigned long
 136 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
 137                               unsigned long *dst,
 138                               unsigned long soff,
 139                               long len, unsigned long checksum,
 140                               int *errp)
 141 {
 142         unsigned long first;
 143         unsigned long word, carry;
 144         unsigned long lastsrc = 7+len+(unsigned long)src;
 145         int err = 0;
 146
 147         err |= __get_user_u(first,src);
 148         carry = 0;
 149         while (len >= 0) {
 150                 unsigned long second;
 151
 152                 err |= __get_user_u(second, src+1);
 153                 extql(first, soff, word);
 154                 len -= 8;
 155                 src++;
 156                 extqh(second, soff, first);
 157                 checksum += carry;
 158                 word |= first;
 159                 first = second;
 160                 checksum += word;
 161                 *dst = word;
 162                 dst++;
 163                 carry = checksum < word;
 164         }
 165         len += 8;
 166         checksum += carry;
 167         if (len) {
 168                 unsigned long tmp;
 169                 unsigned long second;
 170                 err |= __get_user_u(second, lastsrc);
 171                 tmp = *dst;
 172                 extql(first, soff, word);
 173                 extqh(second, soff, first);
 174                 word |= first;
 175                 mskql(word, len, word);
 176                 checksum += word;
 177                 mskqh(tmp, len, tmp);
 178                 carry = checksum < word;
 179                 *dst = word | tmp;
 180                 checksum += carry;
 181         }
 182         if (err && errp) *errp = err;
 183         return checksum;
 184 }
 185
 186 /*
 187  * This is slightly less fun than the above..
 188  */
 189 static inline unsigned long
 190 csum_partial_cfu_src_aligned(const unsigned long __user *src,
 191                              unsigned long *dst,
 192                              unsigned long doff,
 193                              long len, unsigned long checksum,
 194                              unsigned long partial_dest,
 195                              int *errp)
 196 {
 197         unsigned long carry = 0;
 198         unsigned long word;
 199         unsigned long second_dest;
 200         int err = 0;
 201
 202         mskql(partial_dest, doff, partial_dest);
 203         while (len >= 0) {
 204                 err |= __get_user(word, src);
 205                 len -= 8;
 206                 insql(word, doff, second_dest);
 207                 checksum += carry;
 208                 stq_u(partial_dest | second_dest, dst);
 209                 src++;
 210                 checksum += word;
 211                 insqh(word, doff, partial_dest);
 212                 carry = checksum < word;
 213                 dst++;
 214         }
 215         len += 8;
 216         if (len) {
 217                 checksum += carry;
 218                 err |= __get_user(word, src);
 219                 mskql(word, len, word);
 220                 len -= 8;
 221                 checksum += word;
 222                 insql(word, doff, second_dest);
 223                 len += doff;
 224                 carry = checksum < word;
 225                 partial_dest |= second_dest;
 226                 if (len >= 0) {
 227                         stq_u(partial_dest, dst);
 228                         if (!len) goto out;
 229                         dst++;
 230                         insqh(word, doff, partial_dest);
 231                 }
 232                 doff = len;
 233         }
 234         ldq_u(second_dest, dst);
 235         mskqh(second_dest, doff, second_dest);
 236         stq_u(partial_dest | second_dest, dst);
 237 out:
 238         checksum += carry;
 239         if (err && errp) *errp = err;
 240         return checksum;
 241 }
 242
 243 /*
 244  * This is so totally un-fun that it's frightening. Don't
 245  * look at this too closely, you'll go blind.
 246  */
 247 static inline unsigned long
 248 csum_partial_cfu_unaligned(const unsigned long __user * src,
 249                            unsigned long * dst,
 250                            unsigned long soff, unsigned long doff,
 251                            long len, unsigned long checksum,
 252                            unsigned long partial_dest,
 253                            int *errp)
 254 {
 255         unsigned long carry = 0;
 256         unsigned long first;
 257         unsigned long lastsrc;
 258         int err = 0;
 259
 260         err |= __get_user_u(first, src);
 261         lastsrc = 7+len+(unsigned long)src;
 262         mskql(partial_dest, doff, partial_dest);
 263         while (len >= 0) {
 264                 unsigned long second, word;
 265                 unsigned long second_dest;
 266
 267                 err |= __get_user_u(second, src+1);
 268                 extql(first, soff, word);
 269                 checksum += carry;
 270                 len -= 8;
 271                 extqh(second, soff, first);
 272                 src++;
 273                 word |= first;
 274                 first = second;
 275                 insql(word, doff, second_dest);
 276                 checksum += word;
 277                 stq_u(partial_dest | second_dest, dst);
 278                 carry = checksum < word;
 279                 insqh(word, doff, partial_dest);
 280                 dst++;
 281         }
 282         len += doff;
 283         checksum += carry;
 284         if (len >= 0) {
 285                 unsigned long second, word;
 286                 unsigned long second_dest;
 287
 288                 err |= __get_user_u(second, lastsrc);
 289                 extql(first, soff, word);
 290                 extqh(second, soff, first);
 291                 word |= first;
 292                 first = second;
 293                 mskql(word, len-doff, word);
 294                 checksum += word;
 295                 insql(word, doff, second_dest);
 296                 carry = checksum < word;
 297                 stq_u(partial_dest | second_dest, dst);
 298                 if (len) {
 299                         ldq_u(second_dest, dst+1);
 300                         insqh(word, doff, partial_dest);
 301                         mskqh(second_dest, len, second_dest);
 302                         stq_u(partial_dest | second_dest, dst+1);
 303                 }
 304                 checksum += carry;
 305         } else {
 306                 unsigned long second, word;
 307                 unsigned long second_dest;
 308
 309                 err |= __get_user_u(second, lastsrc);
 310                 extql(first, soff, word);
 311                 extqh(second, soff, first);
 312                 word |= first;
 313                 ldq_u(second_dest, dst);
 314                 mskql(word, len-doff, word);
 315                 checksum += word;
 316                 mskqh(second_dest, len, second_dest);
 317                 carry = checksum < word;
 318                 insql(word, doff, word);
 319                 stq_u(partial_dest | word | second_dest, dst);
 320                 checksum += carry;
 321         }
 322         if (err && errp) *errp = err;
 323         return checksum;
 324 }
 325
 326 __wsum
 327 csum_partial_copy_from_user(const void __user *src, void *dst, int len,
 328                                __wsum sum, int *errp)
 329 {
 330         unsigned long checksum = (__force u32) sum;
 331         unsigned long soff = 7 & (unsigned long) src;
 332         unsigned long doff = 7 & (unsigned long) dst;
 333
 334         if (len) {
 335                 if (!access_ok(VERIFY_READ, src, len)) {
 336                         if (errp) *errp = -EFAULT;
 337                         memset(dst, 0, len);
 338                         return sum;
 339                 }
 340                 if (!doff) {
 341                         if (!soff)
 342                                 checksum = csum_partial_cfu_aligned(
 343                                         (const unsigned long __user *) src,
 344                                         (unsigned long *) dst,
 345                                         len-8, checksum, errp);
 346                         else
 347                                 checksum = csum_partial_cfu_dest_aligned(
 348                                         (const unsigned long __user *) src,
 349                                         (unsigned long *) dst,
 350                                         soff, len-8, checksum, errp);
 351                 } else {
 352                         unsigned long partial_dest;
 353                         ldq_u(partial_dest, dst);
 354                         if (!soff)
 355                                 checksum = csum_partial_cfu_src_aligned(
 356                                         (const unsigned long __user *) src,
 357                                         (unsigned long *) dst,
 358                                         doff, len-8, checksum,
 359                                         partial_dest, errp);
 360                         else
 361                                 checksum = csum_partial_cfu_unaligned(
 362                                         (const unsigned long __user *) src,
 363                                         (unsigned long *) dst,
 364                                         soff, doff, len-8, checksum,
 365                                         partial_dest, errp);
 366                 }
 367                 checksum = from64to16 (checksum);
 368         }
 369         return (__force __wsum)checksum;
 370 }
 371 EXPORT_SYMBOL(csum_partial_copy_from_user);
 372
 373 __wsum
 374 csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
 375 {
 376         __wsum checksum;
 377         mm_segment_t oldfs = get_fs();
 378         set_fs(KERNEL_DS);
 379         checksum = csum_partial_copy_from_user((__force const void __user *)src,
 380                                                 dst, len, sum, NULL);
 381         set_fs(oldfs);
 382         return checksum;
 383 }
 384 EXPORT_SYMBOL(csum_partial_copy_nocheck);