arch/alpha/lib/csum_partial_copy.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * csum_partial_copy - do IP checksumming and copy
   4  *
   5  * (C) Copyright 1996 Linus Torvalds
   6  * accelerated versions (and 21264 assembly versions ) contributed by
   7  *      Rick Gorton     <rick.gorton@alpha-processor.com>
   8  *
   9  * Don't look at this too closely - you'll go mad. The things
  10  * we do for performance..
  11  */
  12
  13 #include <linux/types.h>
  14 #include <linux/string.h>
  15 #include <linux/uaccess.h>
  16
  17
  18 #define ldq_u(x,y) \
  19 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
  20
  21 #define stq_u(x,y) \
  22 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
  23
  24 #define extql(x,y,z) \
  25 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  26
  27 #define extqh(x,y,z) \
  28 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  29
  30 #define mskql(x,y,z) \
  31 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  32
  33 #define mskqh(x,y,z) \
  34 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  35
  36 #define insql(x,y,z) \
  37 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  38
  39 #define insqh(x,y,z) \
  40 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  41
  42
  43 #define __get_user_u(x,ptr)                             \
  44 ({                                                      \
  45         long __guu_err;                                 \
  46         __asm__ __volatile__(                           \
  47         "1:     ldq_u %0,%2\n"                          \
  48         "2:\n"                                          \
  49         EXC(1b,2b,%0,%1)                                \
  50                 : "=r"(x), "=r"(__guu_err)              \
  51                 : "m"(__m(ptr)), "1"(0));               \
  52         __guu_err;                                      \
  53 })
  54
  55 #define __put_user_u(x,ptr)                             \
  56 ({                                                      \
  57         long __puu_err;                                 \
  58         __asm__ __volatile__(                           \
  59         "1:     stq_u %2,%1\n"                          \
  60         "2:\n"                                          \
  61         EXC(1b,2b,$31,%0)                               \
  62                 : "=r"(__puu_err)                       \
  63                 : "m"(__m(addr)), "rJ"(x), "0"(0));     \
  64         __puu_err;                                      \
  65 })
  66
  67
  68 static inline unsigned short from64to16(unsigned long x)
  69 {
  70         /* Using extract instructions is a bit more efficient
  71            than the original shift/bitmask version.  */
  72
  73         union {
  74                 unsigned long   ul;
  75                 unsigned int    ui[2];
  76                 unsigned short  us[4];
  77         } in_v, tmp_v, out_v;
  78
  79         in_v.ul = x;
  80         tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
  81
  82         /* Since the bits of tmp_v.sh[3] are going to always be zero,
  83            we don't have to bother to add that in.  */
  84         out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
  85                         + (unsigned long) tmp_v.us[2];
  86
  87         /* Similarly, out_v.us[2] is always zero for the final add.  */
  88         return out_v.us[0] + out_v.us[1];
  89 }
  90
  91
  92
  93 /*
  94  * Ok. This isn't fun, but this is the EASY case.
  95  */
  96 static inline unsigned long
  97 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
  98                          long len, unsigned long checksum,
  99                          int *errp)
 100 {
 101         unsigned long carry = 0;
 102         int err = 0;
 103
 104         while (len >= 0) {
 105                 unsigned long word;
 106                 err |= __get_user(word, src);
 107                 checksum += carry;
 108                 src++;
 109                 checksum += word;
 110                 len -= 8;
 111                 carry = checksum < word;
 112                 *dst = word;
 113                 dst++;
 114         }
 115         len += 8;
 116         checksum += carry;
 117         if (len) {
 118                 unsigned long word, tmp;
 119                 err |= __get_user(word, src);
 120                 tmp = *dst;
 121                 mskql(word, len, word);
 122                 checksum += word;
 123                 mskqh(tmp, len, tmp);
 124                 carry = checksum < word;
 125                 *dst = word | tmp;
 126                 checksum += carry;
 127         }
 128         if (err && errp) *errp = err;
 129         return checksum;
 130 }
 131
 132 /*
 133  * This is even less fun, but this is still reasonably
 134  * easy.
 135  */
 136 static inline unsigned long
 137 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
 138                               unsigned long *dst,
 139                               unsigned long soff,
 140                               long len, unsigned long checksum,
 141                               int *errp)
 142 {
 143         unsigned long first;
 144         unsigned long word, carry;
 145         unsigned long lastsrc = 7+len+(unsigned long)src;
 146         int err = 0;
 147
 148         err |= __get_user_u(first,src);
 149         carry = 0;
 150         while (len >= 0) {
 151                 unsigned long second;
 152
 153                 err |= __get_user_u(second, src+1);
 154                 extql(first, soff, word);
 155                 len -= 8;
 156                 src++;
 157                 extqh(second, soff, first);
 158                 checksum += carry;
 159                 word |= first;
 160                 first = second;
 161                 checksum += word;
 162                 *dst = word;
 163                 dst++;
 164                 carry = checksum < word;
 165         }
 166         len += 8;
 167         checksum += carry;
 168         if (len) {
 169                 unsigned long tmp;
 170                 unsigned long second;
 171                 err |= __get_user_u(second, lastsrc);
 172                 tmp = *dst;
 173                 extql(first, soff, word);
 174                 extqh(second, soff, first);
 175                 word |= first;
 176                 mskql(word, len, word);
 177                 checksum += word;
 178                 mskqh(tmp, len, tmp);
 179                 carry = checksum < word;
 180                 *dst = word | tmp;
 181                 checksum += carry;
 182         }
 183         if (err && errp) *errp = err;
 184         return checksum;
 185 }
 186
 187 /*
 188  * This is slightly less fun than the above..
 189  */
 190 static inline unsigned long
 191 csum_partial_cfu_src_aligned(const unsigned long __user *src,
 192                              unsigned long *dst,
 193                              unsigned long doff,
 194                              long len, unsigned long checksum,
 195                              unsigned long partial_dest,
 196                              int *errp)
 197 {
 198         unsigned long carry = 0;
 199         unsigned long word;
 200         unsigned long second_dest;
 201         int err = 0;
 202
 203         mskql(partial_dest, doff, partial_dest);
 204         while (len >= 0) {
 205                 err |= __get_user(word, src);
 206                 len -= 8;
 207                 insql(word, doff, second_dest);
 208                 checksum += carry;
 209                 stq_u(partial_dest | second_dest, dst);
 210                 src++;
 211                 checksum += word;
 212                 insqh(word, doff, partial_dest);
 213                 carry = checksum < word;
 214                 dst++;
 215         }
 216         len += 8;
 217         if (len) {
 218                 checksum += carry;
 219                 err |= __get_user(word, src);
 220                 mskql(word, len, word);
 221                 len -= 8;
 222                 checksum += word;
 223                 insql(word, doff, second_dest);
 224                 len += doff;
 225                 carry = checksum < word;
 226                 partial_dest |= second_dest;
 227                 if (len >= 0) {
 228                         stq_u(partial_dest, dst);
 229                         if (!len) goto out;
 230                         dst++;
 231                         insqh(word, doff, partial_dest);
 232                 }
 233                 doff = len;
 234         }
 235         ldq_u(second_dest, dst);
 236         mskqh(second_dest, doff, second_dest);
 237         stq_u(partial_dest | second_dest, dst);
 238 out:
 239         checksum += carry;
 240         if (err && errp) *errp = err;
 241         return checksum;
 242 }
 243
 244 /*
 245  * This is so totally un-fun that it's frightening. Don't
 246  * look at this too closely, you'll go blind.
 247  */
 248 static inline unsigned long
 249 csum_partial_cfu_unaligned(const unsigned long __user * src,
 250                            unsigned long * dst,
 251                            unsigned long soff, unsigned long doff,
 252                            long len, unsigned long checksum,
 253                            unsigned long partial_dest,
 254                            int *errp)
 255 {
 256         unsigned long carry = 0;
 257         unsigned long first;
 258         unsigned long lastsrc;
 259         int err = 0;
 260
 261         err |= __get_user_u(first, src);
 262         lastsrc = 7+len+(unsigned long)src;
 263         mskql(partial_dest, doff, partial_dest);
 264         while (len >= 0) {
 265                 unsigned long second, word;
 266                 unsigned long second_dest;
 267
 268                 err |= __get_user_u(second, src+1);
 269                 extql(first, soff, word);
 270                 checksum += carry;
 271                 len -= 8;
 272                 extqh(second, soff, first);
 273                 src++;
 274                 word |= first;
 275                 first = second;
 276                 insql(word, doff, second_dest);
 277                 checksum += word;
 278                 stq_u(partial_dest | second_dest, dst);
 279                 carry = checksum < word;
 280                 insqh(word, doff, partial_dest);
 281                 dst++;
 282         }
 283         len += doff;
 284         checksum += carry;
 285         if (len >= 0) {
 286                 unsigned long second, word;
 287                 unsigned long second_dest;
 288
 289                 err |= __get_user_u(second, lastsrc);
 290                 extql(first, soff, word);
 291                 extqh(second, soff, first);
 292                 word |= first;
 293                 first = second;
 294                 mskql(word, len-doff, word);
 295                 checksum += word;
 296                 insql(word, doff, second_dest);
 297                 carry = checksum < word;
 298                 stq_u(partial_dest | second_dest, dst);
 299                 if (len) {
 300                         ldq_u(second_dest, dst+1);
 301                         insqh(word, doff, partial_dest);
 302                         mskqh(second_dest, len, second_dest);
 303                         stq_u(partial_dest | second_dest, dst+1);
 304                 }
 305                 checksum += carry;
 306         } else {
 307                 unsigned long second, word;
 308                 unsigned long second_dest;
 309
 310                 err |= __get_user_u(second, lastsrc);
 311                 extql(first, soff, word);
 312                 extqh(second, soff, first);
 313                 word |= first;
 314                 ldq_u(second_dest, dst);
 315                 mskql(word, len-doff, word);
 316                 checksum += word;
 317                 mskqh(second_dest, len, second_dest);
 318                 carry = checksum < word;
 319                 insql(word, doff, word);
 320                 stq_u(partial_dest | word | second_dest, dst);
 321                 checksum += carry;
 322         }
 323         if (err && errp) *errp = err;
 324         return checksum;
 325 }
 326
 327 __wsum
 328 csum_partial_copy_from_user(const void __user *src, void *dst, int len,
 329                                __wsum sum, int *errp)
 330 {
 331         unsigned long checksum = (__force u32) sum;
 332         unsigned long soff = 7 & (unsigned long) src;
 333         unsigned long doff = 7 & (unsigned long) dst;
 334
 335         if (len) {
 336                 if (!access_ok(VERIFY_READ, src, len)) {
 337                         if (errp) *errp = -EFAULT;
 338                         memset(dst, 0, len);
 339                         return sum;
 340                 }
 341                 if (!doff) {
 342                         if (!soff)
 343                                 checksum = csum_partial_cfu_aligned(
 344                                         (const unsigned long __user *) src,
 345                                         (unsigned long *) dst,
 346                                         len-8, checksum, errp);
 347                         else
 348                                 checksum = csum_partial_cfu_dest_aligned(
 349                                         (const unsigned long __user *) src,
 350                                         (unsigned long *) dst,
 351                                         soff, len-8, checksum, errp);
 352                 } else {
 353                         unsigned long partial_dest;
 354                         ldq_u(partial_dest, dst);
 355                         if (!soff)
 356                                 checksum = csum_partial_cfu_src_aligned(
 357                                         (const unsigned long __user *) src,
 358                                         (unsigned long *) dst,
 359                                         doff, len-8, checksum,
 360                                         partial_dest, errp);
 361                         else
 362                                 checksum = csum_partial_cfu_unaligned(
 363                                         (const unsigned long __user *) src,
 364                                         (unsigned long *) dst,
 365                                         soff, doff, len-8, checksum,
 366                                         partial_dest, errp);
 367                 }
 368                 checksum = from64to16 (checksum);
 369         }
 370         return (__force __wsum)checksum;
 371 }
 372 EXPORT_SYMBOL(csum_partial_copy_from_user);
 373
 374 __wsum
 375 csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
 376 {
 377         __wsum checksum;
 378         mm_segment_t oldfs = get_fs();
 379         set_fs(KERNEL_DS);
 380         checksum = csum_partial_copy_from_user((__force const void __user *)src,
 381                                                 dst, len, sum, NULL);
 382         set_fs(oldfs);
 383         return checksum;
 384 }
 385 EXPORT_SYMBOL(csum_partial_copy_nocheck);