arch/x86/lib/usercopy_32.c

   1 /*
   2  * User address space access functions.
   3  * The non inlined parts of asm-i386/uaccess.h are here.
   4  *
   5  * Copyright 1997 Andi Kleen <ak@muc.de>
   6  * Copyright 1997 Linus Torvalds
   7  */
   8 #include <linux/mm.h>
   9 #include <linux/highmem.h>
  10 #include <linux/blkdev.h>
  11 #include <linux/module.h>
  12 #include <linux/backing-dev.h>
  13 #include <linux/interrupt.h>
  14 #include <asm/uaccess.h>
  15 #include <asm/mmx.h>
  16
  17 #ifdef CONFIG_X86_INTEL_USERCOPY
  18 /*
  19  * Alignment at which movsl is preferred for bulk memory copies.
  20  */
  21 struct movsl_mask movsl_mask __read_mostly;
  22 #endif
  23
  24 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
  25 {
  26 #ifdef CONFIG_X86_INTEL_USERCOPY
  27         if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
  28                 return 0;
  29 #endif
  30         return 1;
  31 }
  32 #define movsl_is_ok(a1, a2, n) \
  33         __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
  34
  35 /*
  36  * Copy a null terminated string from userspace.
  37  */
  38
  39 #define __do_strncpy_from_user(dst, src, count, res)                       \
  40 do {                                                                       \
  41         int __d0, __d1, __d2;                                              \
  42         might_fault();                                                     \
  43         __asm__ __volatile__(                                              \
  44                 "       testl %1,%1\n"                                     \
  45                 "       jz 2f\n"                                           \
  46                 "0:     lodsb\n"                                           \
  47                 "       stosb\n"                                           \
  48                 "       testb %%al,%%al\n"                                 \
  49                 "       jz 1f\n"                                           \
  50                 "       decl %1\n"                                         \
  51                 "       jnz 0b\n"                                          \
  52                 "1:     subl %1,%0\n"                                      \
  53                 "2:\n"                                                     \
  54                 ".section .fixup,\"ax\"\n"                                 \
  55                 "3:     movl %5,%0\n"                                      \
  56                 "       jmp 2b\n"                                          \
  57                 ".previous\n"                                              \
  58                 _ASM_EXTABLE(0b,3b)                                        \
  59                 : "=&d"(res), "=&c"(count), "=&a" (__d0), "=&S" (__d1),    \
  60                   "=&D" (__d2)                                             \
  61                 : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
  62                 : "memory");                                               \
  63 } while (0)
  64
  65 /**
  66  * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
  67  * @dst:   Destination address, in kernel space.  This buffer must be at
  68  *         least @count bytes long.
  69  * @src:   Source address, in user space.
  70  * @count: Maximum number of bytes to copy, including the trailing NUL.
  71  *
  72  * Copies a NUL-terminated string from userspace to kernel space.
  73  * Caller must check the specified block with access_ok() before calling
  74  * this function.
  75  *
  76  * On success, returns the length of the string (not including the trailing
  77  * NUL).
  78  *
  79  * If access to userspace fails, returns -EFAULT (some data may have been
  80  * copied).
  81  *
  82  * If @count is smaller than the length of the string, copies @count bytes
  83  * and returns @count.
  84  */
  85 long
  86 __strncpy_from_user(char *dst, const char __user *src, long count)
  87 {
  88         long res;
  89         __do_strncpy_from_user(dst, src, count, res);
  90         return res;
  91 }
  92 EXPORT_SYMBOL(__strncpy_from_user);
  93
  94 /**
  95  * strncpy_from_user: - Copy a NUL terminated string from userspace.
  96  * @dst:   Destination address, in kernel space.  This buffer must be at
  97  *         least @count bytes long.
  98  * @src:   Source address, in user space.
  99  * @count: Maximum number of bytes to copy, including the trailing NUL.
 100  *
 101  * Copies a NUL-terminated string from userspace to kernel space.
 102  *
 103  * On success, returns the length of the string (not including the trailing
 104  * NUL).
 105  *
 106  * If access to userspace fails, returns -EFAULT (some data may have been
 107  * copied).
 108  *
 109  * If @count is smaller than the length of the string, copies @count bytes
 110  * and returns @count.
 111  */
 112 long
 113 strncpy_from_user(char *dst, const char __user *src, long count)
 114 {
 115         long res = -EFAULT;
 116         if (access_ok(VERIFY_READ, src, 1))
 117                 __do_strncpy_from_user(dst, src, count, res);
 118         return res;
 119 }
 120 EXPORT_SYMBOL(strncpy_from_user);
 121
 122 /*
 123  * Zero Userspace
 124  */
 125
 126 #define __do_clear_user(addr,size)                                      \
 127 do {                                                                    \
 128         int __d0;                                                       \
 129         might_fault();                                                  \
 130         __asm__ __volatile__(                                           \
 131                 "0:     rep; stosl\n"                                   \
 132                 "       movl %2,%0\n"                                   \
 133                 "1:     rep; stosb\n"                                   \
 134                 "2:\n"                                                  \
 135                 ".section .fixup,\"ax\"\n"                              \
 136                 "3:     lea 0(%2,%0,4),%0\n"                            \
 137                 "       jmp 2b\n"                                       \
 138                 ".previous\n"                                           \
 139                 _ASM_EXTABLE(0b,3b)                                     \
 140                 _ASM_EXTABLE(1b,2b)                                     \
 141                 : "=&c"(size), "=&D" (__d0)                             \
 142                 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));     \
 143 } while (0)
 144
 145 /**
 146  * clear_user: - Zero a block of memory in user space.
 147  * @to:   Destination address, in user space.
 148  * @n:    Number of bytes to zero.
 149  *
 150  * Zero a block of memory in user space.
 151  *
 152  * Returns number of bytes that could not be cleared.
 153  * On success, this will be zero.
 154  */
 155 unsigned long
 156 clear_user(void __user *to, unsigned long n)
 157 {
 158         might_fault();
 159         if (access_ok(VERIFY_WRITE, to, n))
 160                 __do_clear_user(to, n);
 161         return n;
 162 }
 163 EXPORT_SYMBOL(clear_user);
 164
 165 /**
 166  * __clear_user: - Zero a block of memory in user space, with less checking.
 167  * @to:   Destination address, in user space.
 168  * @n:    Number of bytes to zero.
 169  *
 170  * Zero a block of memory in user space.  Caller must check
 171  * the specified block with access_ok() before calling this function.
 172  *
 173  * Returns number of bytes that could not be cleared.
 174  * On success, this will be zero.
 175  */
 176 unsigned long
 177 __clear_user(void __user *to, unsigned long n)
 178 {
 179         __do_clear_user(to, n);
 180         return n;
 181 }
 182 EXPORT_SYMBOL(__clear_user);
 183
 184 /**
 185  * strnlen_user: - Get the size of a string in user space.
 186  * @s: The string to measure.
 187  * @n: The maximum valid length
 188  *
 189  * Get the size of a NUL-terminated string in user space.
 190  *
 191  * Returns the size of the string INCLUDING the terminating NUL.
 192  * On exception, returns 0.
 193  * If the string is too long, returns a value greater than @n.
 194  */
 195 long strnlen_user(const char __user *s, long n)
 196 {
 197         unsigned long mask = -__addr_ok(s);
 198         unsigned long res, tmp;
 199
 200         might_fault();
 201
 202         __asm__ __volatile__(
 203                 "       testl %0, %0\n"
 204                 "       jz 3f\n"
 205                 "       andl %0,%%ecx\n"
 206                 "0:     repne; scasb\n"
 207                 "       setne %%al\n"
 208                 "       subl %%ecx,%0\n"
 209                 "       addl %0,%%eax\n"
 210                 "1:\n"
 211                 ".section .fixup,\"ax\"\n"
 212                 "2:     xorl %%eax,%%eax\n"
 213                 "       jmp 1b\n"
 214                 "3:     movb $1,%%al\n"
 215                 "       jmp 1b\n"
 216                 ".previous\n"
 217                 ".section __ex_table,\"a\"\n"
 218                 "       .align 4\n"
 219                 "       .long 0b,2b\n"
 220                 ".previous"
 221                 :"=&r" (n), "=&D" (s), "=&a" (res), "=&c" (tmp)
 222                 :"0" (n), "1" (s), "2" (0), "3" (mask)
 223                 :"cc");
 224         return res & mask;
 225 }
 226 EXPORT_SYMBOL(strnlen_user);
 227
 228 #ifdef CONFIG_X86_INTEL_USERCOPY
 229 static unsigned long
 230 __copy_user_intel(void __user *to, const void *from, unsigned long size)
 231 {
 232         int d0, d1;
 233         __asm__ __volatile__(
 234                        "       .align 2,0x90\n"
 235                        "1:     movl 32(%4), %%eax\n"
 236                        "       cmpl $67, %0\n"
 237                        "       jbe 3f\n"
 238                        "2:     movl 64(%4), %%eax\n"
 239                        "       .align 2,0x90\n"
 240                        "3:     movl 0(%4), %%eax\n"
 241                        "4:     movl 4(%4), %%edx\n"
 242                        "5:     movl %%eax, 0(%3)\n"
 243                        "6:     movl %%edx, 4(%3)\n"
 244                        "7:     movl 8(%4), %%eax\n"
 245                        "8:     movl 12(%4),%%edx\n"
 246                        "9:     movl %%eax, 8(%3)\n"
 247                        "10:    movl %%edx, 12(%3)\n"
 248                        "11:    movl 16(%4), %%eax\n"
 249                        "12:    movl 20(%4), %%edx\n"
 250                        "13:    movl %%eax, 16(%3)\n"
 251                        "14:    movl %%edx, 20(%3)\n"
 252                        "15:    movl 24(%4), %%eax\n"
 253                        "16:    movl 28(%4), %%edx\n"
 254                        "17:    movl %%eax, 24(%3)\n"
 255                        "18:    movl %%edx, 28(%3)\n"
 256                        "19:    movl 32(%4), %%eax\n"
 257                        "20:    movl 36(%4), %%edx\n"
 258                        "21:    movl %%eax, 32(%3)\n"
 259                        "22:    movl %%edx, 36(%3)\n"
 260                        "23:    movl 40(%4), %%eax\n"
 261                        "24:    movl 44(%4), %%edx\n"
 262                        "25:    movl %%eax, 40(%3)\n"
 263                        "26:    movl %%edx, 44(%3)\n"
 264                        "27:    movl 48(%4), %%eax\n"
 265                        "28:    movl 52(%4), %%edx\n"
 266                        "29:    movl %%eax, 48(%3)\n"
 267                        "30:    movl %%edx, 52(%3)\n"
 268                        "31:    movl 56(%4), %%eax\n"
 269                        "32:    movl 60(%4), %%edx\n"
 270                        "33:    movl %%eax, 56(%3)\n"
 271                        "34:    movl %%edx, 60(%3)\n"
 272                        "       addl $-64, %0\n"
 273                        "       addl $64, %4\n"
 274                        "       addl $64, %3\n"
 275                        "       cmpl $63, %0\n"
 276                        "       ja  1b\n"
 277                        "35:    movl  %0, %%eax\n"
 278                        "       shrl  $2, %0\n"
 279                        "       andl  $3, %%eax\n"
 280                        "       cld\n"
 281                        "99:    rep; movsl\n"
 282                        "36:    movl %%eax, %0\n"
 283                        "37:    rep; movsb\n"
 284                        "100:\n"
 285                        ".section .fixup,\"ax\"\n"
 286                        "101:   lea 0(%%eax,%0,4),%0\n"
 287                        "       jmp 100b\n"
 288                        ".previous\n"
 289                        ".section __ex_table,\"a\"\n"
 290                        "       .align 4\n"
 291                        "       .long 1b,100b\n"
 292                        "       .long 2b,100b\n"
 293                        "       .long 3b,100b\n"
 294                        "       .long 4b,100b\n"
 295                        "       .long 5b,100b\n"
 296                        "       .long 6b,100b\n"
 297                        "       .long 7b,100b\n"
 298                        "       .long 8b,100b\n"
 299                        "       .long 9b,100b\n"
 300                        "       .long 10b,100b\n"
 301                        "       .long 11b,100b\n"
 302                        "       .long 12b,100b\n"
 303                        "       .long 13b,100b\n"
 304                        "       .long 14b,100b\n"
 305                        "       .long 15b,100b\n"
 306                        "       .long 16b,100b\n"
 307                        "       .long 17b,100b\n"
 308                        "       .long 18b,100b\n"
 309                        "       .long 19b,100b\n"
 310                        "       .long 20b,100b\n"
 311                        "       .long 21b,100b\n"
 312                        "       .long 22b,100b\n"
 313                        "       .long 23b,100b\n"
 314                        "       .long 24b,100b\n"
 315                        "       .long 25b,100b\n"
 316                        "       .long 26b,100b\n"
 317                        "       .long 27b,100b\n"
 318                        "       .long 28b,100b\n"
 319                        "       .long 29b,100b\n"
 320                        "       .long 30b,100b\n"
 321                        "       .long 31b,100b\n"
 322                        "       .long 32b,100b\n"
 323                        "       .long 33b,100b\n"
 324                        "       .long 34b,100b\n"
 325                        "       .long 35b,100b\n"
 326                        "       .long 36b,100b\n"
 327                        "       .long 37b,100b\n"
 328                        "       .long 99b,101b\n"
 329                        ".previous"
 330                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 331                        :  "1"(to), "2"(from), "0"(size)
 332                        : "eax", "edx", "memory");
 333         return size;
 334 }
 335
 336 static unsigned long
 337 __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size)
 338 {
 339         int d0, d1;
 340         __asm__ __volatile__(
 341                        "        .align 2,0x90\n"
 342                        "0:      movl 32(%4), %%eax\n"
 343                        "        cmpl $67, %0\n"
 344                        "        jbe 2f\n"
 345                        "1:      movl 64(%4), %%eax\n"
 346                        "        .align 2,0x90\n"
 347                        "2:      movl 0(%4), %%eax\n"
 348                        "21:     movl 4(%4), %%edx\n"
 349                        "        movl %%eax, 0(%3)\n"
 350                        "        movl %%edx, 4(%3)\n"
 351                        "3:      movl 8(%4), %%eax\n"
 352                        "31:     movl 12(%4),%%edx\n"
 353                        "        movl %%eax, 8(%3)\n"
 354                        "        movl %%edx, 12(%3)\n"
 355                        "4:      movl 16(%4), %%eax\n"
 356                        "41:     movl 20(%4), %%edx\n"
 357                        "        movl %%eax, 16(%3)\n"
 358                        "        movl %%edx, 20(%3)\n"
 359                        "10:     movl 24(%4), %%eax\n"
 360                        "51:     movl 28(%4), %%edx\n"
 361                        "        movl %%eax, 24(%3)\n"
 362                        "        movl %%edx, 28(%3)\n"
 363                        "11:     movl 32(%4), %%eax\n"
 364                        "61:     movl 36(%4), %%edx\n"
 365                        "        movl %%eax, 32(%3)\n"
 366                        "        movl %%edx, 36(%3)\n"
 367                        "12:     movl 40(%4), %%eax\n"
 368                        "71:     movl 44(%4), %%edx\n"
 369                        "        movl %%eax, 40(%3)\n"
 370                        "        movl %%edx, 44(%3)\n"
 371                        "13:     movl 48(%4), %%eax\n"
 372                        "81:     movl 52(%4), %%edx\n"
 373                        "        movl %%eax, 48(%3)\n"
 374                        "        movl %%edx, 52(%3)\n"
 375                        "14:     movl 56(%4), %%eax\n"
 376                        "91:     movl 60(%4), %%edx\n"
 377                        "        movl %%eax, 56(%3)\n"
 378                        "        movl %%edx, 60(%3)\n"
 379                        "        addl $-64, %0\n"
 380                        "        addl $64, %4\n"
 381                        "        addl $64, %3\n"
 382                        "        cmpl $63, %0\n"
 383                        "        ja  0b\n"
 384                        "5:      movl  %0, %%eax\n"
 385                        "        shrl  $2, %0\n"
 386                        "        andl $3, %%eax\n"
 387                        "        cld\n"
 388                        "6:      rep; movsl\n"
 389                        "        movl %%eax,%0\n"
 390                        "7:      rep; movsb\n"
 391                        "8:\n"
 392                        ".section .fixup,\"ax\"\n"
 393                        "9:      lea 0(%%eax,%0,4),%0\n"
 394                        "16:     pushl %0\n"
 395                        "        pushl %%eax\n"
 396                        "        xorl %%eax,%%eax\n"
 397                        "        rep; stosb\n"
 398                        "        popl %%eax\n"
 399                        "        popl %0\n"
 400                        "        jmp 8b\n"
 401                        ".previous\n"
 402                        ".section __ex_table,\"a\"\n"
 403                        "        .align 4\n"
 404                        "        .long 0b,16b\n"
 405                        "        .long 1b,16b\n"
 406                        "        .long 2b,16b\n"
 407                        "        .long 21b,16b\n"
 408                        "        .long 3b,16b\n"
 409                        "        .long 31b,16b\n"
 410                        "        .long 4b,16b\n"
 411                        "        .long 41b,16b\n"
 412                        "        .long 10b,16b\n"
 413                        "        .long 51b,16b\n"
 414                        "        .long 11b,16b\n"
 415                        "        .long 61b,16b\n"
 416                        "        .long 12b,16b\n"
 417                        "        .long 71b,16b\n"
 418                        "        .long 13b,16b\n"
 419                        "        .long 81b,16b\n"
 420                        "        .long 14b,16b\n"
 421                        "        .long 91b,16b\n"
 422                        "        .long 6b,9b\n"
 423                        "        .long 7b,16b\n"
 424                        ".previous"
 425                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 426                        :  "1"(to), "2"(from), "0"(size)
 427                        : "eax", "edx", "memory");
 428         return size;
 429 }
 430
 431 /*
 432  * Non Temporal Hint version of __copy_user_zeroing_intel.  It is cache aware.
 433  * hyoshiok@miraclelinux.com
 434  */
 435
 436 static unsigned long __copy_user_zeroing_intel_nocache(void *to,
 437                                 const void __user *from, unsigned long size)
 438 {
 439         int d0, d1;
 440
 441         __asm__ __volatile__(
 442                "        .align 2,0x90\n"
 443                "0:      movl 32(%4), %%eax\n"
 444                "        cmpl $67, %0\n"
 445                "        jbe 2f\n"
 446                "1:      movl 64(%4), %%eax\n"
 447                "        .align 2,0x90\n"
 448                "2:      movl 0(%4), %%eax\n"
 449                "21:     movl 4(%4), %%edx\n"
 450                "        movnti %%eax, 0(%3)\n"
 451                "        movnti %%edx, 4(%3)\n"
 452                "3:      movl 8(%4), %%eax\n"
 453                "31:     movl 12(%4),%%edx\n"
 454                "        movnti %%eax, 8(%3)\n"
 455                "        movnti %%edx, 12(%3)\n"
 456                "4:      movl 16(%4), %%eax\n"
 457                "41:     movl 20(%4), %%edx\n"
 458                "        movnti %%eax, 16(%3)\n"
 459                "        movnti %%edx, 20(%3)\n"
 460                "10:     movl 24(%4), %%eax\n"
 461                "51:     movl 28(%4), %%edx\n"
 462                "        movnti %%eax, 24(%3)\n"
 463                "        movnti %%edx, 28(%3)\n"
 464                "11:     movl 32(%4), %%eax\n"
 465                "61:     movl 36(%4), %%edx\n"
 466                "        movnti %%eax, 32(%3)\n"
 467                "        movnti %%edx, 36(%3)\n"
 468                "12:     movl 40(%4), %%eax\n"
 469                "71:     movl 44(%4), %%edx\n"
 470                "        movnti %%eax, 40(%3)\n"
 471                "        movnti %%edx, 44(%3)\n"
 472                "13:     movl 48(%4), %%eax\n"
 473                "81:     movl 52(%4), %%edx\n"
 474                "        movnti %%eax, 48(%3)\n"
 475                "        movnti %%edx, 52(%3)\n"
 476                "14:     movl 56(%4), %%eax\n"
 477                "91:     movl 60(%4), %%edx\n"
 478                "        movnti %%eax, 56(%3)\n"
 479                "        movnti %%edx, 60(%3)\n"
 480                "        addl $-64, %0\n"
 481                "        addl $64, %4\n"
 482                "        addl $64, %3\n"
 483                "        cmpl $63, %0\n"
 484                "        ja  0b\n"
 485                "        sfence \n"
 486                "5:      movl  %0, %%eax\n"
 487                "        shrl  $2, %0\n"
 488                "        andl $3, %%eax\n"
 489                "        cld\n"
 490                "6:      rep; movsl\n"
 491                "        movl %%eax,%0\n"
 492                "7:      rep; movsb\n"
 493                "8:\n"
 494                ".section .fixup,\"ax\"\n"
 495                "9:      lea 0(%%eax,%0,4),%0\n"
 496                "16:     pushl %0\n"
 497                "        pushl %%eax\n"
 498                "        xorl %%eax,%%eax\n"
 499                "        rep; stosb\n"
 500                "        popl %%eax\n"
 501                "        popl %0\n"
 502                "        jmp 8b\n"
 503                ".previous\n"
 504                ".section __ex_table,\"a\"\n"
 505                "        .align 4\n"
 506                "        .long 0b,16b\n"
 507                "        .long 1b,16b\n"
 508                "        .long 2b,16b\n"
 509                "        .long 21b,16b\n"
 510                "        .long 3b,16b\n"
 511                "        .long 31b,16b\n"
 512                "        .long 4b,16b\n"
 513                "        .long 41b,16b\n"
 514                "        .long 10b,16b\n"
 515                "        .long 51b,16b\n"
 516                "        .long 11b,16b\n"
 517                "        .long 61b,16b\n"
 518                "        .long 12b,16b\n"
 519                "        .long 71b,16b\n"
 520                "        .long 13b,16b\n"
 521                "        .long 81b,16b\n"
 522                "        .long 14b,16b\n"
 523                "        .long 91b,16b\n"
 524                "        .long 6b,9b\n"
 525                "        .long 7b,16b\n"
 526                ".previous"
 527                : "=&c"(size), "=&D" (d0), "=&S" (d1)
 528                :  "1"(to), "2"(from), "0"(size)
 529                : "eax", "edx", "memory");
 530         return size;
 531 }
 532
 533 static unsigned long __copy_user_intel_nocache(void *to,
 534                                 const void __user *from, unsigned long size)
 535 {
 536         int d0, d1;
 537
 538         __asm__ __volatile__(
 539                "        .align 2,0x90\n"
 540                "0:      movl 32(%4), %%eax\n"
 541                "        cmpl $67, %0\n"
 542                "        jbe 2f\n"
 543                "1:      movl 64(%4), %%eax\n"
 544                "        .align 2,0x90\n"
 545                "2:      movl 0(%4), %%eax\n"
 546                "21:     movl 4(%4), %%edx\n"
 547                "        movnti %%eax, 0(%3)\n"
 548                "        movnti %%edx, 4(%3)\n"
 549                "3:      movl 8(%4), %%eax\n"
 550                "31:     movl 12(%4),%%edx\n"
 551                "        movnti %%eax, 8(%3)\n"
 552                "        movnti %%edx, 12(%3)\n"
 553                "4:      movl 16(%4), %%eax\n"
 554                "41:     movl 20(%4), %%edx\n"
 555                "        movnti %%eax, 16(%3)\n"
 556                "        movnti %%edx, 20(%3)\n"
 557                "10:     movl 24(%4), %%eax\n"
 558                "51:     movl 28(%4), %%edx\n"
 559                "        movnti %%eax, 24(%3)\n"
 560                "        movnti %%edx, 28(%3)\n"
 561                "11:     movl 32(%4), %%eax\n"
 562                "61:     movl 36(%4), %%edx\n"
 563                "        movnti %%eax, 32(%3)\n"
 564                "        movnti %%edx, 36(%3)\n"
 565                "12:     movl 40(%4), %%eax\n"
 566                "71:     movl 44(%4), %%edx\n"
 567                "        movnti %%eax, 40(%3)\n"
 568                "        movnti %%edx, 44(%3)\n"
 569                "13:     movl 48(%4), %%eax\n"
 570                "81:     movl 52(%4), %%edx\n"
 571                "        movnti %%eax, 48(%3)\n"
 572                "        movnti %%edx, 52(%3)\n"
 573                "14:     movl 56(%4), %%eax\n"
 574                "91:     movl 60(%4), %%edx\n"
 575                "        movnti %%eax, 56(%3)\n"
 576                "        movnti %%edx, 60(%3)\n"
 577                "        addl $-64, %0\n"
 578                "        addl $64, %4\n"
 579                "        addl $64, %3\n"
 580                "        cmpl $63, %0\n"
 581                "        ja  0b\n"
 582                "        sfence \n"
 583                "5:      movl  %0, %%eax\n"
 584                "        shrl  $2, %0\n"
 585                "        andl $3, %%eax\n"
 586                "        cld\n"
 587                "6:      rep; movsl\n"
 588                "        movl %%eax,%0\n"
 589                "7:      rep; movsb\n"
 590                "8:\n"
 591                ".section .fixup,\"ax\"\n"
 592                "9:      lea 0(%%eax,%0,4),%0\n"
 593                "16:     jmp 8b\n"
 594                ".previous\n"
 595                ".section __ex_table,\"a\"\n"
 596                "        .align 4\n"
 597                "        .long 0b,16b\n"
 598                "        .long 1b,16b\n"
 599                "        .long 2b,16b\n"
 600                "        .long 21b,16b\n"
 601                "        .long 3b,16b\n"
 602                "        .long 31b,16b\n"
 603                "        .long 4b,16b\n"
 604                "        .long 41b,16b\n"
 605                "        .long 10b,16b\n"
 606                "        .long 51b,16b\n"
 607                "        .long 11b,16b\n"
 608                "        .long 61b,16b\n"
 609                "        .long 12b,16b\n"
 610                "        .long 71b,16b\n"
 611                "        .long 13b,16b\n"
 612                "        .long 81b,16b\n"
 613                "        .long 14b,16b\n"
 614                "        .long 91b,16b\n"
 615                "        .long 6b,9b\n"
 616                "        .long 7b,16b\n"
 617                ".previous"
 618                : "=&c"(size), "=&D" (d0), "=&S" (d1)
 619                :  "1"(to), "2"(from), "0"(size)
 620                : "eax", "edx", "memory");
 621         return size;
 622 }
 623
 624 #else
 625
 626 /*
 627  * Leave these declared but undefined.  They should not be any references to
 628  * them
 629  */
 630 unsigned long __copy_user_zeroing_intel(void *to, const void __user *from,
 631                                         unsigned long size);
 632 unsigned long __copy_user_intel(void __user *to, const void *from,
 633                                         unsigned long size);
 634 unsigned long __copy_user_zeroing_intel_nocache(void *to,
 635                                 const void __user *from, unsigned long size);
 636 #endif /* CONFIG_X86_INTEL_USERCOPY */
 637
 638 /* Generic arbitrary sized copy.  */
 639 #define __copy_user(to, from, size)                                     \
 640 do {                                                                    \
 641         int __d0, __d1, __d2;                                           \
 642         __asm__ __volatile__(                                           \
 643                 "       cmp  $7,%0\n"                                   \
 644                 "       jbe  1f\n"                                      \
 645                 "       movl %1,%0\n"                                   \
 646                 "       negl %0\n"                                      \
 647                 "       andl $7,%0\n"                                   \
 648                 "       subl %0,%3\n"                                   \
 649                 "4:     rep; movsb\n"                                   \
 650                 "       movl %3,%0\n"                                   \
 651                 "       shrl $2,%0\n"                                   \
 652                 "       andl $3,%3\n"                                   \
 653                 "       .align 2,0x90\n"                                \
 654                 "0:     rep; movsl\n"                                   \
 655                 "       movl %3,%0\n"                                   \
 656                 "1:     rep; movsb\n"                                   \
 657                 "2:\n"                                                  \
 658                 ".section .fixup,\"ax\"\n"                              \
 659                 "5:     addl %3,%0\n"                                   \
 660                 "       jmp 2b\n"                                       \
 661                 "3:     lea 0(%3,%0,4),%0\n"                            \
 662                 "       jmp 2b\n"                                       \
 663                 ".previous\n"                                           \
 664                 ".section __ex_table,\"a\"\n"                           \
 665                 "       .align 4\n"                                     \
 666                 "       .long 4b,5b\n"                                  \
 667                 "       .long 0b,3b\n"                                  \
 668                 "       .long 1b,2b\n"                                  \
 669                 ".previous"                                             \
 670                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 671                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 672                 : "memory");                                            \
 673 } while (0)
 674
 675 #define __copy_user_zeroing(to, from, size)                             \
 676 do {                                                                    \
 677         int __d0, __d1, __d2;                                           \
 678         __asm__ __volatile__(                                           \
 679                 "       cmp  $7,%0\n"                                   \
 680                 "       jbe  1f\n"                                      \
 681                 "       movl %1,%0\n"                                   \
 682                 "       negl %0\n"                                      \
 683                 "       andl $7,%0\n"                                   \
 684                 "       subl %0,%3\n"                                   \
 685                 "4:     rep; movsb\n"                                   \
 686                 "       movl %3,%0\n"                                   \
 687                 "       shrl $2,%0\n"                                   \
 688                 "       andl $3,%3\n"                                   \
 689                 "       .align 2,0x90\n"                                \
 690                 "0:     rep; movsl\n"                                   \
 691                 "       movl %3,%0\n"                                   \
 692                 "1:     rep; movsb\n"                                   \
 693                 "2:\n"                                                  \
 694                 ".section .fixup,\"ax\"\n"                              \
 695                 "5:     addl %3,%0\n"                                   \
 696                 "       jmp 6f\n"                                       \
 697                 "3:     lea 0(%3,%0,4),%0\n"                            \
 698                 "6:     pushl %0\n"                                     \
 699                 "       pushl %%eax\n"                                  \
 700                 "       xorl %%eax,%%eax\n"                             \
 701                 "       rep; stosb\n"                                   \
 702                 "       popl %%eax\n"                                   \
 703                 "       popl %0\n"                                      \
 704                 "       jmp 2b\n"                                       \
 705                 ".previous\n"                                           \
 706                 ".section __ex_table,\"a\"\n"                           \
 707                 "       .align 4\n"                                     \
 708                 "       .long 4b,5b\n"                                  \
 709                 "       .long 0b,3b\n"                                  \
 710                 "       .long 1b,6b\n"                                  \
 711                 ".previous"                                             \
 712                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 713                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 714                 : "memory");                                            \
 715 } while (0)
 716
 717 unsigned long __copy_to_user_ll(void __user *to, const void *from,
 718                                 unsigned long n)
 719 {
 720 #ifndef CONFIG_X86_WP_WORKS_OK
 721         if (unlikely(boot_cpu_data.wp_works_ok == 0) &&
 722                         ((unsigned long)to) < TASK_SIZE) {
 723                 /*
 724                  * When we are in an atomic section (see
 725                  * mm/filemap.c:file_read_actor), return the full
 726                  * length to take the slow path.
 727                  */
 728                 if (in_atomic())
 729                         return n;
 730
 731                 /*
 732                  * CPU does not honor the WP bit when writing
 733                  * from supervisory mode, and due to preemption or SMP,
 734                  * the page tables can change at any time.
 735                  * Do it manually.      Manfred <manfred@colorfullife.com>
 736                  */
 737                 while (n) {
 738                         unsigned long offset = ((unsigned long)to)%PAGE_SIZE;
 739                         unsigned long len = PAGE_SIZE - offset;
 740                         int retval;
 741                         struct page *pg;
 742                         void *maddr;
 743
 744                         if (len > n)
 745                                 len = n;
 746
 747 survive:
 748                         down_read(&current->mm->mmap_sem);
 749                         retval = get_user_pages(current, current->mm,
 750                                         (unsigned long)to, 1, 1, 0, &pg, NULL);
 751
 752                         if (retval == -ENOMEM && is_global_init(current)) {
 753                                 up_read(&current->mm->mmap_sem);
 754                                 congestion_wait(WRITE, HZ/50);
 755                                 goto survive;
 756                         }
 757
 758                         if (retval != 1) {
 759                                 up_read(&current->mm->mmap_sem);
 760                                 break;
 761                         }
 762
 763                         maddr = kmap_atomic(pg, KM_USER0);
 764                         memcpy(maddr + offset, from, len);
 765                         kunmap_atomic(maddr, KM_USER0);
 766                         set_page_dirty_lock(pg);
 767                         put_page(pg);
 768                         up_read(&current->mm->mmap_sem);
 769
 770                         from += len;
 771                         to += len;
 772                         n -= len;
 773                 }
 774                 return n;
 775         }
 776 #endif
 777         if (movsl_is_ok(to, from, n))
 778                 __copy_user(to, from, n);
 779         else
 780                 n = __copy_user_intel(to, from, n);
 781         return n;
 782 }
 783 EXPORT_SYMBOL(__copy_to_user_ll);
 784
 785 unsigned long __copy_from_user_ll(void *to, const void __user *from,
 786                                         unsigned long n)
 787 {
 788         if (movsl_is_ok(to, from, n))
 789                 __copy_user_zeroing(to, from, n);
 790         else
 791                 n = __copy_user_zeroing_intel(to, from, n);
 792         return n;
 793 }
 794 EXPORT_SYMBOL(__copy_from_user_ll);
 795
 796 unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from,
 797                                          unsigned long n)
 798 {
 799         if (movsl_is_ok(to, from, n))
 800                 __copy_user(to, from, n);
 801         else
 802                 n = __copy_user_intel((void __user *)to,
 803                                       (const void *)from, n);
 804         return n;
 805 }
 806 EXPORT_SYMBOL(__copy_from_user_ll_nozero);
 807
 808 unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from,
 809                                         unsigned long n)
 810 {
 811 #ifdef CONFIG_X86_INTEL_USERCOPY
 812         if (n > 64 && cpu_has_xmm2)
 813                 n = __copy_user_zeroing_intel_nocache(to, from, n);
 814         else
 815                 __copy_user_zeroing(to, from, n);
 816 #else
 817         __copy_user_zeroing(to, from, n);
 818 #endif
 819         return n;
 820 }
 821 EXPORT_SYMBOL(__copy_from_user_ll_nocache);
 822
 823 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
 824                                         unsigned long n)
 825 {
 826 #ifdef CONFIG_X86_INTEL_USERCOPY
 827         if (n > 64 && cpu_has_xmm2)
 828                 n = __copy_user_intel_nocache(to, from, n);
 829         else
 830                 __copy_user(to, from, n);
 831 #else
 832         __copy_user(to, from, n);
 833 #endif
 834         return n;
 835 }
 836 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
 837
 838 /**
 839  * copy_to_user: - Copy a block of data into user space.
 840  * @to:   Destination address, in user space.
 841  * @from: Source address, in kernel space.
 842  * @n:    Number of bytes to copy.
 843  *
 844  * Context: User context only.  This function may sleep.
 845  *
 846  * Copy data from kernel space to user space.
 847  *
 848  * Returns number of bytes that could not be copied.
 849  * On success, this will be zero.
 850  */
 851 unsigned long
 852 copy_to_user(void __user *to, const void *from, unsigned long n)
 853 {
 854         if (access_ok(VERIFY_WRITE, to, n))
 855                 n = __copy_to_user(to, from, n);
 856         return n;
 857 }
 858 EXPORT_SYMBOL(copy_to_user);
 859
 860 /**
 861  * copy_from_user: - Copy a block of data from user space.
 862  * @to:   Destination address, in kernel space.
 863  * @from: Source address, in user space.
 864  * @n:    Number of bytes to copy.
 865  *
 866  * Context: User context only.  This function may sleep.
 867  *
 868  * Copy data from user space to kernel space.
 869  *
 870  * Returns number of bytes that could not be copied.
 871  * On success, this will be zero.
 872  *
 873  * If some data could not be copied, this function will pad the copied
 874  * data to the requested size using zero bytes.
 875  */
 876 unsigned long
 877 copy_from_user(void *to, const void __user *from, unsigned long n)
 878 {
 879         if (access_ok(VERIFY_READ, from, n))
 880                 n = __copy_from_user(to, from, n);
 881         else
 882                 memset(to, 0, n);
 883         return n;
 884 }
 885 EXPORT_SYMBOL(copy_from_user);