arch/powerpc/lib/copyuser_64.S

   1 /*
   2  * arch/ppc64/lib/copyuser.S
   3  *
   4  * Copyright (C) 2002 Paul Mackerras, IBM Corp.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11 #include <asm/processor.h>
  12 #include <asm/ppc_asm.h>
  13
  14         .align  7
  15 _GLOBAL(__copy_tofrom_user)
  16         /* first check for a whole page copy on a page boundary */
  17         cmpldi  cr1,r5,16
  18         cmpdi   cr6,r5,4096
  19         or      r0,r3,r4
  20         neg     r6,r3           /* LS 3 bits = # bytes to 8-byte dest bdry */
  21         andi.   r0,r0,4095
  22         std     r3,-24(r1)
  23         crand   cr0*4+2,cr0*4+2,cr6*4+2
  24         std     r4,-16(r1)
  25         std     r5,-8(r1)
  26         dcbt    0,r4
  27         beq     .Lcopy_page_4K
  28         andi.   r6,r6,7
  29         mtcrf   0x01,r5
  30         blt     cr1,.Lshort_copy
  31         bne     .Ldst_unaligned
  32 .Ldst_aligned:
  33         andi.   r0,r4,7
  34         addi    r3,r3,-16
  35         bne     .Lsrc_unaligned
  36         srdi    r7,r5,4
  37 20:     ld      r9,0(r4)
  38         addi    r4,r4,-8
  39         mtctr   r7
  40         andi.   r5,r5,7
  41         bf      cr7*4+0,22f
  42         addi    r3,r3,8
  43         addi    r4,r4,8
  44         mr      r8,r9
  45         blt     cr1,72f
  46 21:     ld      r9,8(r4)
  47 70:     std     r8,8(r3)
  48 22:     ldu     r8,16(r4)
  49 71:     stdu    r9,16(r3)
  50         bdnz    21b
  51 72:     std     r8,8(r3)
  52         beq+    3f
  53         addi    r3,r3,16
  54 23:     ld      r9,8(r4)
  55 .Ldo_tail:
  56         bf      cr7*4+1,1f
  57         rotldi  r9,r9,32
  58 73:     stw     r9,0(r3)
  59         addi    r3,r3,4
  60 1:      bf      cr7*4+2,2f
  61         rotldi  r9,r9,16
  62 74:     sth     r9,0(r3)
  63         addi    r3,r3,2
  64 2:      bf      cr7*4+3,3f
  65         rotldi  r9,r9,8
  66 75:     stb     r9,0(r3)
  67 3:      li      r3,0
  68         blr
  69
  70 .Lsrc_unaligned:
  71         srdi    r6,r5,3
  72         addi    r5,r5,-16
  73         subf    r4,r0,r4
  74         srdi    r7,r5,4
  75         sldi    r10,r0,3
  76         cmpldi  cr6,r6,3
  77         andi.   r5,r5,7
  78         mtctr   r7
  79         subfic  r11,r10,64
  80         add     r5,r5,r0
  81         bt      cr7*4+0,28f
  82
  83 24:     ld      r9,0(r4)        /* 3+2n loads, 2+2n stores */
  84 25:     ld      r0,8(r4)
  85         sld     r6,r9,r10
  86 26:     ldu     r9,16(r4)
  87         srd     r7,r0,r11
  88         sld     r8,r0,r10
  89         or      r7,r7,r6
  90         blt     cr6,79f
  91 27:     ld      r0,8(r4)
  92         b       2f
  93
  94 28:     ld      r0,0(r4)        /* 4+2n loads, 3+2n stores */
  95 29:     ldu     r9,8(r4)
  96         sld     r8,r0,r10
  97         addi    r3,r3,-8
  98         blt     cr6,5f
  99 30:     ld      r0,8(r4)
 100         srd     r12,r9,r11
 101         sld     r6,r9,r10
 102 31:     ldu     r9,16(r4)
 103         or      r12,r8,r12
 104         srd     r7,r0,r11
 105         sld     r8,r0,r10
 106         addi    r3,r3,16
 107         beq     cr6,78f
 108
 109 1:      or      r7,r7,r6
 110 32:     ld      r0,8(r4)
 111 76:     std     r12,8(r3)
 112 2:      srd     r12,r9,r11
 113         sld     r6,r9,r10
 114 33:     ldu     r9,16(r4)
 115         or      r12,r8,r12
 116 77:     stdu    r7,16(r3)
 117         srd     r7,r0,r11
 118         sld     r8,r0,r10
 119         bdnz    1b
 120
 121 78:     std     r12,8(r3)
 122         or      r7,r7,r6
 123 79:     std     r7,16(r3)
 124 5:      srd     r12,r9,r11
 125         or      r12,r8,r12
 126 80:     std     r12,24(r3)
 127         bne     6f
 128         li      r3,0
 129         blr
 130 6:      cmpwi   cr1,r5,8
 131         addi    r3,r3,32
 132         sld     r9,r9,r10
 133         ble     cr1,.Ldo_tail
 134 34:     ld      r0,8(r4)
 135         srd     r7,r0,r11
 136         or      r9,r7,r9
 137         b       .Ldo_tail
 138
 139 .Ldst_unaligned:
 140         mtcrf   0x01,r6         /* put #bytes to 8B bdry into cr7 */
 141         subf    r5,r6,r5
 142         li      r7,0
 143         cmpldi  r1,r5,16
 144         bf      cr7*4+3,1f
 145 35:     lbz     r0,0(r4)
 146 81:     stb     r0,0(r3)
 147         addi    r7,r7,1
 148 1:      bf      cr7*4+2,2f
 149 36:     lhzx    r0,r7,r4
 150 82:     sthx    r0,r7,r3
 151         addi    r7,r7,2
 152 2:      bf      cr7*4+1,3f
 153 37:     lwzx    r0,r7,r4
 154 83:     stwx    r0,r7,r3
 155 3:      mtcrf   0x01,r5
 156         add     r4,r6,r4
 157         add     r3,r6,r3
 158         b       .Ldst_aligned
 159
 160 .Lshort_copy:
 161         bf      cr7*4+0,1f
 162 38:     lwz     r0,0(r4)
 163 39:     lwz     r9,4(r4)
 164         addi    r4,r4,8
 165 84:     stw     r0,0(r3)
 166 85:     stw     r9,4(r3)
 167         addi    r3,r3,8
 168 1:      bf      cr7*4+1,2f
 169 40:     lwz     r0,0(r4)
 170         addi    r4,r4,4
 171 86:     stw     r0,0(r3)
 172         addi    r3,r3,4
 173 2:      bf      cr7*4+2,3f
 174 41:     lhz     r0,0(r4)
 175         addi    r4,r4,2
 176 87:     sth     r0,0(r3)
 177         addi    r3,r3,2
 178 3:      bf      cr7*4+3,4f
 179 42:     lbz     r0,0(r4)
 180 88:     stb     r0,0(r3)
 181 4:      li      r3,0
 182         blr
 183
 184 /*
 185  * exception handlers follow
 186  * we have to return the number of bytes not copied
 187  * for an exception on a load, we set the rest of the destination to 0
 188  */
 189
 190 136:
 191 137:
 192         add     r3,r3,r7
 193         b       1f
 194 130:
 195 131:
 196         addi    r3,r3,8
 197 120:
 198 122:
 199 124:
 200 125:
 201 126:
 202 127:
 203 128:
 204 129:
 205 133:
 206         addi    r3,r3,8
 207 121:
 208 132:
 209         addi    r3,r3,8
 210 123:
 211 134:
 212 135:
 213 138:
 214 139:
 215 140:
 216 141:
 217 142:
 218
 219 /*
 220  * here we have had a fault on a load and r3 points to the first
 221  * unmodified byte of the destination
 222  */
 223 1:      ld      r6,-24(r1)
 224         ld      r4,-16(r1)
 225         ld      r5,-8(r1)
 226         subf    r6,r6,r3
 227         add     r4,r4,r6
 228         subf    r5,r6,r5        /* #bytes left to go */
 229
 230 /*
 231  * first see if we can copy any more bytes before hitting another exception
 232  */
 233         mtctr   r5
 234 43:     lbz     r0,0(r4)
 235         addi    r4,r4,1
 236 89:     stb     r0,0(r3)
 237         addi    r3,r3,1
 238         bdnz    43b
 239         li      r3,0            /* huh? all copied successfully this time? */
 240         blr
 241
 242 /*
 243  * here we have trapped again, need to clear ctr bytes starting at r3
 244  */
 245 143:    mfctr   r5
 246         li      r0,0
 247         mr      r4,r3
 248         mr      r3,r5           /* return the number of bytes not copied */
 249 1:      andi.   r9,r4,7
 250         beq     3f
 251 90:     stb     r0,0(r4)
 252         addic.  r5,r5,-1
 253         addi    r4,r4,1
 254         bne     1b
 255         blr
 256 3:      cmpldi  cr1,r5,8
 257         srdi    r9,r5,3
 258         andi.   r5,r5,7
 259         blt     cr1,93f
 260         mtctr   r9
 261 91:     std     r0,0(r4)
 262         addi    r4,r4,8
 263         bdnz    91b
 264 93:     beqlr
 265         mtctr   r5
 266 92:     stb     r0,0(r4)
 267         addi    r4,r4,1
 268         bdnz    92b
 269         blr
 270
 271 /*
 272  * exception handlers for stores: we just need to work
 273  * out how many bytes weren't copied
 274  */
 275 182:
 276 183:
 277         add     r3,r3,r7
 278         b       1f
 279 180:
 280         addi    r3,r3,8
 281 171:
 282 177:
 283         addi    r3,r3,8
 284 170:
 285 172:
 286 176:
 287 178:
 288         addi    r3,r3,4
 289 185:
 290         addi    r3,r3,4
 291 173:
 292 174:
 293 175:
 294 179:
 295 181:
 296 184:
 297 186:
 298 187:
 299 188:
 300 189:
 301 1:
 302         ld      r6,-24(r1)
 303         ld      r5,-8(r1)
 304         add     r6,r6,r5
 305         subf    r3,r3,r6        /* #bytes not copied */
 306 190:
 307 191:
 308 192:
 309         blr                     /* #bytes not copied in r3 */
 310
 311         .section __ex_table,"a"
 312         .align  3
 313         .llong  20b,120b
 314         .llong  21b,121b
 315         .llong  70b,170b
 316         .llong  22b,122b
 317         .llong  71b,171b
 318         .llong  72b,172b
 319         .llong  23b,123b
 320         .llong  73b,173b
 321         .llong  74b,174b
 322         .llong  75b,175b
 323         .llong  24b,124b
 324         .llong  25b,125b
 325         .llong  26b,126b
 326         .llong  27b,127b
 327         .llong  28b,128b
 328         .llong  29b,129b
 329         .llong  30b,130b
 330         .llong  31b,131b
 331         .llong  32b,132b
 332         .llong  76b,176b
 333         .llong  33b,133b
 334         .llong  77b,177b
 335         .llong  78b,178b
 336         .llong  79b,179b
 337         .llong  80b,180b
 338         .llong  34b,134b
 339         .llong  35b,135b
 340         .llong  81b,181b
 341         .llong  36b,136b
 342         .llong  82b,182b
 343         .llong  37b,137b
 344         .llong  83b,183b
 345         .llong  38b,138b
 346         .llong  39b,139b
 347         .llong  84b,184b
 348         .llong  85b,185b
 349         .llong  40b,140b
 350         .llong  86b,186b
 351         .llong  41b,141b
 352         .llong  87b,187b
 353         .llong  42b,142b
 354         .llong  88b,188b
 355         .llong  43b,143b
 356         .llong  89b,189b
 357         .llong  90b,190b
 358         .llong  91b,191b
 359         .llong  92b,192b
 360
 361         .text
 362
 363 /*
 364  * Routine to copy a whole page of data, optimized for POWER4.
 365  * On POWER4 it is more than 50% faster than the simple loop
 366  * above (following the .Ldst_aligned label) but it runs slightly
 367  * slower on POWER3.
 368  */
 369 .Lcopy_page_4K:
 370         std     r31,-32(1)
 371         std     r30,-40(1)
 372         std     r29,-48(1)
 373         std     r28,-56(1)
 374         std     r27,-64(1)
 375         std     r26,-72(1)
 376         std     r25,-80(1)
 377         std     r24,-88(1)
 378         std     r23,-96(1)
 379         std     r22,-104(1)
 380         std     r21,-112(1)
 381         std     r20,-120(1)
 382         li      r5,4096/32 - 1
 383         addi    r3,r3,-8
 384         li      r0,5
 385 0:      addi    r5,r5,-24
 386         mtctr   r0
 387 20:     ld      r22,640(4)
 388 21:     ld      r21,512(4)
 389 22:     ld      r20,384(4)
 390 23:     ld      r11,256(4)
 391 24:     ld      r9,128(4)
 392 25:     ld      r7,0(4)
 393 26:     ld      r25,648(4)
 394 27:     ld      r24,520(4)
 395 28:     ld      r23,392(4)
 396 29:     ld      r10,264(4)
 397 30:     ld      r8,136(4)
 398 31:     ldu     r6,8(4)
 399         cmpwi   r5,24
 400 1:
 401 32:     std     r22,648(3)
 402 33:     std     r21,520(3)
 403 34:     std     r20,392(3)
 404 35:     std     r11,264(3)
 405 36:     std     r9,136(3)
 406 37:     std     r7,8(3)
 407 38:     ld      r28,648(4)
 408 39:     ld      r27,520(4)
 409 40:     ld      r26,392(4)
 410 41:     ld      r31,264(4)
 411 42:     ld      r30,136(4)
 412 43:     ld      r29,8(4)
 413 44:     std     r25,656(3)
 414 45:     std     r24,528(3)
 415 46:     std     r23,400(3)
 416 47:     std     r10,272(3)
 417 48:     std     r8,144(3)
 418 49:     std     r6,16(3)
 419 50:     ld      r22,656(4)
 420 51:     ld      r21,528(4)
 421 52:     ld      r20,400(4)
 422 53:     ld      r11,272(4)
 423 54:     ld      r9,144(4)
 424 55:     ld      r7,16(4)
 425 56:     std     r28,664(3)
 426 57:     std     r27,536(3)
 427 58:     std     r26,408(3)
 428 59:     std     r31,280(3)
 429 60:     std     r30,152(3)
 430 61:     stdu    r29,24(3)
 431 62:     ld      r25,664(4)
 432 63:     ld      r24,536(4)
 433 64:     ld      r23,408(4)
 434 65:     ld      r10,280(4)
 435 66:     ld      r8,152(4)
 436 67:     ldu     r6,24(4)
 437         bdnz    1b
 438 68:     std     r22,648(3)
 439 69:     std     r21,520(3)
 440 70:     std     r20,392(3)
 441 71:     std     r11,264(3)
 442 72:     std     r9,136(3)
 443 73:     std     r7,8(3)
 444 74:     addi    r4,r4,640
 445 75:     addi    r3,r3,648
 446         bge     0b
 447         mtctr   r5
 448 76:     ld      r7,0(4)
 449 77:     ld      r8,8(4)
 450 78:     ldu     r9,16(4)
 451 3:
 452 79:     ld      r10,8(4)
 453 80:     std     r7,8(3)
 454 81:     ld      r7,16(4)
 455 82:     std     r8,16(3)
 456 83:     ld      r8,24(4)
 457 84:     std     r9,24(3)
 458 85:     ldu     r9,32(4)
 459 86:     stdu    r10,32(3)
 460         bdnz    3b
 461 4:
 462 87:     ld      r10,8(4)
 463 88:     std     r7,8(3)
 464 89:     std     r8,16(3)
 465 90:     std     r9,24(3)
 466 91:     std     r10,32(3)
 467 9:      ld      r20,-120(1)
 468         ld      r21,-112(1)
 469         ld      r22,-104(1)
 470         ld      r23,-96(1)
 471         ld      r24,-88(1)
 472         ld      r25,-80(1)
 473         ld      r26,-72(1)
 474         ld      r27,-64(1)
 475         ld      r28,-56(1)
 476         ld      r29,-48(1)
 477         ld      r30,-40(1)
 478         ld      r31,-32(1)
 479         li      r3,0
 480         blr
 481
 482 /*
 483  * on an exception, reset to the beginning and jump back into the
 484  * standard __copy_tofrom_user
 485  */
 486 100:    ld      r20,-120(1)
 487         ld      r21,-112(1)
 488         ld      r22,-104(1)
 489         ld      r23,-96(1)
 490         ld      r24,-88(1)
 491         ld      r25,-80(1)
 492         ld      r26,-72(1)
 493         ld      r27,-64(1)
 494         ld      r28,-56(1)
 495         ld      r29,-48(1)
 496         ld      r30,-40(1)
 497         ld      r31,-32(1)
 498         ld      r3,-24(r1)
 499         ld      r4,-16(r1)
 500         li      r5,4096
 501         b       .Ldst_aligned
 502
 503         .section __ex_table,"a"
 504         .align  3
 505         .llong  20b,100b
 506         .llong  21b,100b
 507         .llong  22b,100b
 508         .llong  23b,100b
 509         .llong  24b,100b
 510         .llong  25b,100b
 511         .llong  26b,100b
 512         .llong  27b,100b
 513         .llong  28b,100b
 514         .llong  29b,100b
 515         .llong  30b,100b
 516         .llong  31b,100b
 517         .llong  32b,100b
 518         .llong  33b,100b
 519         .llong  34b,100b
 520         .llong  35b,100b
 521         .llong  36b,100b
 522         .llong  37b,100b
 523         .llong  38b,100b
 524         .llong  39b,100b
 525         .llong  40b,100b
 526         .llong  41b,100b
 527         .llong  42b,100b
 528         .llong  43b,100b
 529         .llong  44b,100b
 530         .llong  45b,100b
 531         .llong  46b,100b
 532         .llong  47b,100b
 533         .llong  48b,100b
 534         .llong  49b,100b
 535         .llong  50b,100b
 536         .llong  51b,100b
 537         .llong  52b,100b
 538         .llong  53b,100b
 539         .llong  54b,100b
 540         .llong  55b,100b
 541         .llong  56b,100b
 542         .llong  57b,100b
 543         .llong  58b,100b
 544         .llong  59b,100b
 545         .llong  60b,100b
 546         .llong  61b,100b
 547         .llong  62b,100b
 548         .llong  63b,100b
 549         .llong  64b,100b
 550         .llong  65b,100b
 551         .llong  66b,100b
 552         .llong  67b,100b
 553         .llong  68b,100b
 554         .llong  69b,100b
 555         .llong  70b,100b
 556         .llong  71b,100b
 557         .llong  72b,100b
 558         .llong  73b,100b
 559         .llong  74b,100b
 560         .llong  75b,100b
 561         .llong  76b,100b
 562         .llong  77b,100b
 563         .llong  78b,100b
 564         .llong  79b,100b
 565         .llong  80b,100b
 566         .llong  81b,100b
 567         .llong  82b,100b
 568         .llong  83b,100b
 569         .llong  84b,100b
 570         .llong  85b,100b
 571         .llong  86b,100b
 572         .llong  87b,100b
 573         .llong  88b,100b
 574         .llong  89b,100b
 575         .llong  90b,100b
 576         .llong  91b,100b