arch/sparc/kernel/unaligned.c

   1 /* $Id: unaligned.c,v 1.23 2001/12/21 00:54:31 davem Exp $
   2  * unaligned.c: Unaligned load/store trap handling with special
   3  *              cases for the kernel to do them more quickly.
   4  *
   5  * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
   6  * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   7  */
   8
   9
  10 #include <linux/kernel.h>
  11 #include <linux/sched.h>
  12 #include <linux/mm.h>
  13 #include <linux/module.h>
  14 #include <asm/ptrace.h>
  15 #include <asm/processor.h>
  16 #include <asm/system.h>
  17 #include <asm/uaccess.h>
  18 #include <linux/smp.h>
  19 #include <linux/smp_lock.h>
  20
  21 /* #define DEBUG_MNA */
  22
  23 enum direction {
  24         load,    /* ld, ldd, ldh, ldsh */
  25         store,   /* st, std, sth, stsh */
  26         both,    /* Swap, ldstub, etc. */
  27         fpload,
  28         fpstore,
  29         invalid,
  30 };
  31
  32 #ifdef DEBUG_MNA
  33 static char *dirstrings[] = {
  34   "load", "store", "both", "fpload", "fpstore", "invalid"
  35 };
  36 #endif
  37
  38 static inline enum direction decode_direction(unsigned int insn)
  39 {
  40         unsigned long tmp = (insn >> 21) & 1;
  41
  42         if(!tmp)
  43                 return load;
  44         else {
  45                 if(((insn>>19)&0x3f) == 15)
  46                         return both;
  47                 else
  48                         return store;
  49         }
  50 }
  51
  52 /* 8 = double-word, 4 = word, 2 = half-word */
  53 static inline int decode_access_size(unsigned int insn)
  54 {
  55         insn = (insn >> 19) & 3;
  56
  57         if(!insn)
  58                 return 4;
  59         else if(insn == 3)
  60                 return 8;
  61         else if(insn == 2)
  62                 return 2;
  63         else {
  64                 printk("Impossible unaligned trap. insn=%08x\n", insn);
  65                 die_if_kernel("Byte sized unaligned access?!?!", current->thread.kregs);
  66                 return 4; /* just to keep gcc happy. */
  67         }
  68 }
  69
  70 /* 0x400000 = signed, 0 = unsigned */
  71 static inline int decode_signedness(unsigned int insn)
  72 {
  73         return (insn & 0x400000);
  74 }
  75
  76 static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2,
  77                                        unsigned int rd)
  78 {
  79         if(rs2 >= 16 || rs1 >= 16 || rd >= 16) {
  80                 /* Wheee... */
  81                 __asm__ __volatile__("save %sp, -0x40, %sp\n\t"
  82                                      "save %sp, -0x40, %sp\n\t"
  83                                      "save %sp, -0x40, %sp\n\t"
  84                                      "save %sp, -0x40, %sp\n\t"
  85                                      "save %sp, -0x40, %sp\n\t"
  86                                      "save %sp, -0x40, %sp\n\t"
  87                                      "save %sp, -0x40, %sp\n\t"
  88                                      "restore; restore; restore; restore;\n\t"
  89                                      "restore; restore; restore;\n\t");
  90         }
  91 }
  92
  93 static inline int sign_extend_imm13(int imm)
  94 {
  95         return imm << 19 >> 19;
  96 }
  97
  98 static inline unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs)
  99 {
 100         struct reg_window *win;
 101
 102         if(reg < 16)
 103                 return (!reg ? 0 : regs->u_regs[reg]);
 104
 105         /* Ho hum, the slightly complicated case. */
 106         win = (struct reg_window *) regs->u_regs[UREG_FP];
 107         return win->locals[reg - 16]; /* yes, I know what this does... */
 108 }
 109
 110 static inline unsigned long safe_fetch_reg(unsigned int reg, struct pt_regs *regs)
 111 {
 112         struct reg_window __user *win;
 113         unsigned long ret;
 114
 115         if (reg < 16)
 116                 return (!reg ? 0 : regs->u_regs[reg]);
 117
 118         /* Ho hum, the slightly complicated case. */
 119         win = (struct reg_window __user *) regs->u_regs[UREG_FP];
 120
 121         if ((unsigned long)win & 3)
 122                 return -1;
 123
 124         if (get_user(ret, &win->locals[reg - 16]))
 125                 return -1;
 126
 127         return ret;
 128 }
 129
 130 static inline unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs)
 131 {
 132         struct reg_window *win;
 133
 134         if(reg < 16)
 135                 return &regs->u_regs[reg];
 136         win = (struct reg_window *) regs->u_regs[UREG_FP];
 137         return &win->locals[reg - 16];
 138 }
 139
 140 static unsigned long compute_effective_address(struct pt_regs *regs,
 141                                                unsigned int insn)
 142 {
 143         unsigned int rs1 = (insn >> 14) & 0x1f;
 144         unsigned int rs2 = insn & 0x1f;
 145         unsigned int rd = (insn >> 25) & 0x1f;
 146
 147         if(insn & 0x2000) {
 148                 maybe_flush_windows(rs1, 0, rd);
 149                 return (fetch_reg(rs1, regs) + sign_extend_imm13(insn));
 150         } else {
 151                 maybe_flush_windows(rs1, rs2, rd);
 152                 return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs));
 153         }
 154 }
 155
 156 unsigned long safe_compute_effective_address(struct pt_regs *regs,
 157                                              unsigned int insn)
 158 {
 159         unsigned int rs1 = (insn >> 14) & 0x1f;
 160         unsigned int rs2 = insn & 0x1f;
 161         unsigned int rd = (insn >> 25) & 0x1f;
 162
 163         if(insn & 0x2000) {
 164                 maybe_flush_windows(rs1, 0, rd);
 165                 return (safe_fetch_reg(rs1, regs) + sign_extend_imm13(insn));
 166         } else {
 167                 maybe_flush_windows(rs1, rs2, rd);
 168                 return (safe_fetch_reg(rs1, regs) + safe_fetch_reg(rs2, regs));
 169         }
 170 }
 171
 172 /* This is just to make gcc think panic does return... */
 173 static void unaligned_panic(char *str)
 174 {
 175         panic(str);
 176 }
 177
 178 #define do_integer_load(dest_reg, size, saddr, is_signed, errh) ({              \
 179 __asm__ __volatile__ (                                                          \
 180         "cmp    %1, 8\n\t"                                                      \
 181         "be     9f\n\t"                                                         \
 182         " cmp   %1, 4\n\t"                                                      \
 183         "be     6f\n"                                                           \
 184 "4:\t"  " ldub  [%2], %%l1\n"                                                   \
 185 "5:\t"  "ldub   [%2 + 1], %%l2\n\t"                                             \
 186         "sll    %%l1, 8, %%l1\n\t"                                              \
 187         "tst    %3\n\t"                                                         \
 188         "be     3f\n\t"                                                         \
 189         " add   %%l1, %%l2, %%l1\n\t"                                           \
 190         "sll    %%l1, 16, %%l1\n\t"                                             \
 191         "sra    %%l1, 16, %%l1\n"                                               \
 192 "3:\t"  "b      0f\n\t"                                                         \
 193         " st    %%l1, [%0]\n"                                                   \
 194 "6:\t"  "ldub   [%2 + 1], %%l2\n\t"                                             \
 195         "sll    %%l1, 24, %%l1\n"                                               \
 196 "7:\t"  "ldub   [%2 + 2], %%g7\n\t"                                             \
 197         "sll    %%l2, 16, %%l2\n"                                               \
 198 "8:\t"  "ldub   [%2 + 3], %%g1\n\t"                                             \
 199         "sll    %%g7, 8, %%g7\n\t"                                              \
 200         "or     %%l1, %%l2, %%l1\n\t"                                           \
 201         "or     %%g7, %%g1, %%g7\n\t"                                           \
 202         "or     %%l1, %%g7, %%l1\n\t"                                           \
 203         "b      0f\n\t"                                                         \
 204         " st    %%l1, [%0]\n"                                                   \
 205 "9:\t"  "ldub   [%2], %%l1\n"                                                   \
 206 "10:\t" "ldub   [%2 + 1], %%l2\n\t"                                             \
 207         "sll    %%l1, 24, %%l1\n"                                               \
 208 "11:\t" "ldub   [%2 + 2], %%g7\n\t"                                             \
 209         "sll    %%l2, 16, %%l2\n"                                               \
 210 "12:\t" "ldub   [%2 + 3], %%g1\n\t"                                             \
 211         "sll    %%g7, 8, %%g7\n\t"                                              \
 212         "or     %%l1, %%l2, %%l1\n\t"                                           \
 213         "or     %%g7, %%g1, %%g7\n\t"                                           \
 214         "or     %%l1, %%g7, %%g7\n"                                             \
 215 "13:\t" "ldub   [%2 + 4], %%l1\n\t"                                             \
 216         "st     %%g7, [%0]\n"                                                   \
 217 "14:\t" "ldub   [%2 + 5], %%l2\n\t"                                             \
 218         "sll    %%l1, 24, %%l1\n"                                               \
 219 "15:\t" "ldub   [%2 + 6], %%g7\n\t"                                             \
 220         "sll    %%l2, 16, %%l2\n"                                               \
 221 "16:\t" "ldub   [%2 + 7], %%g1\n\t"                                             \
 222         "sll    %%g7, 8, %%g7\n\t"                                              \
 223         "or     %%l1, %%l2, %%l1\n\t"                                           \
 224         "or     %%g7, %%g1, %%g7\n\t"                                           \
 225         "or     %%l1, %%g7, %%g7\n\t"                                           \
 226         "st     %%g7, [%0 + 4]\n"                                               \
 227 "0:\n\n\t"                                                                      \
 228         ".section __ex_table,#alloc\n\t"                                        \
 229         ".word  4b, " #errh "\n\t"                                              \
 230         ".word  5b, " #errh "\n\t"                                              \
 231         ".word  6b, " #errh "\n\t"                                              \
 232         ".word  7b, " #errh "\n\t"                                              \
 233         ".word  8b, " #errh "\n\t"                                              \
 234         ".word  9b, " #errh "\n\t"                                              \
 235         ".word  10b, " #errh "\n\t"                                             \
 236         ".word  11b, " #errh "\n\t"                                             \
 237         ".word  12b, " #errh "\n\t"                                             \
 238         ".word  13b, " #errh "\n\t"                                             \
 239         ".word  14b, " #errh "\n\t"                                             \
 240         ".word  15b, " #errh "\n\t"                                             \
 241         ".word  16b, " #errh "\n\n\t"                                           \
 242         ".previous\n\t"                                                         \
 243         : : "r" (dest_reg), "r" (size), "r" (saddr), "r" (is_signed)            \
 244         : "l1", "l2", "g7", "g1", "cc");                                        \
 245 })
 246
 247 #define store_common(dst_addr, size, src_val, errh) ({                          \
 248 __asm__ __volatile__ (                                                          \
 249         "ld     [%2], %%l1\n"                                                   \
 250         "cmp    %1, 2\n\t"                                                      \
 251         "be     2f\n\t"                                                         \
 252         " cmp   %1, 4\n\t"                                                      \
 253         "be     1f\n\t"                                                         \
 254         " srl   %%l1, 24, %%l2\n\t"                                             \
 255         "srl    %%l1, 16, %%g7\n"                                               \
 256 "4:\t"  "stb    %%l2, [%0]\n\t"                                                 \
 257         "srl    %%l1, 8, %%l2\n"                                                \
 258 "5:\t"  "stb    %%g7, [%0 + 1]\n\t"                                             \
 259         "ld     [%2 + 4], %%g7\n"                                               \
 260 "6:\t"  "stb    %%l2, [%0 + 2]\n\t"                                             \
 261         "srl    %%g7, 24, %%l2\n"                                               \
 262 "7:\t"  "stb    %%l1, [%0 + 3]\n\t"                                             \
 263         "srl    %%g7, 16, %%l1\n"                                               \
 264 "8:\t"  "stb    %%l2, [%0 + 4]\n\t"                                             \
 265         "srl    %%g7, 8, %%l2\n"                                                \
 266 "9:\t"  "stb    %%l1, [%0 + 5]\n"                                               \
 267 "10:\t" "stb    %%l2, [%0 + 6]\n\t"                                             \
 268         "b      0f\n"                                                           \
 269 "11:\t" " stb   %%g7, [%0 + 7]\n"                                               \
 270 "1:\t"  "srl    %%l1, 16, %%g7\n"                                               \
 271 "12:\t" "stb    %%l2, [%0]\n\t"                                                 \
 272         "srl    %%l1, 8, %%l2\n"                                                \
 273 "13:\t" "stb    %%g7, [%0 + 1]\n"                                               \
 274 "14:\t" "stb    %%l2, [%0 + 2]\n\t"                                             \
 275         "b      0f\n"                                                           \
 276 "15:\t" " stb   %%l1, [%0 + 3]\n"                                               \
 277 "2:\t"  "srl    %%l1, 8, %%l2\n"                                                \
 278 "16:\t" "stb    %%l2, [%0]\n"                                                   \
 279 "17:\t" "stb    %%l1, [%0 + 1]\n"                                               \
 280 "0:\n\n\t"                                                                      \
 281         ".section __ex_table,#alloc\n\t"                                        \
 282         ".word  4b, " #errh "\n\t"                                              \
 283         ".word  5b, " #errh "\n\t"                                              \
 284         ".word  6b, " #errh "\n\t"                                              \
 285         ".word  7b, " #errh "\n\t"                                              \
 286         ".word  8b, " #errh "\n\t"                                              \
 287         ".word  9b, " #errh "\n\t"                                              \
 288         ".word  10b, " #errh "\n\t"                                             \
 289         ".word  11b, " #errh "\n\t"                                             \
 290         ".word  12b, " #errh "\n\t"                                             \
 291         ".word  13b, " #errh "\n\t"                                             \
 292         ".word  14b, " #errh "\n\t"                                             \
 293         ".word  15b, " #errh "\n\t"                                             \
 294         ".word  16b, " #errh "\n\t"                                             \
 295         ".word  17b, " #errh "\n\n\t"                                           \
 296         ".previous\n\t"                                                         \
 297         : : "r" (dst_addr), "r" (size), "r" (src_val)                           \
 298         : "l1", "l2", "g7", "g1", "cc");                                        \
 299 })
 300
 301 #define do_integer_store(reg_num, size, dst_addr, regs, errh) ({                \
 302         unsigned long *src_val;                                                 \
 303         static unsigned long zero[2] = { 0, };                                  \
 304                                                                                 \
 305         if (reg_num) src_val = fetch_reg_addr(reg_num, regs);                   \
 306         else {                                                                  \
 307                 src_val = &zero[0];                                             \
 308                 if (size == 8)                                                  \
 309                         zero[1] = fetch_reg(1, regs);                           \
 310         }                                                                       \
 311         store_common(dst_addr, size, src_val, errh);                            \
 312 })
 313
 314 extern void smp_capture(void);
 315 extern void smp_release(void);
 316
 317 #define do_atomic(srcdest_reg, mem, errh) ({                                    \
 318         unsigned long flags, tmp;                                               \
 319                                                                                 \
 320         smp_capture();                                                          \
 321         local_irq_save(flags);                                                  \
 322         tmp = *srcdest_reg;                                                     \
 323         do_integer_load(srcdest_reg, 4, mem, 0, errh);                          \
 324         store_common(mem, 4, &tmp, errh);                                       \
 325         local_irq_restore(flags);                                               \
 326         smp_release();                                                          \
 327 })
 328
 329 static inline void advance(struct pt_regs *regs)
 330 {
 331         regs->pc   = regs->npc;
 332         regs->npc += 4;
 333 }
 334
 335 static inline int floating_point_load_or_store_p(unsigned int insn)
 336 {
 337         return (insn >> 24) & 1;
 338 }
 339
 340 static inline int ok_for_kernel(unsigned int insn)
 341 {
 342         return !floating_point_load_or_store_p(insn);
 343 }
 344
 345 void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn) __asm__ ("kernel_mna_trap_fault");
 346
 347 void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn)
 348 {
 349         unsigned long g2 = regs->u_regs [UREG_G2];
 350         unsigned long fixup = search_extables_range(regs->pc, &g2);
 351
 352         if (!fixup) {
 353                 unsigned long address = compute_effective_address(regs, insn);
 354                 if(address < PAGE_SIZE) {
 355                         printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference in mna handler");
 356                 } else
 357                         printk(KERN_ALERT "Unable to handle kernel paging request in mna handler");
 358                 printk(KERN_ALERT " at virtual address %08lx\n",address);
 359                 printk(KERN_ALERT "current->{mm,active_mm}->context = %08lx\n",
 360                         (current->mm ? current->mm->context :
 361                         current->active_mm->context));
 362                 printk(KERN_ALERT "current->{mm,active_mm}->pgd = %08lx\n",
 363                         (current->mm ? (unsigned long) current->mm->pgd :
 364                         (unsigned long) current->active_mm->pgd));
 365                 die_if_kernel("Oops", regs);
 366                 /* Not reached */
 367         }
 368         regs->pc = fixup;
 369         regs->npc = regs->pc + 4;
 370         regs->u_regs [UREG_G2] = g2;
 371 }
 372
 373 asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn)
 374 {
 375         enum direction dir = decode_direction(insn);
 376         int size = decode_access_size(insn);
 377
 378         if(!ok_for_kernel(insn) || dir == both) {
 379                 printk("Unsupported unaligned load/store trap for kernel at <%08lx>.\n",
 380                        regs->pc);
 381                 unaligned_panic("Wheee. Kernel does fpu/atomic unaligned load/store.");
 382
 383                 __asm__ __volatile__ ("\n"
 384 "kernel_unaligned_trap_fault:\n\t"
 385                 "mov    %0, %%o0\n\t"
 386                 "call   kernel_mna_trap_fault\n\t"
 387                 " mov   %1, %%o1\n\t"
 388                 :
 389                 : "r" (regs), "r" (insn)
 390                 : "o0", "o1", "o2", "o3", "o4", "o5", "o7",
 391                   "g1", "g2", "g3", "g4", "g5", "g7", "cc");
 392         } else {
 393                 unsigned long addr = compute_effective_address(regs, insn);
 394
 395 #ifdef DEBUG_MNA
 396                 printk("KMNA: pc=%08lx [dir=%s addr=%08lx size=%d] retpc[%08lx]\n",
 397                        regs->pc, dirstrings[dir], addr, size, regs->u_regs[UREG_RETPC]);
 398 #endif
 399                 switch(dir) {
 400                 case load:
 401                         do_integer_load(fetch_reg_addr(((insn>>25)&0x1f), regs),
 402                                         size, (unsigned long *) addr,
 403                                         decode_signedness(insn),
 404                                         kernel_unaligned_trap_fault);
 405                         break;
 406
 407                 case store:
 408                         do_integer_store(((insn>>25)&0x1f), size,
 409                                          (unsigned long *) addr, regs,
 410                                          kernel_unaligned_trap_fault);
 411                         break;
 412 #if 0 /* unsupported */
 413                 case both:
 414                         do_atomic(fetch_reg_addr(((insn>>25)&0x1f), regs),
 415                                   (unsigned long *) addr,
 416                                   kernel_unaligned_trap_fault);
 417                         break;
 418 #endif
 419                 default:
 420                         panic("Impossible kernel unaligned trap.");
 421                         /* Not reached... */
 422                 }
 423                 advance(regs);
 424         }
 425 }
 426
 427 static inline int ok_for_user(struct pt_regs *regs, unsigned int insn,
 428                               enum direction dir)
 429 {
 430         unsigned int reg;
 431         int check = (dir == load) ? VERIFY_READ : VERIFY_WRITE;
 432         int size = ((insn >> 19) & 3) == 3 ? 8 : 4;
 433
 434         if ((regs->pc | regs->npc) & 3)
 435                 return 0;
 436
 437         /* Must access_ok() in all the necessary places. */
 438 #define WINREG_ADDR(regnum) \
 439         ((void __user *)(((unsigned long *)regs->u_regs[UREG_FP])+(regnum)))
 440
 441         reg = (insn >> 25) & 0x1f;
 442         if (reg >= 16) {
 443                 if (!access_ok(check, WINREG_ADDR(reg - 16), size))
 444                         return -EFAULT;
 445         }
 446         reg = (insn >> 14) & 0x1f;
 447         if (reg >= 16) {
 448                 if (!access_ok(check, WINREG_ADDR(reg - 16), size))
 449                         return -EFAULT;
 450         }
 451         if (!(insn & 0x2000)) {
 452                 reg = (insn & 0x1f);
 453                 if (reg >= 16) {
 454                         if (!access_ok(check, WINREG_ADDR(reg - 16), size))
 455                                 return -EFAULT;
 456                 }
 457         }
 458 #undef WINREG_ADDR
 459         return 0;
 460 }
 461
 462 void user_mna_trap_fault(struct pt_regs *regs, unsigned int insn) __asm__ ("user_mna_trap_fault");
 463
 464 void user_mna_trap_fault(struct pt_regs *regs, unsigned int insn)
 465 {
 466         siginfo_t info;
 467
 468         info.si_signo = SIGBUS;
 469         info.si_errno = 0;
 470         info.si_code = BUS_ADRALN;
 471         info.si_addr = (void __user *)safe_compute_effective_address(regs, insn);
 472         info.si_trapno = 0;
 473         send_sig_info(SIGBUS, &info, current);
 474 }
 475
 476 asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn)
 477 {
 478         enum direction dir;
 479
 480         lock_kernel();
 481         if(!(current->thread.flags & SPARC_FLAG_UNALIGNED) ||
 482            (((insn >> 30) & 3) != 3))
 483                 goto kill_user;
 484         dir = decode_direction(insn);
 485         if(!ok_for_user(regs, insn, dir)) {
 486                 goto kill_user;
 487         } else {
 488                 int size = decode_access_size(insn);
 489                 unsigned long addr;
 490
 491                 if(floating_point_load_or_store_p(insn)) {
 492                         printk("User FPU load/store unaligned unsupported.\n");
 493                         goto kill_user;
 494                 }
 495
 496                 addr = compute_effective_address(regs, insn);
 497                 switch(dir) {
 498                 case load:
 499                         do_integer_load(fetch_reg_addr(((insn>>25)&0x1f), regs),
 500                                         size, (unsigned long *) addr,
 501                                         decode_signedness(insn),
 502                                         user_unaligned_trap_fault);
 503                         break;
 504
 505                 case store:
 506                         do_integer_store(((insn>>25)&0x1f), size,
 507                                          (unsigned long *) addr, regs,
 508                                          user_unaligned_trap_fault);
 509                         break;
 510
 511                 case both:
 512 #if 0 /* unsupported */
 513                         do_atomic(fetch_reg_addr(((insn>>25)&0x1f), regs),
 514                                   (unsigned long *) addr,
 515                                   user_unaligned_trap_fault);
 516 #else
 517                         /*
 518                          * This was supported in 2.4. However, we question
 519                          * the value of SWAP instruction across word boundaries.
 520                          */
 521                         printk("Unaligned SWAP unsupported.\n");
 522                         goto kill_user;
 523 #endif
 524                         break;
 525
 526                 default:
 527                         unaligned_panic("Impossible user unaligned trap.");
 528
 529                         __asm__ __volatile__ ("\n"
 530 "user_unaligned_trap_fault:\n\t"
 531                         "mov    %0, %%o0\n\t"
 532                         "call   user_mna_trap_fault\n\t"
 533                         " mov   %1, %%o1\n\t"
 534                         :
 535                         : "r" (regs), "r" (insn)
 536                         : "o0", "o1", "o2", "o3", "o4", "o5", "o7",
 537                           "g1", "g2", "g3", "g4", "g5", "g7", "cc");
 538                         goto out;
 539                 }
 540                 advance(regs);
 541                 goto out;
 542         }
 543
 544 kill_user:
 545         user_mna_trap_fault(regs, insn);
 546 out:
 547         unlock_kernel();
 548 }