arch/s390/mm/fault.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  S390 version
   4  *    Copyright IBM Corp. 1999
   5  *    Author(s): Hartmut Penner (hp@de.ibm.com)
   6  *               Ulrich Weigand (uweigand@de.ibm.com)
   7  *
   8  *  Derived from "arch/i386/mm/fault.c"
   9  *    Copyright (C) 1995  Linus Torvalds
  10  */
  11
  12 #include <linux/kernel_stat.h>
  13 #include <linux/mmu_context.h>
  14 #include <linux/perf_event.h>
  15 #include <linux/signal.h>
  16 #include <linux/sched.h>
  17 #include <linux/sched/debug.h>
  18 #include <linux/jump_label.h>
  19 #include <linux/kernel.h>
  20 #include <linux/errno.h>
  21 #include <linux/string.h>
  22 #include <linux/types.h>
  23 #include <linux/ptrace.h>
  24 #include <linux/mman.h>
  25 #include <linux/mm.h>
  26 #include <linux/compat.h>
  27 #include <linux/smp.h>
  28 #include <linux/kdebug.h>
  29 #include <linux/init.h>
  30 #include <linux/console.h>
  31 #include <linux/extable.h>
  32 #include <linux/hardirq.h>
  33 #include <linux/kprobes.h>
  34 #include <linux/uaccess.h>
  35 #include <linux/hugetlb.h>
  36 #include <linux/kfence.h>
  37 #include <asm/asm-extable.h>
  38 #include <asm/asm-offsets.h>
  39 #include <asm/ptrace.h>
  40 #include <asm/fault.h>
  41 #include <asm/diag.h>
  42 #include <asm/gmap.h>
  43 #include <asm/irq.h>
  44 #include <asm/facility.h>
  45 #include <asm/uv.h>
  46 #include "../kernel/entry.h"
  47
  48 enum fault_type {
  49         KERNEL_FAULT,
  50         USER_FAULT,
  51         GMAP_FAULT,
  52 };
  53
  54 static DEFINE_STATIC_KEY_FALSE(have_store_indication);
  55
  56 static int __init fault_init(void)
  57 {
  58         if (test_facility(75))
  59                 static_branch_enable(&have_store_indication);
  60         return 0;
  61 }
  62 early_initcall(fault_init);
  63
  64 /*
  65  * Find out which address space caused the exception.
  66  */
  67 static enum fault_type get_fault_type(struct pt_regs *regs)
  68 {
  69         union teid teid = { .val = regs->int_parm_long };
  70
  71         if (likely(teid.as == PSW_BITS_AS_PRIMARY)) {
  72                 if (user_mode(regs))
  73                         return USER_FAULT;
  74                 if (!IS_ENABLED(CONFIG_PGSTE))
  75                         return KERNEL_FAULT;
  76                 if (test_pt_regs_flag(regs, PIF_GUEST_FAULT))
  77                         return GMAP_FAULT;
  78                 return KERNEL_FAULT;
  79         }
  80         if (teid.as == PSW_BITS_AS_SECONDARY)
  81                 return USER_FAULT;
  82         /* Access register mode, not used in the kernel */
  83         if (teid.as == PSW_BITS_AS_ACCREG)
  84                 return USER_FAULT;
  85         /* Home space -> access via kernel ASCE */
  86         return KERNEL_FAULT;
  87 }
  88
  89 static unsigned long get_fault_address(struct pt_regs *regs)
  90 {
  91         union teid teid = { .val = regs->int_parm_long };
  92
  93         return teid.addr * PAGE_SIZE;
  94 }
  95
  96 static __always_inline bool fault_is_write(struct pt_regs *regs)
  97 {
  98         union teid teid = { .val = regs->int_parm_long };
  99
 100         if (static_branch_likely(&have_store_indication))
 101                 return teid.fsi == TEID_FSI_STORE;
 102         return false;
 103 }
 104
 105 static void dump_pagetable(unsigned long asce, unsigned long address)
 106 {
 107         unsigned long entry, *table = __va(asce & _ASCE_ORIGIN);
 108
 109         pr_alert("AS:%016lx ", asce);
 110         switch (asce & _ASCE_TYPE_MASK) {
 111         case _ASCE_TYPE_REGION1:
 112                 table += (address & _REGION1_INDEX) >> _REGION1_SHIFT;
 113                 if (get_kernel_nofault(entry, table))
 114                         goto bad;
 115                 pr_cont("R1:%016lx ", entry);
 116                 if (entry & _REGION_ENTRY_INVALID)
 117                         goto out;
 118                 table = __va(entry & _REGION_ENTRY_ORIGIN);
 119                 fallthrough;
 120         case _ASCE_TYPE_REGION2:
 121                 table += (address & _REGION2_INDEX) >> _REGION2_SHIFT;
 122                 if (get_kernel_nofault(entry, table))
 123                         goto bad;
 124                 pr_cont("R2:%016lx ", entry);
 125                 if (entry & _REGION_ENTRY_INVALID)
 126                         goto out;
 127                 table = __va(entry & _REGION_ENTRY_ORIGIN);
 128                 fallthrough;
 129         case _ASCE_TYPE_REGION3:
 130                 table += (address & _REGION3_INDEX) >> _REGION3_SHIFT;
 131                 if (get_kernel_nofault(entry, table))
 132                         goto bad;
 133                 pr_cont("R3:%016lx ", entry);
 134                 if (entry & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE))
 135                         goto out;
 136                 table = __va(entry & _REGION_ENTRY_ORIGIN);
 137                 fallthrough;
 138         case _ASCE_TYPE_SEGMENT:
 139                 table += (address & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
 140                 if (get_kernel_nofault(entry, table))
 141                         goto bad;
 142                 pr_cont("S:%016lx ", entry);
 143                 if (entry & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE))
 144                         goto out;
 145                 table = __va(entry & _SEGMENT_ENTRY_ORIGIN);
 146         }
 147         table += (address & _PAGE_INDEX) >> _PAGE_SHIFT;
 148         if (get_kernel_nofault(entry, table))
 149                 goto bad;
 150         pr_cont("P:%016lx ", entry);
 151 out:
 152         pr_cont("\n");
 153         return;
 154 bad:
 155         pr_cont("BAD\n");
 156 }
 157
 158 static void dump_fault_info(struct pt_regs *regs)
 159 {
 160         union teid teid = { .val = regs->int_parm_long };
 161         unsigned long asce;
 162
 163         pr_alert("Failing address: %016lx TEID: %016lx\n",
 164                  get_fault_address(regs), teid.val);
 165         pr_alert("Fault in ");
 166         switch (teid.as) {
 167         case PSW_BITS_AS_HOME:
 168                 pr_cont("home space ");
 169                 break;
 170         case PSW_BITS_AS_SECONDARY:
 171                 pr_cont("secondary space ");
 172                 break;
 173         case PSW_BITS_AS_ACCREG:
 174                 pr_cont("access register ");
 175                 break;
 176         case PSW_BITS_AS_PRIMARY:
 177                 pr_cont("primary space ");
 178                 break;
 179         }
 180         pr_cont("mode while using ");
 181         switch (get_fault_type(regs)) {
 182         case USER_FAULT:
 183                 asce = S390_lowcore.user_asce.val;
 184                 pr_cont("user ");
 185                 break;
 186         case GMAP_FAULT:
 187                 asce = ((struct gmap *)S390_lowcore.gmap)->asce;
 188                 pr_cont("gmap ");
 189                 break;
 190         case KERNEL_FAULT:
 191                 asce = S390_lowcore.kernel_asce.val;
 192                 pr_cont("kernel ");
 193                 break;
 194         default:
 195                 unreachable();
 196         }
 197         pr_cont("ASCE.\n");
 198         dump_pagetable(asce, get_fault_address(regs));
 199 }
 200
 201 int show_unhandled_signals = 1;
 202
 203 void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault)
 204 {
 205         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
 206
 207         if ((task_pid_nr(current) > 1) && !show_unhandled_signals)
 208                 return;
 209         if (!unhandled_signal(current, signr))
 210                 return;
 211         if (!__ratelimit(&rs))
 212                 return;
 213         pr_alert("User process fault: interruption code %04x ilc:%d ",
 214                  regs->int_code & 0xffff, regs->int_code >> 17);
 215         print_vma_addr(KERN_CONT "in ", regs->psw.addr);
 216         pr_cont("\n");
 217         if (is_mm_fault)
 218                 dump_fault_info(regs);
 219         show_regs(regs);
 220 }
 221
 222 static void do_sigsegv(struct pt_regs *regs, int si_code)
 223 {
 224         report_user_fault(regs, SIGSEGV, 1);
 225         force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs));
 226 }
 227
 228 static void handle_fault_error_nolock(struct pt_regs *regs, int si_code)
 229 {
 230         enum fault_type fault_type;
 231         unsigned long address;
 232         bool is_write;
 233
 234         if (user_mode(regs)) {
 235                 if (WARN_ON_ONCE(!si_code))
 236                         si_code = SEGV_MAPERR;
 237                 return do_sigsegv(regs, si_code);
 238         }
 239         if (fixup_exception(regs))
 240                 return;
 241         fault_type = get_fault_type(regs);
 242         if (fault_type == KERNEL_FAULT) {
 243                 address = get_fault_address(regs);
 244                 is_write = fault_is_write(regs);
 245                 if (kfence_handle_page_fault(address, is_write, regs))
 246                         return;
 247         }
 248         if (fault_type == KERNEL_FAULT)
 249                 pr_alert("Unable to handle kernel pointer dereference in virtual kernel address space\n");
 250         else
 251                 pr_alert("Unable to handle kernel paging request in virtual user address space\n");
 252         dump_fault_info(regs);
 253         die(regs, "Oops");
 254 }
 255
 256 static void handle_fault_error(struct pt_regs *regs, int si_code)
 257 {
 258         struct mm_struct *mm = current->mm;
 259
 260         mmap_read_unlock(mm);
 261         handle_fault_error_nolock(regs, si_code);
 262 }
 263
 264 static void do_sigbus(struct pt_regs *regs)
 265 {
 266         force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs));
 267 }
 268
 269 /*
 270  * This routine handles page faults.  It determines the address,
 271  * and the problem, and then passes it off to one of the appropriate
 272  * routines.
 273  *
 274  * interruption code (int_code):
 275  *   04       Protection           ->  Write-Protection  (suppression)
 276  *   10       Segment translation  ->  Not present       (nullification)
 277  *   11       Page translation     ->  Not present       (nullification)
 278  *   3b       Region third trans.  ->  Not present       (nullification)
 279  */
 280 static void do_exception(struct pt_regs *regs, int access)
 281 {
 282         struct vm_area_struct *vma;
 283         unsigned long address;
 284         struct mm_struct *mm;
 285         enum fault_type type;
 286         unsigned int flags;
 287         struct gmap *gmap;
 288         vm_fault_t fault;
 289         bool is_write;
 290
 291         /*
 292          * The instruction that caused the program check has
 293          * been nullified. Don't signal single step via SIGTRAP.
 294          */
 295         clear_thread_flag(TIF_PER_TRAP);
 296         if (kprobe_page_fault(regs, 14))
 297                 return;
 298         mm = current->mm;
 299         address = get_fault_address(regs);
 300         is_write = fault_is_write(regs);
 301         type = get_fault_type(regs);
 302         switch (type) {
 303         case KERNEL_FAULT:
 304                 return handle_fault_error_nolock(regs, 0);
 305         case USER_FAULT:
 306         case GMAP_FAULT:
 307                 if (faulthandler_disabled() || !mm)
 308                         return handle_fault_error_nolock(regs, 0);
 309                 break;
 310         }
 311         perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 312         flags = FAULT_FLAG_DEFAULT;
 313         if (user_mode(regs))
 314                 flags |= FAULT_FLAG_USER;
 315         if (is_write)
 316                 access = VM_WRITE;
 317         if (access == VM_WRITE)
 318                 flags |= FAULT_FLAG_WRITE;
 319         if (!(flags & FAULT_FLAG_USER))
 320                 goto lock_mmap;
 321         vma = lock_vma_under_rcu(mm, address);
 322         if (!vma)
 323                 goto lock_mmap;
 324         if (!(vma->vm_flags & access)) {
 325                 vma_end_read(vma);
 326                 goto lock_mmap;
 327         }
 328         fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
 329         if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
 330                 vma_end_read(vma);
 331         if (!(fault & VM_FAULT_RETRY)) {
 332                 count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
 333                 if (unlikely(fault & VM_FAULT_ERROR))
 334                         goto error;
 335                 return;
 336         }
 337         count_vm_vma_lock_event(VMA_LOCK_RETRY);
 338         if (fault & VM_FAULT_MAJOR)
 339                 flags |= FAULT_FLAG_TRIED;
 340
 341         /* Quick path to respond to signals */
 342         if (fault_signal_pending(fault, regs)) {
 343                 if (!user_mode(regs))
 344                         handle_fault_error_nolock(regs, 0);
 345                 return;
 346         }
 347 lock_mmap:
 348         mmap_read_lock(mm);
 349         gmap = NULL;
 350         if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) {
 351                 gmap = (struct gmap *)S390_lowcore.gmap;
 352                 current->thread.gmap_addr = address;
 353                 current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE);
 354                 current->thread.gmap_int_code = regs->int_code & 0xffff;
 355                 address = __gmap_translate(gmap, address);
 356                 if (address == -EFAULT)
 357                         return handle_fault_error(regs, SEGV_MAPERR);
 358                 if (gmap->pfault_enabled)
 359                         flags |= FAULT_FLAG_RETRY_NOWAIT;
 360         }
 361 retry:
 362         vma = find_vma(mm, address);
 363         if (!vma)
 364                 return handle_fault_error(regs, SEGV_MAPERR);
 365         if (unlikely(vma->vm_start > address)) {
 366                 if (!(vma->vm_flags & VM_GROWSDOWN))
 367                         return handle_fault_error(regs, SEGV_MAPERR);
 368                 vma = expand_stack(mm, address);
 369                 if (!vma)
 370                         return handle_fault_error_nolock(regs, SEGV_MAPERR);
 371         }
 372         if (unlikely(!(vma->vm_flags & access)))
 373                 return handle_fault_error(regs, SEGV_ACCERR);
 374         fault = handle_mm_fault(vma, address, flags, regs);
 375         if (fault_signal_pending(fault, regs)) {
 376                 if (flags & FAULT_FLAG_RETRY_NOWAIT)
 377                         mmap_read_unlock(mm);
 378                 if (!user_mode(regs))
 379                         handle_fault_error_nolock(regs, 0);
 380                 return;
 381         }
 382         /* The fault is fully completed (including releasing mmap lock) */
 383         if (fault & VM_FAULT_COMPLETED) {
 384                 if (gmap) {
 385                         mmap_read_lock(mm);
 386                         goto gmap;
 387                 }
 388                 return;
 389         }
 390         if (unlikely(fault & VM_FAULT_ERROR)) {
 391                 mmap_read_unlock(mm);
 392                 goto error;
 393         }
 394         if (fault & VM_FAULT_RETRY) {
 395                 if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) {
 396                         /*
 397                          * FAULT_FLAG_RETRY_NOWAIT has been set,
 398                          * mmap_lock has not been released
 399                          */
 400                         current->thread.gmap_pfault = 1;
 401                         return handle_fault_error(regs, 0);
 402                 }
 403                 flags &= ~FAULT_FLAG_RETRY_NOWAIT;
 404                 flags |= FAULT_FLAG_TRIED;
 405                 mmap_read_lock(mm);
 406                 goto retry;
 407         }
 408 gmap:
 409         if (IS_ENABLED(CONFIG_PGSTE) && gmap) {
 410                 address =  __gmap_link(gmap, current->thread.gmap_addr,
 411                                        address);
 412                 if (address == -EFAULT)
 413                         return handle_fault_error(regs, SEGV_MAPERR);
 414                 if (address == -ENOMEM) {
 415                         fault = VM_FAULT_OOM;
 416                         mmap_read_unlock(mm);
 417                         goto error;
 418                 }
 419         }
 420         mmap_read_unlock(mm);
 421         return;
 422 error:
 423         if (fault & VM_FAULT_OOM) {
 424                 if (!user_mode(regs))
 425                         handle_fault_error_nolock(regs, 0);
 426                 else
 427                         pagefault_out_of_memory();
 428         } else if (fault & VM_FAULT_SIGSEGV) {
 429                 if (!user_mode(regs))
 430                         handle_fault_error_nolock(regs, 0);
 431                 else
 432                         do_sigsegv(regs, SEGV_MAPERR);
 433         } else if (fault & VM_FAULT_SIGBUS) {
 434                 if (!user_mode(regs))
 435                         handle_fault_error_nolock(regs, 0);
 436                 else
 437                         do_sigbus(regs);
 438         } else {
 439                 BUG();
 440         }
 441 }
 442
 443 void do_protection_exception(struct pt_regs *regs)
 444 {
 445         union teid teid = { .val = regs->int_parm_long };
 446
 447         /*
 448          * Protection exceptions are suppressing, decrement psw address.
 449          * The exception to this rule are aborted transactions, for these
 450          * the PSW already points to the correct location.
 451          */
 452         if (!(regs->int_code & 0x200))
 453                 regs->psw.addr = __rewind_psw(regs->psw, regs->int_code >> 16);
 454         /*
 455          * Check for low-address protection.  This needs to be treated
 456          * as a special case because the translation exception code
 457          * field is not guaranteed to contain valid data in this case.
 458          */
 459         if (unlikely(!teid.b61)) {
 460                 if (user_mode(regs)) {
 461                         /* Low-address protection in user mode: cannot happen */
 462                         die(regs, "Low-address protection");
 463                 }
 464                 /*
 465                  * Low-address protection in kernel mode means
 466                  * NULL pointer write access in kernel mode.
 467                  */
 468                 return handle_fault_error_nolock(regs, 0);
 469         }
 470         if (unlikely(MACHINE_HAS_NX && teid.b56)) {
 471                 regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK);
 472                 return handle_fault_error_nolock(regs, SEGV_ACCERR);
 473         }
 474         do_exception(regs, VM_WRITE);
 475 }
 476 NOKPROBE_SYMBOL(do_protection_exception);
 477
 478 void do_dat_exception(struct pt_regs *regs)
 479 {
 480         do_exception(regs, VM_ACCESS_FLAGS);
 481 }
 482 NOKPROBE_SYMBOL(do_dat_exception);
 483
 484 #if IS_ENABLED(CONFIG_PGSTE)
 485
 486 void do_secure_storage_access(struct pt_regs *regs)
 487 {
 488         union teid teid = { .val = regs->int_parm_long };
 489         unsigned long addr = get_fault_address(regs);
 490         struct vm_area_struct *vma;
 491         struct mm_struct *mm;
 492         struct page *page;
 493         struct gmap *gmap;
 494         int rc;
 495
 496         /*
 497          * Bit 61 indicates if the address is valid, if it is not the
 498          * kernel should be stopped or SIGSEGV should be sent to the
 499          * process. Bit 61 is not reliable without the misc UV feature,
 500          * therefore this needs to be checked too.
 501          */
 502         if (uv_has_feature(BIT_UV_FEAT_MISC) && !teid.b61) {
 503                 /*
 504                  * When this happens, userspace did something that it
 505                  * was not supposed to do, e.g. branching into secure
 506                  * memory. Trigger a segmentation fault.
 507                  */
 508                 if (user_mode(regs)) {
 509                         send_sig(SIGSEGV, current, 0);
 510                         return;
 511                 }
 512                 /*
 513                  * The kernel should never run into this case and
 514                  * there is no way out of this situation.
 515                  */
 516                 panic("Unexpected PGM 0x3d with TEID bit 61=0");
 517         }
 518         switch (get_fault_type(regs)) {
 519         case GMAP_FAULT:
 520                 mm = current->mm;
 521                 gmap = (struct gmap *)S390_lowcore.gmap;
 522                 mmap_read_lock(mm);
 523                 addr = __gmap_translate(gmap, addr);
 524                 mmap_read_unlock(mm);
 525                 if (IS_ERR_VALUE(addr))
 526                         return handle_fault_error_nolock(regs, SEGV_MAPERR);
 527                 fallthrough;
 528         case USER_FAULT:
 529                 mm = current->mm;
 530                 mmap_read_lock(mm);
 531                 vma = find_vma(mm, addr);
 532                 if (!vma)
 533                         return handle_fault_error(regs, SEGV_MAPERR);
 534                 page = follow_page(vma, addr, FOLL_WRITE | FOLL_GET);
 535                 if (IS_ERR_OR_NULL(page)) {
 536                         mmap_read_unlock(mm);
 537                         break;
 538                 }
 539                 if (arch_make_page_accessible(page))
 540                         send_sig(SIGSEGV, current, 0);
 541                 put_page(page);
 542                 mmap_read_unlock(mm);
 543                 break;
 544         case KERNEL_FAULT:
 545                 page = phys_to_page(addr);
 546                 if (unlikely(!try_get_page(page)))
 547                         break;
 548                 rc = arch_make_page_accessible(page);
 549                 put_page(page);
 550                 if (rc)
 551                         BUG();
 552                 break;
 553         default:
 554                 unreachable();
 555         }
 556 }
 557 NOKPROBE_SYMBOL(do_secure_storage_access);
 558
 559 void do_non_secure_storage_access(struct pt_regs *regs)
 560 {
 561         struct gmap *gmap = (struct gmap *)S390_lowcore.gmap;
 562         unsigned long gaddr = get_fault_address(regs);
 563
 564         if (WARN_ON_ONCE(get_fault_type(regs) != GMAP_FAULT))
 565                 return handle_fault_error_nolock(regs, SEGV_MAPERR);
 566         if (gmap_convert_to_secure(gmap, gaddr) == -EINVAL)
 567                 send_sig(SIGSEGV, current, 0);
 568 }
 569 NOKPROBE_SYMBOL(do_non_secure_storage_access);
 570
 571 void do_secure_storage_violation(struct pt_regs *regs)
 572 {
 573         struct gmap *gmap = (struct gmap *)S390_lowcore.gmap;
 574         unsigned long gaddr = get_fault_address(regs);
 575
 576         /*
 577          * If the VM has been rebooted, its address space might still contain
 578          * secure pages from the previous boot.
 579          * Clear the page so it can be reused.
 580          */
 581         if (!gmap_destroy_page(gmap, gaddr))
 582                 return;
 583         /*
 584          * Either KVM messed up the secure guest mapping or the same
 585          * page is mapped into multiple secure guests.
 586          *
 587          * This exception is only triggered when a guest 2 is running
 588          * and can therefore never occur in kernel context.
 589          */
 590         pr_warn_ratelimited("Secure storage violation in task: %s, pid %d\n",
 591                             current->comm, current->pid);
 592         send_sig(SIGSEGV, current, 0);
 593 }
 594
 595 #endif /* CONFIG_PGSTE */