arch/powerpc/kvm/book3s_64_mmu_hv.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *
   4  * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   5  */
   6
   7 #include <linux/types.h>
   8 #include <linux/string.h>
   9 #include <linux/kvm.h>
  10 #include <linux/kvm_host.h>
  11 #include <linux/highmem.h>
  12 #include <linux/gfp.h>
  13 #include <linux/slab.h>
  14 #include <linux/hugetlb.h>
  15 #include <linux/vmalloc.h>
  16 #include <linux/srcu.h>
  17 #include <linux/anon_inodes.h>
  18 #include <linux/file.h>
  19 #include <linux/debugfs.h>
  20
  21 #include <asm/kvm_ppc.h>
  22 #include <asm/kvm_book3s.h>
  23 #include <asm/book3s/64/mmu-hash.h>
  24 #include <asm/hvcall.h>
  25 #include <asm/synch.h>
  26 #include <asm/ppc-opcode.h>
  27 #include <asm/cputable.h>
  28 #include <asm/pte-walk.h>
  29
  30 #include "book3s.h"
  31 #include "trace_hv.h"
  32
  33 //#define DEBUG_RESIZE_HPT      1
  34
  35 #ifdef DEBUG_RESIZE_HPT
  36 #define resize_hpt_debug(resize, ...)                           \
  37         do {                                                    \
  38                 printk(KERN_DEBUG "RESIZE HPT %p: ", resize);   \
  39                 printk(__VA_ARGS__);                            \
  40         } while (0)
  41 #else
  42 #define resize_hpt_debug(resize, ...)                           \
  43         do { } while (0)
  44 #endif
  45
  46 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
  47                                 long pte_index, unsigned long pteh,
  48                                 unsigned long ptel, unsigned long *pte_idx_ret);
  49
  50 struct kvm_resize_hpt {
  51         /* These fields read-only after init */
  52         struct kvm *kvm;
  53         struct work_struct work;
  54         u32 order;
  55
  56         /* These fields protected by kvm->arch.mmu_setup_lock */
  57
  58         /* Possible values and their usage:
  59          *  <0     an error occurred during allocation,
  60          *  -EBUSY allocation is in the progress,
  61          *  0      allocation made successfully.
  62          */
  63         int error;
  64
  65         /* Private to the work thread, until error != -EBUSY,
  66          * then protected by kvm->arch.mmu_setup_lock.
  67          */
  68         struct kvm_hpt_info hpt;
  69 };
  70
  71 int kvmppc_allocate_hpt(struct kvm_hpt_info *info, u32 order)
  72 {
  73         unsigned long hpt = 0;
  74         int cma = 0;
  75         struct page *page = NULL;
  76         struct revmap_entry *rev;
  77         unsigned long npte;
  78
  79         if ((order < PPC_MIN_HPT_ORDER) || (order > PPC_MAX_HPT_ORDER))
  80                 return -EINVAL;
  81
  82         page = kvm_alloc_hpt_cma(1ul << (order - PAGE_SHIFT));
  83         if (page) {
  84                 hpt = (unsigned long)pfn_to_kaddr(page_to_pfn(page));
  85                 memset((void *)hpt, 0, (1ul << order));
  86                 cma = 1;
  87         }
  88
  89         if (!hpt)
  90                 hpt = __get_free_pages(GFP_KERNEL|__GFP_ZERO|__GFP_RETRY_MAYFAIL
  91                                        |__GFP_NOWARN, order - PAGE_SHIFT);
  92
  93         if (!hpt)
  94                 return -ENOMEM;
  95
  96         /* HPTEs are 2**4 bytes long */
  97         npte = 1ul << (order - 4);
  98
  99         /* Allocate reverse map array */
 100         rev = vmalloc(array_size(npte, sizeof(struct revmap_entry)));
 101         if (!rev) {
 102                 if (cma)
 103                         kvm_free_hpt_cma(page, 1 << (order - PAGE_SHIFT));
 104                 else
 105                         free_pages(hpt, order - PAGE_SHIFT);
 106                 return -ENOMEM;
 107         }
 108
 109         info->order = order;
 110         info->virt = hpt;
 111         info->cma = cma;
 112         info->rev = rev;
 113
 114         return 0;
 115 }
 116
 117 void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info)
 118 {
 119         atomic64_set(&kvm->arch.mmio_update, 0);
 120         kvm->arch.hpt = *info;
 121         kvm->arch.sdr1 = __pa(info->virt) | (info->order - 18);
 122
 123         pr_debug("KVM guest htab at %lx (order %ld), LPID %x\n",
 124                  info->virt, (long)info->order, kvm->arch.lpid);
 125 }
 126
 127 int kvmppc_alloc_reset_hpt(struct kvm *kvm, int order)
 128 {
 129         int err = -EBUSY;
 130         struct kvm_hpt_info info;
 131
 132         mutex_lock(&kvm->arch.mmu_setup_lock);
 133         if (kvm->arch.mmu_ready) {
 134                 kvm->arch.mmu_ready = 0;
 135                 /* order mmu_ready vs. vcpus_running */
 136                 smp_mb();
 137                 if (atomic_read(&kvm->arch.vcpus_running)) {
 138                         kvm->arch.mmu_ready = 1;
 139                         goto out;
 140                 }
 141         }
 142         if (kvm_is_radix(kvm)) {
 143                 err = kvmppc_switch_mmu_to_hpt(kvm);
 144                 if (err)
 145                         goto out;
 146         }
 147
 148         if (kvm->arch.hpt.order == order) {
 149                 /* We already have a suitable HPT */
 150
 151                 /* Set the entire HPT to 0, i.e. invalid HPTEs */
 152                 memset((void *)kvm->arch.hpt.virt, 0, 1ul << order);
 153                 /*
 154                  * Reset all the reverse-mapping chains for all memslots
 155                  */
 156                 kvmppc_rmap_reset(kvm);
 157                 err = 0;
 158                 goto out;
 159         }
 160
 161         if (kvm->arch.hpt.virt) {
 162                 kvmppc_free_hpt(&kvm->arch.hpt);
 163                 kvmppc_rmap_reset(kvm);
 164         }
 165
 166         err = kvmppc_allocate_hpt(&info, order);
 167         if (err < 0)
 168                 goto out;
 169         kvmppc_set_hpt(kvm, &info);
 170
 171 out:
 172         if (err == 0)
 173                 /* Ensure that each vcpu will flush its TLB on next entry. */
 174                 cpumask_setall(&kvm->arch.need_tlb_flush);
 175
 176         mutex_unlock(&kvm->arch.mmu_setup_lock);
 177         return err;
 178 }
 179
 180 void kvmppc_free_hpt(struct kvm_hpt_info *info)
 181 {
 182         vfree(info->rev);
 183         info->rev = NULL;
 184         if (info->cma)
 185                 kvm_free_hpt_cma(virt_to_page(info->virt),
 186                                  1 << (info->order - PAGE_SHIFT));
 187         else if (info->virt)
 188                 free_pages(info->virt, info->order - PAGE_SHIFT);
 189         info->virt = 0;
 190         info->order = 0;
 191 }
 192
 193 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
 194 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize)
 195 {
 196         return (pgsize > 0x1000) ? HPTE_V_LARGE : 0;
 197 }
 198
 199 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
 200 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize)
 201 {
 202         return (pgsize == 0x10000) ? 0x1000 : 0;
 203 }
 204
 205 void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 206                      unsigned long porder)
 207 {
 208         unsigned long i;
 209         unsigned long npages;
 210         unsigned long hp_v, hp_r;
 211         unsigned long addr, hash;
 212         unsigned long psize;
 213         unsigned long hp0, hp1;
 214         unsigned long idx_ret;
 215         long ret;
 216         struct kvm *kvm = vcpu->kvm;
 217
 218         psize = 1ul << porder;
 219         npages = memslot->npages >> (porder - PAGE_SHIFT);
 220
 221         /* VRMA can't be > 1TB */
 222         if (npages > 1ul << (40 - porder))
 223                 npages = 1ul << (40 - porder);
 224         /* Can't use more than 1 HPTE per HPTEG */
 225         if (npages > kvmppc_hpt_mask(&kvm->arch.hpt) + 1)
 226                 npages = kvmppc_hpt_mask(&kvm->arch.hpt) + 1;
 227
 228         hp0 = HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)) |
 229                 HPTE_V_BOLTED | hpte0_pgsize_encoding(psize);
 230         hp1 = hpte1_pgsize_encoding(psize) |
 231                 HPTE_R_R | HPTE_R_C | HPTE_R_M | PP_RWXX;
 232
 233         for (i = 0; i < npages; ++i) {
 234                 addr = i << porder;
 235                 /* can't use hpt_hash since va > 64 bits */
 236                 hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25)))
 237                         & kvmppc_hpt_mask(&kvm->arch.hpt);
 238                 /*
 239                  * We assume that the hash table is empty and no
 240                  * vcpus are using it at this stage.  Since we create
 241                  * at most one HPTE per HPTEG, we just assume entry 7
 242                  * is available and use it.
 243                  */
 244                 hash = (hash << 3) + 7;
 245                 hp_v = hp0 | ((addr >> 16) & ~0x7fUL);
 246                 hp_r = hp1 | addr;
 247                 ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,
 248                                                  &idx_ret);
 249                 if (ret != H_SUCCESS) {
 250                         pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
 251                                addr, ret);
 252                         break;
 253                 }
 254         }
 255 }
 256
 257 int kvmppc_mmu_hv_init(void)
 258 {
 259         unsigned long nr_lpids;
 260
 261         if (!mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE))
 262                 return -EINVAL;
 263
 264         if (cpu_has_feature(CPU_FTR_HVMODE)) {
 265                 if (WARN_ON(mfspr(SPRN_LPID) != 0))
 266                         return -EINVAL;
 267                 nr_lpids = 1UL << mmu_lpid_bits;
 268         } else {
 269                 nr_lpids = 1UL << KVM_MAX_NESTED_GUESTS_SHIFT;
 270         }
 271
 272         if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
 273                 /* POWER7 has 10-bit LPIDs, POWER8 has 12-bit LPIDs */
 274                 if (cpu_has_feature(CPU_FTR_ARCH_207S))
 275                         WARN_ON(nr_lpids != 1UL << 12);
 276                 else
 277                         WARN_ON(nr_lpids != 1UL << 10);
 278
 279                 /*
 280                  * Reserve the last implemented LPID use in partition
 281                  * switching for POWER7 and POWER8.
 282                  */
 283                 nr_lpids -= 1;
 284         }
 285
 286         kvmppc_init_lpid(nr_lpids);
 287
 288         return 0;
 289 }
 290
 291 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
 292                                 long pte_index, unsigned long pteh,
 293                                 unsigned long ptel, unsigned long *pte_idx_ret)
 294 {
 295         long ret;
 296
 297         preempt_disable();
 298         ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,
 299                                 kvm->mm->pgd, false, pte_idx_ret);
 300         preempt_enable();
 301         if (ret == H_TOO_HARD) {
 302                 /* this can't happen */
 303                 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
 304                 ret = H_RESOURCE;       /* or something */
 305         }
 306         return ret;
 307
 308 }
 309
 310 static struct kvmppc_slb *kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu *vcpu,
 311                                                          gva_t eaddr)
 312 {
 313         u64 mask;
 314         int i;
 315
 316         for (i = 0; i < vcpu->arch.slb_nr; i++) {
 317                 if (!(vcpu->arch.slb[i].orige & SLB_ESID_V))
 318                         continue;
 319
 320                 if (vcpu->arch.slb[i].origv & SLB_VSID_B_1T)
 321                         mask = ESID_MASK_1T;
 322                 else
 323                         mask = ESID_MASK;
 324
 325                 if (((vcpu->arch.slb[i].orige ^ eaddr) & mask) == 0)
 326                         return &vcpu->arch.slb[i];
 327         }
 328         return NULL;
 329 }
 330
 331 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v, unsigned long r,
 332                         unsigned long ea)
 333 {
 334         unsigned long ra_mask;
 335
 336         ra_mask = kvmppc_actual_pgsz(v, r) - 1;
 337         return (r & HPTE_R_RPN & ~ra_mask) | (ea & ra_mask);
 338 }
 339
 340 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 341                         struct kvmppc_pte *gpte, bool data, bool iswrite)
 342 {
 343         struct kvm *kvm = vcpu->kvm;
 344         struct kvmppc_slb *slbe;
 345         unsigned long slb_v;
 346         unsigned long pp, key;
 347         unsigned long v, orig_v, gr;
 348         __be64 *hptep;
 349         long int index;
 350         int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR);
 351
 352         if (kvm_is_radix(vcpu->kvm))
 353                 return kvmppc_mmu_radix_xlate(vcpu, eaddr, gpte, data, iswrite);
 354
 355         /* Get SLB entry */
 356         if (virtmode) {
 357                 slbe = kvmppc_mmu_book3s_hv_find_slbe(vcpu, eaddr);
 358                 if (!slbe)
 359                         return -EINVAL;
 360                 slb_v = slbe->origv;
 361         } else {
 362                 /* real mode access */
 363                 slb_v = vcpu->kvm->arch.vrma_slb_v;
 364         }
 365
 366         preempt_disable();
 367         /* Find the HPTE in the hash table */
 368         index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,
 369                                          HPTE_V_VALID | HPTE_V_ABSENT);
 370         if (index < 0) {
 371                 preempt_enable();
 372                 return -ENOENT;
 373         }
 374         hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
 375         v = orig_v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 376         if (cpu_has_feature(CPU_FTR_ARCH_300))
 377                 v = hpte_new_to_old_v(v, be64_to_cpu(hptep[1]));
 378         gr = kvm->arch.hpt.rev[index].guest_rpte;
 379
 380         unlock_hpte(hptep, orig_v);
 381         preempt_enable();
 382
 383         gpte->eaddr = eaddr;
 384         gpte->vpage = ((v & HPTE_V_AVPN) << 4) | ((eaddr >> 12) & 0xfff);
 385
 386         /* Get PP bits and key for permission check */
 387         pp = gr & (HPTE_R_PP0 | HPTE_R_PP);
 388         key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS;
 389         key &= slb_v;
 390
 391         /* Calculate permissions */
 392         gpte->may_read = hpte_read_permission(pp, key);
 393         gpte->may_write = hpte_write_permission(pp, key);
 394         gpte->may_execute = gpte->may_read && !(gr & (HPTE_R_N | HPTE_R_G));
 395
 396         /* Storage key permission check for POWER7 */
 397         if (data && virtmode) {
 398                 int amrfield = hpte_get_skey_perm(gr, vcpu->arch.amr);
 399                 if (amrfield & 1)
 400                         gpte->may_read = 0;
 401                 if (amrfield & 2)
 402                         gpte->may_write = 0;
 403         }
 404
 405         /* Get the guest physical address */
 406         gpte->raddr = kvmppc_mmu_get_real_addr(v, gr, eaddr);
 407         return 0;
 408 }
 409
 410 /*
 411  * Quick test for whether an instruction is a load or a store.
 412  * If the instruction is a load or a store, then this will indicate
 413  * which it is, at least on server processors.  (Embedded processors
 414  * have some external PID instructions that don't follow the rule
 415  * embodied here.)  If the instruction isn't a load or store, then
 416  * this doesn't return anything useful.
 417  */
 418 static int instruction_is_store(ppc_inst_t instr)
 419 {
 420         unsigned int mask;
 421         unsigned int suffix;
 422
 423         mask = 0x10000000;
 424         suffix = ppc_inst_val(instr);
 425         if (ppc_inst_prefixed(instr))
 426                 suffix = ppc_inst_suffix(instr);
 427         else if ((suffix & 0xfc000000) == 0x7c000000)
 428                 mask = 0x100;           /* major opcode 31 */
 429         return (suffix & mask) != 0;
 430 }
 431
 432 int kvmppc_hv_emulate_mmio(struct kvm_vcpu *vcpu,
 433                            unsigned long gpa, gva_t ea, int is_store)
 434 {
 435         ppc_inst_t last_inst;
 436         bool is_prefixed = !!(kvmppc_get_msr(vcpu) & SRR1_PREFIXED);
 437
 438         /*
 439          * Fast path - check if the guest physical address corresponds to a
 440          * device on the FAST_MMIO_BUS, if so we can avoid loading the
 441          * instruction all together, then we can just handle it and return.
 442          */
 443         if (is_store) {
 444                 int idx, ret;
 445
 446                 idx = srcu_read_lock(&vcpu->kvm->srcu);
 447                 ret = kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, (gpa_t) gpa, 0,
 448                                        NULL);
 449                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
 450                 if (!ret) {
 451                         kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + (is_prefixed ? 8 : 4));
 452                         return RESUME_GUEST;
 453                 }
 454         }
 455
 456         /*
 457          * If we fail, we just return to the guest and try executing it again.
 458          */
 459         if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
 460                 EMULATE_DONE)
 461                 return RESUME_GUEST;
 462
 463         /*
 464          * WARNING: We do not know for sure whether the instruction we just
 465          * read from memory is the same that caused the fault in the first
 466          * place.
 467          *
 468          * If the fault is prefixed but the instruction is not or vice
 469          * versa, try again so that we don't advance pc the wrong amount.
 470          */
 471         if (ppc_inst_prefixed(last_inst) != is_prefixed)
 472                 return RESUME_GUEST;
 473
 474         /*
 475          * If the instruction we read is neither an load or a store,
 476          * then it can't access memory, so we don't need to worry about
 477          * enforcing access permissions.  So, assuming it is a load or
 478          * store, we just check that its direction (load or store) is
 479          * consistent with the original fault, since that's what we
 480          * checked the access permissions against.  If there is a mismatch
 481          * we just return and retry the instruction.
 482          */
 483
 484         if (instruction_is_store(last_inst) != !!is_store)
 485                 return RESUME_GUEST;
 486
 487         /*
 488          * Emulated accesses are emulated by looking at the hash for
 489          * translation once, then performing the access later. The
 490          * translation could be invalidated in the meantime in which
 491          * point performing the subsequent memory access on the old
 492          * physical address could possibly be a security hole for the
 493          * guest (but not the host).
 494          *
 495          * This is less of an issue for MMIO stores since they aren't
 496          * globally visible. It could be an issue for MMIO loads to
 497          * a certain extent but we'll ignore it for now.
 498          */
 499
 500         vcpu->arch.paddr_accessed = gpa;
 501         vcpu->arch.vaddr_accessed = ea;
 502         return kvmppc_emulate_mmio(vcpu);
 503 }
 504
 505 int kvmppc_book3s_hv_page_fault(struct kvm_vcpu *vcpu,
 506                                 unsigned long ea, unsigned long dsisr)
 507 {
 508         struct kvm *kvm = vcpu->kvm;
 509         unsigned long hpte[3], r;
 510         unsigned long hnow_v, hnow_r;
 511         __be64 *hptep;
 512         unsigned long mmu_seq, psize, pte_size;
 513         unsigned long gpa_base, gfn_base;
 514         unsigned long gpa, gfn, hva, pfn, hpa;
 515         struct kvm_memory_slot *memslot;
 516         unsigned long *rmap;
 517         struct revmap_entry *rev;
 518         struct page *page;
 519         long index, ret;
 520         bool is_ci;
 521         bool writing, write_ok;
 522         unsigned int shift;
 523         unsigned long rcbits;
 524         long mmio_update;
 525         pte_t pte, *ptep;
 526
 527         if (kvm_is_radix(kvm))
 528                 return kvmppc_book3s_radix_page_fault(vcpu, ea, dsisr);
 529
 530         /*
 531          * Real-mode code has already searched the HPT and found the
 532          * entry we're interested in.  Lock the entry and check that
 533          * it hasn't changed.  If it has, just return and re-execute the
 534          * instruction.
 535          */
 536         if (ea != vcpu->arch.pgfault_addr)
 537                 return RESUME_GUEST;
 538
 539         if (vcpu->arch.pgfault_cache) {
 540                 mmio_update = atomic64_read(&kvm->arch.mmio_update);
 541                 if (mmio_update == vcpu->arch.pgfault_cache->mmio_update) {
 542                         r = vcpu->arch.pgfault_cache->rpte;
 543                         psize = kvmppc_actual_pgsz(vcpu->arch.pgfault_hpte[0],
 544                                                    r);
 545                         gpa_base = r & HPTE_R_RPN & ~(psize - 1);
 546                         gfn_base = gpa_base >> PAGE_SHIFT;
 547                         gpa = gpa_base | (ea & (psize - 1));
 548                         return kvmppc_hv_emulate_mmio(vcpu, gpa, ea,
 549                                                 dsisr & DSISR_ISSTORE);
 550                 }
 551         }
 552         index = vcpu->arch.pgfault_index;
 553         hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
 554         rev = &kvm->arch.hpt.rev[index];
 555         preempt_disable();
 556         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 557                 cpu_relax();
 558         hpte[0] = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 559         hpte[1] = be64_to_cpu(hptep[1]);
 560         hpte[2] = r = rev->guest_rpte;
 561         unlock_hpte(hptep, hpte[0]);
 562         preempt_enable();
 563
 564         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 565                 hpte[0] = hpte_new_to_old_v(hpte[0], hpte[1]);
 566                 hpte[1] = hpte_new_to_old_r(hpte[1]);
 567         }
 568         if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
 569             hpte[1] != vcpu->arch.pgfault_hpte[1])
 570                 return RESUME_GUEST;
 571
 572         /* Translate the logical address and get the page */
 573         psize = kvmppc_actual_pgsz(hpte[0], r);
 574         gpa_base = r & HPTE_R_RPN & ~(psize - 1);
 575         gfn_base = gpa_base >> PAGE_SHIFT;
 576         gpa = gpa_base | (ea & (psize - 1));
 577         gfn = gpa >> PAGE_SHIFT;
 578         memslot = gfn_to_memslot(kvm, gfn);
 579
 580         trace_kvm_page_fault_enter(vcpu, hpte, memslot, ea, dsisr);
 581
 582         /* No memslot means it's an emulated MMIO region */
 583         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
 584                 return kvmppc_hv_emulate_mmio(vcpu, gpa, ea,
 585                                               dsisr & DSISR_ISSTORE);
 586
 587         /*
 588          * This should never happen, because of the slot_is_aligned()
 589          * check in kvmppc_do_h_enter().
 590          */
 591         if (gfn_base < memslot->base_gfn)
 592                 return -EFAULT;
 593
 594         /* used to check for invalidations in progress */
 595         mmu_seq = kvm->mmu_invalidate_seq;
 596         smp_rmb();
 597
 598         ret = -EFAULT;
 599         page = NULL;
 600         writing = (dsisr & DSISR_ISSTORE) != 0;
 601         /* If writing != 0, then the HPTE must allow writing, if we get here */
 602         write_ok = writing;
 603         hva = gfn_to_hva_memslot(memslot, gfn);
 604
 605         /*
 606          * Do a fast check first, since __gfn_to_pfn_memslot doesn't
 607          * do it with !atomic && !async, which is how we call it.
 608          * We always ask for write permission since the common case
 609          * is that the page is writable.
 610          */
 611         if (get_user_page_fast_only(hva, FOLL_WRITE, &page)) {
 612                 write_ok = true;
 613         } else {
 614                 /* Call KVM generic code to do the slow-path check */
 615                 pfn = __gfn_to_pfn_memslot(memslot, gfn, false, false, NULL,
 616                                            writing, &write_ok, NULL);
 617                 if (is_error_noslot_pfn(pfn))
 618                         return -EFAULT;
 619                 page = NULL;
 620                 if (pfn_valid(pfn)) {
 621                         page = pfn_to_page(pfn);
 622                         if (PageReserved(page))
 623                                 page = NULL;
 624                 }
 625         }
 626
 627         /*
 628          * Read the PTE from the process' radix tree and use that
 629          * so we get the shift and attribute bits.
 630          */
 631         spin_lock(&kvm->mmu_lock);
 632         ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift);
 633         pte = __pte(0);
 634         if (ptep)
 635                 pte = READ_ONCE(*ptep);
 636         spin_unlock(&kvm->mmu_lock);
 637         /*
 638          * If the PTE disappeared temporarily due to a THP
 639          * collapse, just return and let the guest try again.
 640          */
 641         if (!pte_present(pte)) {
 642                 if (page)
 643                         put_page(page);
 644                 return RESUME_GUEST;
 645         }
 646         hpa = pte_pfn(pte) << PAGE_SHIFT;
 647         pte_size = PAGE_SIZE;
 648         if (shift)
 649                 pte_size = 1ul << shift;
 650         is_ci = pte_ci(pte);
 651
 652         if (psize > pte_size)
 653                 goto out_put;
 654         if (pte_size > psize)
 655                 hpa |= hva & (pte_size - psize);
 656
 657         /* Check WIMG vs. the actual page we're accessing */
 658         if (!hpte_cache_flags_ok(r, is_ci)) {
 659                 if (is_ci)
 660                         goto out_put;
 661                 /*
 662                  * Allow guest to map emulated device memory as
 663                  * uncacheable, but actually make it cacheable.
 664                  */
 665                 r = (r & ~(HPTE_R_W|HPTE_R_I|HPTE_R_G)) | HPTE_R_M;
 666         }
 667
 668         /*
 669          * Set the HPTE to point to hpa.
 670          * Since the hpa is at PAGE_SIZE granularity, make sure we
 671          * don't mask out lower-order bits if psize < PAGE_SIZE.
 672          */
 673         if (psize < PAGE_SIZE)
 674                 psize = PAGE_SIZE;
 675         r = (r & HPTE_R_KEY_HI) | (r & ~(HPTE_R_PP0 - psize)) | hpa;
 676         if (hpte_is_writable(r) && !write_ok)
 677                 r = hpte_make_readonly(r);
 678         ret = RESUME_GUEST;
 679         preempt_disable();
 680         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 681                 cpu_relax();
 682         hnow_v = be64_to_cpu(hptep[0]);
 683         hnow_r = be64_to_cpu(hptep[1]);
 684         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 685                 hnow_v = hpte_new_to_old_v(hnow_v, hnow_r);
 686                 hnow_r = hpte_new_to_old_r(hnow_r);
 687         }
 688
 689         /*
 690          * If the HPT is being resized, don't update the HPTE,
 691          * instead let the guest retry after the resize operation is complete.
 692          * The synchronization for mmu_ready test vs. set is provided
 693          * by the HPTE lock.
 694          */
 695         if (!kvm->arch.mmu_ready)
 696                 goto out_unlock;
 697
 698         if ((hnow_v & ~HPTE_V_HVLOCK) != hpte[0] || hnow_r != hpte[1] ||
 699             rev->guest_rpte != hpte[2])
 700                 /* HPTE has been changed under us; let the guest retry */
 701                 goto out_unlock;
 702         hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
 703
 704         /* Always put the HPTE in the rmap chain for the page base address */
 705         rmap = &memslot->arch.rmap[gfn_base - memslot->base_gfn];
 706         lock_rmap(rmap);
 707
 708         /* Check if we might have been invalidated; let the guest retry if so */
 709         ret = RESUME_GUEST;
 710         if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) {
 711                 unlock_rmap(rmap);
 712                 goto out_unlock;
 713         }
 714
 715         /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
 716         rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
 717         r &= rcbits | ~(HPTE_R_R | HPTE_R_C);
 718
 719         if (be64_to_cpu(hptep[0]) & HPTE_V_VALID) {
 720                 /* HPTE was previously valid, so we need to invalidate it */
 721                 unlock_rmap(rmap);
 722                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 723                 kvmppc_invalidate_hpte(kvm, hptep, index);
 724                 /* don't lose previous R and C bits */
 725                 r |= be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 726         } else {
 727                 kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
 728         }
 729
 730         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 731                 r = hpte_old_to_new_r(hpte[0], r);
 732                 hpte[0] = hpte_old_to_new_v(hpte[0]);
 733         }
 734         hptep[1] = cpu_to_be64(r);
 735         eieio();
 736         __unlock_hpte(hptep, hpte[0]);
 737         asm volatile("ptesync" : : : "memory");
 738         preempt_enable();
 739         if (page && hpte_is_writable(r))
 740                 set_page_dirty_lock(page);
 741
 742  out_put:
 743         trace_kvm_page_fault_exit(vcpu, hpte, ret);
 744
 745         if (page)
 746                 put_page(page);
 747         return ret;
 748
 749  out_unlock:
 750         __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 751         preempt_enable();
 752         goto out_put;
 753 }
 754
 755 void kvmppc_rmap_reset(struct kvm *kvm)
 756 {
 757         struct kvm_memslots *slots;
 758         struct kvm_memory_slot *memslot;
 759         int srcu_idx, bkt;
 760
 761         srcu_idx = srcu_read_lock(&kvm->srcu);
 762         slots = kvm_memslots(kvm);
 763         kvm_for_each_memslot(memslot, bkt, slots) {
 764                 /* Mutual exclusion with kvm_unmap_hva_range etc. */
 765                 spin_lock(&kvm->mmu_lock);
 766                 /*
 767                  * This assumes it is acceptable to lose reference and
 768                  * change bits across a reset.
 769                  */
 770                 memset(memslot->arch.rmap, 0,
 771                        memslot->npages * sizeof(*memslot->arch.rmap));
 772                 spin_unlock(&kvm->mmu_lock);
 773         }
 774         srcu_read_unlock(&kvm->srcu, srcu_idx);
 775 }
 776
 777 /* Must be called with both HPTE and rmap locked */
 778 static void kvmppc_unmap_hpte(struct kvm *kvm, unsigned long i,
 779                               struct kvm_memory_slot *memslot,
 780                               unsigned long *rmapp, unsigned long gfn)
 781 {
 782         __be64 *hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 783         struct revmap_entry *rev = kvm->arch.hpt.rev;
 784         unsigned long j, h;
 785         unsigned long ptel, psize, rcbits;
 786
 787         j = rev[i].forw;
 788         if (j == i) {
 789                 /* chain is now empty */
 790                 *rmapp &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX);
 791         } else {
 792                 /* remove i from chain */
 793                 h = rev[i].back;
 794                 rev[h].forw = j;
 795                 rev[j].back = h;
 796                 rev[i].forw = rev[i].back = i;
 797                 *rmapp = (*rmapp & ~KVMPPC_RMAP_INDEX) | j;
 798         }
 799
 800         /* Now check and modify the HPTE */
 801         ptel = rev[i].guest_rpte;
 802         psize = kvmppc_actual_pgsz(be64_to_cpu(hptep[0]), ptel);
 803         if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 804             hpte_rpn(ptel, psize) == gfn) {
 805                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 806                 kvmppc_invalidate_hpte(kvm, hptep, i);
 807                 hptep[1] &= ~cpu_to_be64(HPTE_R_KEY_HI | HPTE_R_KEY_LO);
 808                 /* Harvest R and C */
 809                 rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 810                 *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT;
 811                 if ((rcbits & HPTE_R_C) && memslot->dirty_bitmap)
 812                         kvmppc_update_dirty_map(memslot, gfn, psize);
 813                 if (rcbits & ~rev[i].guest_rpte) {
 814                         rev[i].guest_rpte = ptel | rcbits;
 815                         note_hpte_modification(kvm, &rev[i]);
 816                 }
 817         }
 818 }
 819
 820 static void kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 821                             unsigned long gfn)
 822 {
 823         unsigned long i;
 824         __be64 *hptep;
 825         unsigned long *rmapp;
 826
 827         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 828         for (;;) {
 829                 lock_rmap(rmapp);
 830                 if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 831                         unlock_rmap(rmapp);
 832                         break;
 833                 }
 834
 835                 /*
 836                  * To avoid an ABBA deadlock with the HPTE lock bit,
 837                  * we can't spin on the HPTE lock while holding the
 838                  * rmap chain lock.
 839                  */
 840                 i = *rmapp & KVMPPC_RMAP_INDEX;
 841                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 842                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 843                         /* unlock rmap before spinning on the HPTE lock */
 844                         unlock_rmap(rmapp);
 845                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 846                                 cpu_relax();
 847                         continue;
 848                 }
 849
 850                 kvmppc_unmap_hpte(kvm, i, memslot, rmapp, gfn);
 851                 unlock_rmap(rmapp);
 852                 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 853         }
 854 }
 855
 856 bool kvm_unmap_gfn_range_hv(struct kvm *kvm, struct kvm_gfn_range *range)
 857 {
 858         gfn_t gfn;
 859
 860         if (kvm_is_radix(kvm)) {
 861                 for (gfn = range->start; gfn < range->end; gfn++)
 862                         kvm_unmap_radix(kvm, range->slot, gfn);
 863         } else {
 864                 for (gfn = range->start; gfn < range->end; gfn++)
 865                         kvm_unmap_rmapp(kvm, range->slot, gfn);
 866         }
 867
 868         return false;
 869 }
 870
 871 void kvmppc_core_flush_memslot_hv(struct kvm *kvm,
 872                                   struct kvm_memory_slot *memslot)
 873 {
 874         unsigned long gfn;
 875         unsigned long n;
 876         unsigned long *rmapp;
 877
 878         gfn = memslot->base_gfn;
 879         rmapp = memslot->arch.rmap;
 880         if (kvm_is_radix(kvm)) {
 881                 kvmppc_radix_flush_memslot(kvm, memslot);
 882                 return;
 883         }
 884
 885         for (n = memslot->npages; n; --n, ++gfn) {
 886                 /*
 887                  * Testing the present bit without locking is OK because
 888                  * the memslot has been marked invalid already, and hence
 889                  * no new HPTEs referencing this page can be created,
 890                  * thus the present bit can't go from 0 to 1.
 891                  */
 892                 if (*rmapp & KVMPPC_RMAP_PRESENT)
 893                         kvm_unmap_rmapp(kvm, memslot, gfn);
 894                 ++rmapp;
 895         }
 896 }
 897
 898 static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 899                           unsigned long gfn)
 900 {
 901         struct revmap_entry *rev = kvm->arch.hpt.rev;
 902         unsigned long head, i, j;
 903         __be64 *hptep;
 904         bool ret = false;
 905         unsigned long *rmapp;
 906
 907         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 908  retry:
 909         lock_rmap(rmapp);
 910         if (*rmapp & KVMPPC_RMAP_REFERENCED) {
 911                 *rmapp &= ~KVMPPC_RMAP_REFERENCED;
 912                 ret = true;
 913         }
 914         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 915                 unlock_rmap(rmapp);
 916                 return ret;
 917         }
 918
 919         i = head = *rmapp & KVMPPC_RMAP_INDEX;
 920         do {
 921                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 922                 j = rev[i].forw;
 923
 924                 /* If this HPTE isn't referenced, ignore it */
 925                 if (!(be64_to_cpu(hptep[1]) & HPTE_R_R))
 926                         continue;
 927
 928                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 929                         /* unlock rmap before spinning on the HPTE lock */
 930                         unlock_rmap(rmapp);
 931                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 932                                 cpu_relax();
 933                         goto retry;
 934                 }
 935
 936                 /* Now check and modify the HPTE */
 937                 if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 938                     (be64_to_cpu(hptep[1]) & HPTE_R_R)) {
 939                         kvmppc_clear_ref_hpte(kvm, hptep, i);
 940                         if (!(rev[i].guest_rpte & HPTE_R_R)) {
 941                                 rev[i].guest_rpte |= HPTE_R_R;
 942                                 note_hpte_modification(kvm, &rev[i]);
 943                         }
 944                         ret = true;
 945                 }
 946                 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 947         } while ((i = j) != head);
 948
 949         unlock_rmap(rmapp);
 950         return ret;
 951 }
 952
 953 bool kvm_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range)
 954 {
 955         gfn_t gfn;
 956         bool ret = false;
 957
 958         if (kvm_is_radix(kvm)) {
 959                 for (gfn = range->start; gfn < range->end; gfn++)
 960                         ret |= kvm_age_radix(kvm, range->slot, gfn);
 961         } else {
 962                 for (gfn = range->start; gfn < range->end; gfn++)
 963                         ret |= kvm_age_rmapp(kvm, range->slot, gfn);
 964         }
 965
 966         return ret;
 967 }
 968
 969 static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 970                                unsigned long gfn)
 971 {
 972         struct revmap_entry *rev = kvm->arch.hpt.rev;
 973         unsigned long head, i, j;
 974         unsigned long *hp;
 975         bool ret = true;
 976         unsigned long *rmapp;
 977
 978         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 979         if (*rmapp & KVMPPC_RMAP_REFERENCED)
 980                 return true;
 981
 982         lock_rmap(rmapp);
 983         if (*rmapp & KVMPPC_RMAP_REFERENCED)
 984                 goto out;
 985
 986         if (*rmapp & KVMPPC_RMAP_PRESENT) {
 987                 i = head = *rmapp & KVMPPC_RMAP_INDEX;
 988                 do {
 989                         hp = (unsigned long *)(kvm->arch.hpt.virt + (i << 4));
 990                         j = rev[i].forw;
 991                         if (be64_to_cpu(hp[1]) & HPTE_R_R)
 992                                 goto out;
 993                 } while ((i = j) != head);
 994         }
 995         ret = false;
 996
 997  out:
 998         unlock_rmap(rmapp);
 999         return ret;
1000 }
1001
1002 bool kvm_test_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range)
1003 {
1004         WARN_ON(range->start + 1 != range->end);
1005
1006         if (kvm_is_radix(kvm))
1007                 return kvm_test_age_radix(kvm, range->slot, range->start);
1008         else
1009                 return kvm_test_age_rmapp(kvm, range->slot, range->start);
1010 }
1011
1012 bool kvm_set_spte_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range)
1013 {
1014         WARN_ON(range->start + 1 != range->end);
1015
1016         if (kvm_is_radix(kvm))
1017                 kvm_unmap_radix(kvm, range->slot, range->start);
1018         else
1019                 kvm_unmap_rmapp(kvm, range->slot, range->start);
1020
1021         return false;
1022 }
1023
1024 static int vcpus_running(struct kvm *kvm)
1025 {
1026         return atomic_read(&kvm->arch.vcpus_running) != 0;
1027 }
1028
1029 /*
1030  * Returns the number of system pages that are dirty.
1031  * This can be more than 1 if we find a huge-page HPTE.
1032  */
1033 static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
1034 {
1035         struct revmap_entry *rev = kvm->arch.hpt.rev;
1036         unsigned long head, i, j;
1037         unsigned long n;
1038         unsigned long v, r;
1039         __be64 *hptep;
1040         int npages_dirty = 0;
1041
1042  retry:
1043         lock_rmap(rmapp);
1044         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
1045                 unlock_rmap(rmapp);
1046                 return npages_dirty;
1047         }
1048
1049         i = head = *rmapp & KVMPPC_RMAP_INDEX;
1050         do {
1051                 unsigned long hptep1;
1052                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
1053                 j = rev[i].forw;
1054
1055                 /*
1056                  * Checking the C (changed) bit here is racy since there
1057                  * is no guarantee about when the hardware writes it back.
1058                  * If the HPTE is not writable then it is stable since the
1059                  * page can't be written to, and we would have done a tlbie
1060                  * (which forces the hardware to complete any writeback)
1061                  * when making the HPTE read-only.
1062                  * If vcpus are running then this call is racy anyway
1063                  * since the page could get dirtied subsequently, so we
1064                  * expect there to be a further call which would pick up
1065                  * any delayed C bit writeback.
1066                  * Otherwise we need to do the tlbie even if C==0 in
1067                  * order to pick up any delayed writeback of C.
1068                  */
1069                 hptep1 = be64_to_cpu(hptep[1]);
1070                 if (!(hptep1 & HPTE_R_C) &&
1071                     (!hpte_is_writable(hptep1) || vcpus_running(kvm)))
1072                         continue;
1073
1074                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
1075                         /* unlock rmap before spinning on the HPTE lock */
1076                         unlock_rmap(rmapp);
1077                         while (hptep[0] & cpu_to_be64(HPTE_V_HVLOCK))
1078                                 cpu_relax();
1079                         goto retry;
1080                 }
1081
1082                 /* Now check and modify the HPTE */
1083                 if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID))) {
1084                         __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
1085                         continue;
1086                 }
1087
1088                 /* need to make it temporarily absent so C is stable */
1089                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
1090                 kvmppc_invalidate_hpte(kvm, hptep, i);
1091                 v = be64_to_cpu(hptep[0]);
1092                 r = be64_to_cpu(hptep[1]);
1093                 if (r & HPTE_R_C) {
1094                         hptep[1] = cpu_to_be64(r & ~HPTE_R_C);
1095                         if (!(rev[i].guest_rpte & HPTE_R_C)) {
1096                                 rev[i].guest_rpte |= HPTE_R_C;
1097                                 note_hpte_modification(kvm, &rev[i]);
1098                         }
1099                         n = kvmppc_actual_pgsz(v, r);
1100                         n = (n + PAGE_SIZE - 1) >> PAGE_SHIFT;
1101                         if (n > npages_dirty)
1102                                 npages_dirty = n;
1103                         eieio();
1104                 }
1105                 v &= ~HPTE_V_ABSENT;
1106                 v |= HPTE_V_VALID;
1107                 __unlock_hpte(hptep, v);
1108         } while ((i = j) != head);
1109
1110         unlock_rmap(rmapp);
1111         return npages_dirty;
1112 }
1113
1114 void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa,
1115                               struct kvm_memory_slot *memslot,
1116                               unsigned long *map)
1117 {
1118         unsigned long gfn;
1119
1120         if (!vpa->dirty || !vpa->pinned_addr)
1121                 return;
1122         gfn = vpa->gpa >> PAGE_SHIFT;
1123         if (gfn < memslot->base_gfn ||
1124             gfn >= memslot->base_gfn + memslot->npages)
1125                 return;
1126
1127         vpa->dirty = false;
1128         if (map)
1129                 __set_bit_le(gfn - memslot->base_gfn, map);
1130 }
1131
1132 long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm,
1133                         struct kvm_memory_slot *memslot, unsigned long *map)
1134 {
1135         unsigned long i;
1136         unsigned long *rmapp;
1137
1138         preempt_disable();
1139         rmapp = memslot->arch.rmap;
1140         for (i = 0; i < memslot->npages; ++i) {
1141                 int npages = kvm_test_clear_dirty_npages(kvm, rmapp);
1142                 /*
1143                  * Note that if npages > 0 then i must be a multiple of npages,
1144                  * since we always put huge-page HPTEs in the rmap chain
1145                  * corresponding to their page base address.
1146                  */
1147                 if (npages)
1148                         set_dirty_bits(map, i, npages);
1149                 ++rmapp;
1150         }
1151         preempt_enable();
1152         return 0;
1153 }
1154
1155 void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,
1156                             unsigned long *nb_ret)
1157 {
1158         struct kvm_memory_slot *memslot;
1159         unsigned long gfn = gpa >> PAGE_SHIFT;
1160         struct page *page, *pages[1];
1161         int npages;
1162         unsigned long hva, offset;
1163         int srcu_idx;
1164
1165         srcu_idx = srcu_read_lock(&kvm->srcu);
1166         memslot = gfn_to_memslot(kvm, gfn);
1167         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
1168                 goto err;
1169         hva = gfn_to_hva_memslot(memslot, gfn);
1170         npages = get_user_pages_fast(hva, 1, FOLL_WRITE, pages);
1171         if (npages < 1)
1172                 goto err;
1173         page = pages[0];
1174         srcu_read_unlock(&kvm->srcu, srcu_idx);
1175
1176         offset = gpa & (PAGE_SIZE - 1);
1177         if (nb_ret)
1178                 *nb_ret = PAGE_SIZE - offset;
1179         return page_address(page) + offset;
1180
1181  err:
1182         srcu_read_unlock(&kvm->srcu, srcu_idx);
1183         return NULL;
1184 }
1185
1186 void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
1187                              bool dirty)
1188 {
1189         struct page *page = virt_to_page(va);
1190         struct kvm_memory_slot *memslot;
1191         unsigned long gfn;
1192         int srcu_idx;
1193
1194         put_page(page);
1195
1196         if (!dirty)
1197                 return;
1198
1199         /* We need to mark this page dirty in the memslot dirty_bitmap, if any */
1200         gfn = gpa >> PAGE_SHIFT;
1201         srcu_idx = srcu_read_lock(&kvm->srcu);
1202         memslot = gfn_to_memslot(kvm, gfn);
1203         if (memslot && memslot->dirty_bitmap)
1204                 set_bit_le(gfn - memslot->base_gfn, memslot->dirty_bitmap);
1205         srcu_read_unlock(&kvm->srcu, srcu_idx);
1206 }
1207
1208 /*
1209  * HPT resizing
1210  */
1211 static int resize_hpt_allocate(struct kvm_resize_hpt *resize)
1212 {
1213         int rc;
1214
1215         rc = kvmppc_allocate_hpt(&resize->hpt, resize->order);
1216         if (rc < 0)
1217                 return rc;
1218
1219         resize_hpt_debug(resize, "%s(): HPT @ 0x%lx\n", __func__,
1220                          resize->hpt.virt);
1221
1222         return 0;
1223 }
1224
1225 static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
1226                                             unsigned long idx)
1227 {
1228         struct kvm *kvm = resize->kvm;
1229         struct kvm_hpt_info *old = &kvm->arch.hpt;
1230         struct kvm_hpt_info *new = &resize->hpt;
1231         unsigned long old_hash_mask = (1ULL << (old->order - 7)) - 1;
1232         unsigned long new_hash_mask = (1ULL << (new->order - 7)) - 1;
1233         __be64 *hptep, *new_hptep;
1234         unsigned long vpte, rpte, guest_rpte;
1235         int ret;
1236         struct revmap_entry *rev;
1237         unsigned long apsize, avpn, pteg, hash;
1238         unsigned long new_idx, new_pteg, replace_vpte;
1239         int pshift;
1240
1241         hptep = (__be64 *)(old->virt + (idx << 4));
1242
1243         /* Guest is stopped, so new HPTEs can't be added or faulted
1244          * in, only unmapped or altered by host actions.  So, it's
1245          * safe to check this before we take the HPTE lock */
1246         vpte = be64_to_cpu(hptep[0]);
1247         if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
1248                 return 0; /* nothing to do */
1249
1250         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
1251                 cpu_relax();
1252
1253         vpte = be64_to_cpu(hptep[0]);
1254
1255         ret = 0;
1256         if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
1257                 /* Nothing to do */
1258                 goto out;
1259
1260         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1261                 rpte = be64_to_cpu(hptep[1]);
1262                 vpte = hpte_new_to_old_v(vpte, rpte);
1263         }
1264
1265         /* Unmap */
1266         rev = &old->rev[idx];
1267         guest_rpte = rev->guest_rpte;
1268
1269         ret = -EIO;
1270         apsize = kvmppc_actual_pgsz(vpte, guest_rpte);
1271         if (!apsize)
1272                 goto out;
1273
1274         if (vpte & HPTE_V_VALID) {
1275                 unsigned long gfn = hpte_rpn(guest_rpte, apsize);
1276                 int srcu_idx = srcu_read_lock(&kvm->srcu);
1277                 struct kvm_memory_slot *memslot =
1278                         __gfn_to_memslot(kvm_memslots(kvm), gfn);
1279
1280                 if (memslot) {
1281                         unsigned long *rmapp;
1282                         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
1283
1284                         lock_rmap(rmapp);
1285                         kvmppc_unmap_hpte(kvm, idx, memslot, rmapp, gfn);
1286                         unlock_rmap(rmapp);
1287                 }
1288
1289                 srcu_read_unlock(&kvm->srcu, srcu_idx);
1290         }
1291
1292         /* Reload PTE after unmap */
1293         vpte = be64_to_cpu(hptep[0]);
1294         BUG_ON(vpte & HPTE_V_VALID);
1295         BUG_ON(!(vpte & HPTE_V_ABSENT));
1296
1297         ret = 0;
1298         if (!(vpte & HPTE_V_BOLTED))
1299                 goto out;
1300
1301         rpte = be64_to_cpu(hptep[1]);
1302
1303         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1304                 vpte = hpte_new_to_old_v(vpte, rpte);
1305                 rpte = hpte_new_to_old_r(rpte);
1306         }
1307
1308         pshift = kvmppc_hpte_base_page_shift(vpte, rpte);
1309         avpn = HPTE_V_AVPN_VAL(vpte) & ~(((1ul << pshift) - 1) >> 23);
1310         pteg = idx / HPTES_PER_GROUP;
1311         if (vpte & HPTE_V_SECONDARY)
1312                 pteg = ~pteg;
1313
1314         if (!(vpte & HPTE_V_1TB_SEG)) {
1315                 unsigned long offset, vsid;
1316
1317                 /* We only have 28 - 23 bits of offset in avpn */
1318                 offset = (avpn & 0x1f) << 23;
1319                 vsid = avpn >> 5;
1320                 /* We can find more bits from the pteg value */
1321                 if (pshift < 23)
1322                         offset |= ((vsid ^ pteg) & old_hash_mask) << pshift;
1323
1324                 hash = vsid ^ (offset >> pshift);
1325         } else {
1326                 unsigned long offset, vsid;
1327
1328                 /* We only have 40 - 23 bits of seg_off in avpn */
1329                 offset = (avpn & 0x1ffff) << 23;
1330                 vsid = avpn >> 17;
1331                 if (pshift < 23)
1332                         offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask) << pshift;
1333
1334                 hash = vsid ^ (vsid << 25) ^ (offset >> pshift);
1335         }
1336
1337         new_pteg = hash & new_hash_mask;
1338         if (vpte & HPTE_V_SECONDARY)
1339                 new_pteg = ~hash & new_hash_mask;
1340
1341         new_idx = new_pteg * HPTES_PER_GROUP + (idx % HPTES_PER_GROUP);
1342         new_hptep = (__be64 *)(new->virt + (new_idx << 4));
1343
1344         replace_vpte = be64_to_cpu(new_hptep[0]);
1345         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1346                 unsigned long replace_rpte = be64_to_cpu(new_hptep[1]);
1347                 replace_vpte = hpte_new_to_old_v(replace_vpte, replace_rpte);
1348         }
1349
1350         if (replace_vpte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1351                 BUG_ON(new->order >= old->order);
1352
1353                 if (replace_vpte & HPTE_V_BOLTED) {
1354                         if (vpte & HPTE_V_BOLTED)
1355                                 /* Bolted collision, nothing we can do */
1356                                 ret = -ENOSPC;
1357                         /* Discard the new HPTE */
1358                         goto out;
1359                 }
1360
1361                 /* Discard the previous HPTE */
1362         }
1363
1364         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1365                 rpte = hpte_old_to_new_r(vpte, rpte);
1366                 vpte = hpte_old_to_new_v(vpte);
1367         }
1368
1369         new_hptep[1] = cpu_to_be64(rpte);
1370         new->rev[new_idx].guest_rpte = guest_rpte;
1371         /* No need for a barrier, since new HPT isn't active */
1372         new_hptep[0] = cpu_to_be64(vpte);
1373         unlock_hpte(new_hptep, vpte);
1374
1375 out:
1376         unlock_hpte(hptep, vpte);
1377         return ret;
1378 }
1379
1380 static int resize_hpt_rehash(struct kvm_resize_hpt *resize)
1381 {
1382         struct kvm *kvm = resize->kvm;
1383         unsigned  long i;
1384         int rc;
1385
1386         for (i = 0; i < kvmppc_hpt_npte(&kvm->arch.hpt); i++) {
1387                 rc = resize_hpt_rehash_hpte(resize, i);
1388                 if (rc != 0)
1389                         return rc;
1390         }
1391
1392         return 0;
1393 }
1394
1395 static void resize_hpt_pivot(struct kvm_resize_hpt *resize)
1396 {
1397         struct kvm *kvm = resize->kvm;
1398         struct kvm_hpt_info hpt_tmp;
1399
1400         /* Exchange the pending tables in the resize structure with
1401          * the active tables */
1402
1403         resize_hpt_debug(resize, "resize_hpt_pivot()\n");
1404
1405         spin_lock(&kvm->mmu_lock);
1406         asm volatile("ptesync" : : : "memory");
1407
1408         hpt_tmp = kvm->arch.hpt;
1409         kvmppc_set_hpt(kvm, &resize->hpt);
1410         resize->hpt = hpt_tmp;
1411
1412         spin_unlock(&kvm->mmu_lock);
1413
1414         synchronize_srcu_expedited(&kvm->srcu);
1415
1416         if (cpu_has_feature(CPU_FTR_ARCH_300))
1417                 kvmppc_setup_partition_table(kvm);
1418
1419         resize_hpt_debug(resize, "resize_hpt_pivot() done\n");
1420 }
1421
1422 static void resize_hpt_release(struct kvm *kvm, struct kvm_resize_hpt *resize)
1423 {
1424         if (WARN_ON(!mutex_is_locked(&kvm->arch.mmu_setup_lock)))
1425                 return;
1426
1427         if (!resize)
1428                 return;
1429
1430         if (resize->error != -EBUSY) {
1431                 if (resize->hpt.virt)
1432                         kvmppc_free_hpt(&resize->hpt);
1433                 kfree(resize);
1434         }
1435
1436         if (kvm->arch.resize_hpt == resize)
1437                 kvm->arch.resize_hpt = NULL;
1438 }
1439
1440 static void resize_hpt_prepare_work(struct work_struct *work)
1441 {
1442         struct kvm_resize_hpt *resize = container_of(work,
1443                                                      struct kvm_resize_hpt,
1444                                                      work);
1445         struct kvm *kvm = resize->kvm;
1446         int err = 0;
1447
1448         if (WARN_ON(resize->error != -EBUSY))
1449                 return;
1450
1451         mutex_lock(&kvm->arch.mmu_setup_lock);
1452
1453         /* Request is still current? */
1454         if (kvm->arch.resize_hpt == resize) {
1455                 /* We may request large allocations here:
1456                  * do not sleep with kvm->arch.mmu_setup_lock held for a while.
1457                  */
1458                 mutex_unlock(&kvm->arch.mmu_setup_lock);
1459
1460                 resize_hpt_debug(resize, "%s(): order = %d\n", __func__,
1461                                  resize->order);
1462
1463                 err = resize_hpt_allocate(resize);
1464
1465                 /* We have strict assumption about -EBUSY
1466                  * when preparing for HPT resize.
1467                  */
1468                 if (WARN_ON(err == -EBUSY))
1469                         err = -EINPROGRESS;
1470
1471                 mutex_lock(&kvm->arch.mmu_setup_lock);
1472                 /* It is possible that kvm->arch.resize_hpt != resize
1473                  * after we grab kvm->arch.mmu_setup_lock again.
1474                  */
1475         }
1476
1477         resize->error = err;
1478
1479         if (kvm->arch.resize_hpt != resize)
1480                 resize_hpt_release(kvm, resize);
1481
1482         mutex_unlock(&kvm->arch.mmu_setup_lock);
1483 }
1484
1485 int kvm_vm_ioctl_resize_hpt_prepare(struct kvm *kvm,
1486                                     struct kvm_ppc_resize_hpt *rhpt)
1487 {
1488         unsigned long flags = rhpt->flags;
1489         unsigned long shift = rhpt->shift;
1490         struct kvm_resize_hpt *resize;
1491         int ret;
1492
1493         if (flags != 0 || kvm_is_radix(kvm))
1494                 return -EINVAL;
1495
1496         if (shift && ((shift < 18) || (shift > 46)))
1497                 return -EINVAL;
1498
1499         mutex_lock(&kvm->arch.mmu_setup_lock);
1500
1501         resize = kvm->arch.resize_hpt;
1502
1503         if (resize) {
1504                 if (resize->order == shift) {
1505                         /* Suitable resize in progress? */
1506                         ret = resize->error;
1507                         if (ret == -EBUSY)
1508                                 ret = 100; /* estimated time in ms */
1509                         else if (ret)
1510                                 resize_hpt_release(kvm, resize);
1511
1512                         goto out;
1513                 }
1514
1515                 /* not suitable, cancel it */
1516                 resize_hpt_release(kvm, resize);
1517         }
1518
1519         ret = 0;
1520         if (!shift)
1521                 goto out; /* nothing to do */
1522
1523         /* start new resize */
1524
1525         resize = kzalloc(sizeof(*resize), GFP_KERNEL);
1526         if (!resize) {
1527                 ret = -ENOMEM;
1528                 goto out;
1529         }
1530
1531         resize->error = -EBUSY;
1532         resize->order = shift;
1533         resize->kvm = kvm;
1534         INIT_WORK(&resize->work, resize_hpt_prepare_work);
1535         kvm->arch.resize_hpt = resize;
1536
1537         schedule_work(&resize->work);
1538
1539         ret = 100; /* estimated time in ms */
1540
1541 out:
1542         mutex_unlock(&kvm->arch.mmu_setup_lock);
1543         return ret;
1544 }
1545
1546 static void resize_hpt_boot_vcpu(void *opaque)
1547 {
1548         /* Nothing to do, just force a KVM exit */
1549 }
1550
1551 int kvm_vm_ioctl_resize_hpt_commit(struct kvm *kvm,
1552                                    struct kvm_ppc_resize_hpt *rhpt)
1553 {
1554         unsigned long flags = rhpt->flags;
1555         unsigned long shift = rhpt->shift;
1556         struct kvm_resize_hpt *resize;
1557         int ret;
1558
1559         if (flags != 0 || kvm_is_radix(kvm))
1560                 return -EINVAL;
1561
1562         if (shift && ((shift < 18) || (shift > 46)))
1563                 return -EINVAL;
1564
1565         mutex_lock(&kvm->arch.mmu_setup_lock);
1566
1567         resize = kvm->arch.resize_hpt;
1568
1569         /* This shouldn't be possible */
1570         ret = -EIO;
1571         if (WARN_ON(!kvm->arch.mmu_ready))
1572                 goto out_no_hpt;
1573
1574         /* Stop VCPUs from running while we mess with the HPT */
1575         kvm->arch.mmu_ready = 0;
1576         smp_mb();
1577
1578         /* Boot all CPUs out of the guest so they re-read
1579          * mmu_ready */
1580         on_each_cpu(resize_hpt_boot_vcpu, NULL, 1);
1581
1582         ret = -ENXIO;
1583         if (!resize || (resize->order != shift))
1584                 goto out;
1585
1586         ret = resize->error;
1587         if (ret)
1588                 goto out;
1589
1590         ret = resize_hpt_rehash(resize);
1591         if (ret)
1592                 goto out;
1593
1594         resize_hpt_pivot(resize);
1595
1596 out:
1597         /* Let VCPUs run again */
1598         kvm->arch.mmu_ready = 1;
1599         smp_mb();
1600 out_no_hpt:
1601         resize_hpt_release(kvm, resize);
1602         mutex_unlock(&kvm->arch.mmu_setup_lock);
1603         return ret;
1604 }
1605
1606 /*
1607  * Functions for reading and writing the hash table via reads and
1608  * writes on a file descriptor.
1609  *
1610  * Reads return the guest view of the hash table, which has to be
1611  * pieced together from the real hash table and the guest_rpte
1612  * values in the revmap array.
1613  *
1614  * On writes, each HPTE written is considered in turn, and if it
1615  * is valid, it is written to the HPT as if an H_ENTER with the
1616  * exact flag set was done.  When the invalid count is non-zero
1617  * in the header written to the stream, the kernel will make
1618  * sure that that many HPTEs are invalid, and invalidate them
1619  * if not.
1620  */
1621
1622 struct kvm_htab_ctx {
1623         unsigned long   index;
1624         unsigned long   flags;
1625         struct kvm      *kvm;
1626         int             first_pass;
1627 };
1628
1629 #define HPTE_SIZE       (2 * sizeof(unsigned long))
1630
1631 /*
1632  * Returns 1 if this HPT entry has been modified or has pending
1633  * R/C bit changes.
1634  */
1635 static int hpte_dirty(struct revmap_entry *revp, __be64 *hptp)
1636 {
1637         unsigned long rcbits_unset;
1638
1639         if (revp->guest_rpte & HPTE_GR_MODIFIED)
1640                 return 1;
1641
1642         /* Also need to consider changes in reference and changed bits */
1643         rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1644         if ((be64_to_cpu(hptp[0]) & HPTE_V_VALID) &&
1645             (be64_to_cpu(hptp[1]) & rcbits_unset))
1646                 return 1;
1647
1648         return 0;
1649 }
1650
1651 static long record_hpte(unsigned long flags, __be64 *hptp,
1652                         unsigned long *hpte, struct revmap_entry *revp,
1653                         int want_valid, int first_pass)
1654 {
1655         unsigned long v, r, hr;
1656         unsigned long rcbits_unset;
1657         int ok = 1;
1658         int valid, dirty;
1659
1660         /* Unmodified entries are uninteresting except on the first pass */
1661         dirty = hpte_dirty(revp, hptp);
1662         if (!first_pass && !dirty)
1663                 return 0;
1664
1665         valid = 0;
1666         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1667                 valid = 1;
1668                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) &&
1669                     !(be64_to_cpu(hptp[0]) & HPTE_V_BOLTED))
1670                         valid = 0;
1671         }
1672         if (valid != want_valid)
1673                 return 0;
1674
1675         v = r = 0;
1676         if (valid || dirty) {
1677                 /* lock the HPTE so it's stable and read it */
1678                 preempt_disable();
1679                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
1680                         cpu_relax();
1681                 v = be64_to_cpu(hptp[0]);
1682                 hr = be64_to_cpu(hptp[1]);
1683                 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1684                         v = hpte_new_to_old_v(v, hr);
1685                         hr = hpte_new_to_old_r(hr);
1686                 }
1687
1688                 /* re-evaluate valid and dirty from synchronized HPTE value */
1689                 valid = !!(v & HPTE_V_VALID);
1690                 dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED);
1691
1692                 /* Harvest R and C into guest view if necessary */
1693                 rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1694                 if (valid && (rcbits_unset & hr)) {
1695                         revp->guest_rpte |= (hr &
1696                                 (HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED;
1697                         dirty = 1;
1698                 }
1699
1700                 if (v & HPTE_V_ABSENT) {
1701                         v &= ~HPTE_V_ABSENT;
1702                         v |= HPTE_V_VALID;
1703                         valid = 1;
1704                 }
1705                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED))
1706                         valid = 0;
1707
1708                 r = revp->guest_rpte;
1709                 /* only clear modified if this is the right sort of entry */
1710                 if (valid == want_valid && dirty) {
1711                         r &= ~HPTE_GR_MODIFIED;
1712                         revp->guest_rpte = r;
1713                 }
1714                 unlock_hpte(hptp, be64_to_cpu(hptp[0]));
1715                 preempt_enable();
1716                 if (!(valid == want_valid && (first_pass || dirty)))
1717                         ok = 0;
1718         }
1719         hpte[0] = cpu_to_be64(v);
1720         hpte[1] = cpu_to_be64(r);
1721         return ok;
1722 }
1723
1724 static ssize_t kvm_htab_read(struct file *file, char __user *buf,
1725                              size_t count, loff_t *ppos)
1726 {
1727         struct kvm_htab_ctx *ctx = file->private_data;
1728         struct kvm *kvm = ctx->kvm;
1729         struct kvm_get_htab_header hdr;
1730         __be64 *hptp;
1731         struct revmap_entry *revp;
1732         unsigned long i, nb, nw;
1733         unsigned long __user *lbuf;
1734         struct kvm_get_htab_header __user *hptr;
1735         unsigned long flags;
1736         int first_pass;
1737         unsigned long hpte[2];
1738
1739         if (!access_ok(buf, count))
1740                 return -EFAULT;
1741         if (kvm_is_radix(kvm))
1742                 return 0;
1743
1744         first_pass = ctx->first_pass;
1745         flags = ctx->flags;
1746
1747         i = ctx->index;
1748         hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
1749         revp = kvm->arch.hpt.rev + i;
1750         lbuf = (unsigned long __user *)buf;
1751
1752         nb = 0;
1753         while (nb + sizeof(hdr) + HPTE_SIZE < count) {
1754                 /* Initialize header */
1755                 hptr = (struct kvm_get_htab_header __user *)buf;
1756                 hdr.n_valid = 0;
1757                 hdr.n_invalid = 0;
1758                 nw = nb;
1759                 nb += sizeof(hdr);
1760                 lbuf = (unsigned long __user *)(buf + sizeof(hdr));
1761
1762                 /* Skip uninteresting entries, i.e. clean on not-first pass */
1763                 if (!first_pass) {
1764                         while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1765                                !hpte_dirty(revp, hptp)) {
1766                                 ++i;
1767                                 hptp += 2;
1768                                 ++revp;
1769                         }
1770                 }
1771                 hdr.index = i;
1772
1773                 /* Grab a series of valid entries */
1774                 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1775                        hdr.n_valid < 0xffff &&
1776                        nb + HPTE_SIZE < count &&
1777                        record_hpte(flags, hptp, hpte, revp, 1, first_pass)) {
1778                         /* valid entry, write it out */
1779                         ++hdr.n_valid;
1780                         if (__put_user(hpte[0], lbuf) ||
1781                             __put_user(hpte[1], lbuf + 1))
1782                                 return -EFAULT;
1783                         nb += HPTE_SIZE;
1784                         lbuf += 2;
1785                         ++i;
1786                         hptp += 2;
1787                         ++revp;
1788                 }
1789                 /* Now skip invalid entries while we can */
1790                 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1791                        hdr.n_invalid < 0xffff &&
1792                        record_hpte(flags, hptp, hpte, revp, 0, first_pass)) {
1793                         /* found an invalid entry */
1794                         ++hdr.n_invalid;
1795                         ++i;
1796                         hptp += 2;
1797                         ++revp;
1798                 }
1799
1800                 if (hdr.n_valid || hdr.n_invalid) {
1801                         /* write back the header */
1802                         if (__copy_to_user(hptr, &hdr, sizeof(hdr)))
1803                                 return -EFAULT;
1804                         nw = nb;
1805                         buf = (char __user *)lbuf;
1806                 } else {
1807                         nb = nw;
1808                 }
1809
1810                 /* Check if we've wrapped around the hash table */
1811                 if (i >= kvmppc_hpt_npte(&kvm->arch.hpt)) {
1812                         i = 0;
1813                         ctx->first_pass = 0;
1814                         break;
1815                 }
1816         }
1817
1818         ctx->index = i;
1819
1820         return nb;
1821 }
1822
1823 static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1824                               size_t count, loff_t *ppos)
1825 {
1826         struct kvm_htab_ctx *ctx = file->private_data;
1827         struct kvm *kvm = ctx->kvm;
1828         struct kvm_get_htab_header hdr;
1829         unsigned long i, j;
1830         unsigned long v, r;
1831         unsigned long __user *lbuf;
1832         __be64 *hptp;
1833         unsigned long tmp[2];
1834         ssize_t nb;
1835         long int err, ret;
1836         int mmu_ready;
1837         int pshift;
1838
1839         if (!access_ok(buf, count))
1840                 return -EFAULT;
1841         if (kvm_is_radix(kvm))
1842                 return -EINVAL;
1843
1844         /* lock out vcpus from running while we're doing this */
1845         mutex_lock(&kvm->arch.mmu_setup_lock);
1846         mmu_ready = kvm->arch.mmu_ready;
1847         if (mmu_ready) {
1848                 kvm->arch.mmu_ready = 0;        /* temporarily */
1849                 /* order mmu_ready vs. vcpus_running */
1850                 smp_mb();
1851                 if (atomic_read(&kvm->arch.vcpus_running)) {
1852                         kvm->arch.mmu_ready = 1;
1853                         mutex_unlock(&kvm->arch.mmu_setup_lock);
1854                         return -EBUSY;
1855                 }
1856         }
1857
1858         err = 0;
1859         for (nb = 0; nb + sizeof(hdr) <= count; ) {
1860                 err = -EFAULT;
1861                 if (__copy_from_user(&hdr, buf, sizeof(hdr)))
1862                         break;
1863
1864                 err = 0;
1865                 if (nb + hdr.n_valid * HPTE_SIZE > count)
1866                         break;
1867
1868                 nb += sizeof(hdr);
1869                 buf += sizeof(hdr);
1870
1871                 err = -EINVAL;
1872                 i = hdr.index;
1873                 if (i >= kvmppc_hpt_npte(&kvm->arch.hpt) ||
1874                     i + hdr.n_valid + hdr.n_invalid > kvmppc_hpt_npte(&kvm->arch.hpt))
1875                         break;
1876
1877                 hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
1878                 lbuf = (unsigned long __user *)buf;
1879                 for (j = 0; j < hdr.n_valid; ++j) {
1880                         __be64 hpte_v;
1881                         __be64 hpte_r;
1882
1883                         err = -EFAULT;
1884                         if (__get_user(hpte_v, lbuf) ||
1885                             __get_user(hpte_r, lbuf + 1))
1886                                 goto out;
1887                         v = be64_to_cpu(hpte_v);
1888                         r = be64_to_cpu(hpte_r);
1889                         err = -EINVAL;
1890                         if (!(v & HPTE_V_VALID))
1891                                 goto out;
1892                         pshift = kvmppc_hpte_base_page_shift(v, r);
1893                         if (pshift <= 0)
1894                                 goto out;
1895                         lbuf += 2;
1896                         nb += HPTE_SIZE;
1897
1898                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1899                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1900                         err = -EIO;
1901                         ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
1902                                                          tmp);
1903                         if (ret != H_SUCCESS) {
1904                                 pr_err("%s ret %ld i=%ld v=%lx r=%lx\n", __func__, ret, i, v, r);
1905                                 goto out;
1906                         }
1907                         if (!mmu_ready && is_vrma_hpte(v)) {
1908                                 unsigned long senc, lpcr;
1909
1910                                 senc = slb_pgsize_encoding(1ul << pshift);
1911                                 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
1912                                         (VRMA_VSID << SLB_VSID_SHIFT_1T);
1913                                 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
1914                                         lpcr = senc << (LPCR_VRMASD_SH - 4);
1915                                         kvmppc_update_lpcr(kvm, lpcr,
1916                                                            LPCR_VRMASD);
1917                                 } else {
1918                                         kvmppc_setup_partition_table(kvm);
1919                                 }
1920                                 mmu_ready = 1;
1921                         }
1922                         ++i;
1923                         hptp += 2;
1924                 }
1925
1926                 for (j = 0; j < hdr.n_invalid; ++j) {
1927                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1928                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1929                         ++i;
1930                         hptp += 2;
1931                 }
1932                 err = 0;
1933         }
1934
1935  out:
1936         /* Order HPTE updates vs. mmu_ready */
1937         smp_wmb();
1938         kvm->arch.mmu_ready = mmu_ready;
1939         mutex_unlock(&kvm->arch.mmu_setup_lock);
1940
1941         if (err)
1942                 return err;
1943         return nb;
1944 }
1945
1946 static int kvm_htab_release(struct inode *inode, struct file *filp)
1947 {
1948         struct kvm_htab_ctx *ctx = filp->private_data;
1949
1950         filp->private_data = NULL;
1951         if (!(ctx->flags & KVM_GET_HTAB_WRITE))
1952                 atomic_dec(&ctx->kvm->arch.hpte_mod_interest);
1953         kvm_put_kvm(ctx->kvm);
1954         kfree(ctx);
1955         return 0;
1956 }
1957
1958 static const struct file_operations kvm_htab_fops = {
1959         .read           = kvm_htab_read,
1960         .write          = kvm_htab_write,
1961         .llseek         = default_llseek,
1962         .release        = kvm_htab_release,
1963 };
1964
1965 int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)
1966 {
1967         int ret;
1968         struct kvm_htab_ctx *ctx;
1969         int rwflag;
1970
1971         /* reject flags we don't recognize */
1972         if (ghf->flags & ~(KVM_GET_HTAB_BOLTED_ONLY | KVM_GET_HTAB_WRITE))
1973                 return -EINVAL;
1974         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1975         if (!ctx)
1976                 return -ENOMEM;
1977         kvm_get_kvm(kvm);
1978         ctx->kvm = kvm;
1979         ctx->index = ghf->start_index;
1980         ctx->flags = ghf->flags;
1981         ctx->first_pass = 1;
1982
1983         rwflag = (ghf->flags & KVM_GET_HTAB_WRITE) ? O_WRONLY : O_RDONLY;
1984         ret = anon_inode_getfd("kvm-htab", &kvm_htab_fops, ctx, rwflag | O_CLOEXEC);
1985         if (ret < 0) {
1986                 kfree(ctx);
1987                 kvm_put_kvm_no_destroy(kvm);
1988                 return ret;
1989         }
1990
1991         if (rwflag == O_RDONLY) {
1992                 mutex_lock(&kvm->slots_lock);
1993                 atomic_inc(&kvm->arch.hpte_mod_interest);
1994                 /* make sure kvmppc_do_h_enter etc. see the increment */
1995                 synchronize_srcu_expedited(&kvm->srcu);
1996                 mutex_unlock(&kvm->slots_lock);
1997         }
1998
1999         return ret;
2000 }
2001
2002 struct debugfs_htab_state {
2003         struct kvm      *kvm;
2004         struct mutex    mutex;
2005         unsigned long   hpt_index;
2006         int             chars_left;
2007         int             buf_index;
2008         char            buf[64];
2009 };
2010
2011 static int debugfs_htab_open(struct inode *inode, struct file *file)
2012 {
2013         struct kvm *kvm = inode->i_private;
2014         struct debugfs_htab_state *p;
2015
2016         p = kzalloc(sizeof(*p), GFP_KERNEL);
2017         if (!p)
2018                 return -ENOMEM;
2019
2020         kvm_get_kvm(kvm);
2021         p->kvm = kvm;
2022         mutex_init(&p->mutex);
2023         file->private_data = p;
2024
2025         return nonseekable_open(inode, file);
2026 }
2027
2028 static int debugfs_htab_release(struct inode *inode, struct file *file)
2029 {
2030         struct debugfs_htab_state *p = file->private_data;
2031
2032         kvm_put_kvm(p->kvm);
2033         kfree(p);
2034         return 0;
2035 }
2036
2037 static ssize_t debugfs_htab_read(struct file *file, char __user *buf,
2038                                  size_t len, loff_t *ppos)
2039 {
2040         struct debugfs_htab_state *p = file->private_data;
2041         ssize_t ret, r;
2042         unsigned long i, n;
2043         unsigned long v, hr, gr;
2044         struct kvm *kvm;
2045         __be64 *hptp;
2046
2047         kvm = p->kvm;
2048         if (kvm_is_radix(kvm))
2049                 return 0;
2050
2051         ret = mutex_lock_interruptible(&p->mutex);
2052         if (ret)
2053                 return ret;
2054
2055         if (p->chars_left) {
2056                 n = p->chars_left;
2057                 if (n > len)
2058                         n = len;
2059                 r = copy_to_user(buf, p->buf + p->buf_index, n);
2060                 n -= r;
2061                 p->chars_left -= n;
2062                 p->buf_index += n;
2063                 buf += n;
2064                 len -= n;
2065                 ret = n;
2066                 if (r) {
2067                         if (!n)
2068                                 ret = -EFAULT;
2069                         goto out;
2070                 }
2071         }
2072
2073         i = p->hpt_index;
2074         hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
2075         for (; len != 0 && i < kvmppc_hpt_npte(&kvm->arch.hpt);
2076              ++i, hptp += 2) {
2077                 if (!(be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)))
2078                         continue;
2079
2080                 /* lock the HPTE so it's stable and read it */
2081                 preempt_disable();
2082                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
2083                         cpu_relax();
2084                 v = be64_to_cpu(hptp[0]) & ~HPTE_V_HVLOCK;
2085                 hr = be64_to_cpu(hptp[1]);
2086                 gr = kvm->arch.hpt.rev[i].guest_rpte;
2087                 unlock_hpte(hptp, v);
2088                 preempt_enable();
2089
2090                 if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT)))
2091                         continue;
2092
2093                 n = scnprintf(p->buf, sizeof(p->buf),
2094                               "%6lx %.16lx %.16lx %.16lx\n",
2095                               i, v, hr, gr);
2096                 p->chars_left = n;
2097                 if (n > len)
2098                         n = len;
2099                 r = copy_to_user(buf, p->buf, n);
2100                 n -= r;
2101                 p->chars_left -= n;
2102                 p->buf_index = n;
2103                 buf += n;
2104                 len -= n;
2105                 ret += n;
2106                 if (r) {
2107                         if (!ret)
2108                                 ret = -EFAULT;
2109                         goto out;
2110                 }
2111         }
2112         p->hpt_index = i;
2113
2114  out:
2115         mutex_unlock(&p->mutex);
2116         return ret;
2117 }
2118
2119 static ssize_t debugfs_htab_write(struct file *file, const char __user *buf,
2120                            size_t len, loff_t *ppos)
2121 {
2122         return -EACCES;
2123 }
2124
2125 static const struct file_operations debugfs_htab_fops = {
2126         .owner   = THIS_MODULE,
2127         .open    = debugfs_htab_open,
2128         .release = debugfs_htab_release,
2129         .read    = debugfs_htab_read,
2130         .write   = debugfs_htab_write,
2131         .llseek  = generic_file_llseek,
2132 };
2133
2134 void kvmppc_mmu_debugfs_init(struct kvm *kvm)
2135 {
2136         debugfs_create_file("htab", 0400, kvm->debugfs_dentry, kvm,
2137                             &debugfs_htab_fops);
2138 }
2139
2140 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu)
2141 {
2142         struct kvmppc_mmu *mmu = &vcpu->arch.mmu;
2143
2144         vcpu->arch.slb_nr = 32;         /* POWER7/POWER8 */
2145
2146         mmu->xlate = kvmppc_mmu_book3s_64_hv_xlate;
2147
2148         vcpu->arch.hflags |= BOOK3S_HFLAG_SLB;
2149 }