arch/powerpc/kvm/book3s_64_mmu_hv.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2, as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14  *
  15  * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  16  */
  17
  18 #include <linux/types.h>
  19 #include <linux/string.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/highmem.h>
  23 #include <linux/gfp.h>
  24 #include <linux/slab.h>
  25 #include <linux/hugetlb.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/srcu.h>
  28 #include <linux/anon_inodes.h>
  29 #include <linux/file.h>
  30 #include <linux/debugfs.h>
  31
  32 #include <asm/tlbflush.h>
  33 #include <asm/kvm_ppc.h>
  34 #include <asm/kvm_book3s.h>
  35 #include <asm/book3s/64/mmu-hash.h>
  36 #include <asm/hvcall.h>
  37 #include <asm/synch.h>
  38 #include <asm/ppc-opcode.h>
  39 #include <asm/cputable.h>
  40 #include <asm/pte-walk.h>
  41
  42 #include "trace_hv.h"
  43
  44 //#define DEBUG_RESIZE_HPT      1
  45
  46 #ifdef DEBUG_RESIZE_HPT
  47 #define resize_hpt_debug(resize, ...)                           \
  48         do {                                                    \
  49                 printk(KERN_DEBUG "RESIZE HPT %p: ", resize);   \
  50                 printk(__VA_ARGS__);                            \
  51         } while (0)
  52 #else
  53 #define resize_hpt_debug(resize, ...)                           \
  54         do { } while (0)
  55 #endif
  56
  57 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
  58                                 long pte_index, unsigned long pteh,
  59                                 unsigned long ptel, unsigned long *pte_idx_ret);
  60
  61 struct kvm_resize_hpt {
  62         /* These fields read-only after init */
  63         struct kvm *kvm;
  64         struct work_struct work;
  65         u32 order;
  66
  67         /* These fields protected by kvm->lock */
  68         int error;
  69         bool prepare_done;
  70
  71         /* Private to the work thread, until prepare_done is true,
  72          * then protected by kvm->resize_hpt_sem */
  73         struct kvm_hpt_info hpt;
  74 };
  75
  76 int kvmppc_allocate_hpt(struct kvm_hpt_info *info, u32 order)
  77 {
  78         unsigned long hpt = 0;
  79         int cma = 0;
  80         struct page *page = NULL;
  81         struct revmap_entry *rev;
  82         unsigned long npte;
  83
  84         if ((order < PPC_MIN_HPT_ORDER) || (order > PPC_MAX_HPT_ORDER))
  85                 return -EINVAL;
  86
  87         page = kvm_alloc_hpt_cma(1ul << (order - PAGE_SHIFT));
  88         if (page) {
  89                 hpt = (unsigned long)pfn_to_kaddr(page_to_pfn(page));
  90                 memset((void *)hpt, 0, (1ul << order));
  91                 cma = 1;
  92         }
  93
  94         if (!hpt)
  95                 hpt = __get_free_pages(GFP_KERNEL|__GFP_ZERO|__GFP_RETRY_MAYFAIL
  96                                        |__GFP_NOWARN, order - PAGE_SHIFT);
  97
  98         if (!hpt)
  99                 return -ENOMEM;
 100
 101         /* HPTEs are 2**4 bytes long */
 102         npte = 1ul << (order - 4);
 103
 104         /* Allocate reverse map array */
 105         rev = vmalloc(sizeof(struct revmap_entry) * npte);
 106         if (!rev) {
 107                 if (cma)
 108                         kvm_free_hpt_cma(page, 1 << (order - PAGE_SHIFT));
 109                 else
 110                         free_pages(hpt, order - PAGE_SHIFT);
 111                 return -ENOMEM;
 112         }
 113
 114         info->order = order;
 115         info->virt = hpt;
 116         info->cma = cma;
 117         info->rev = rev;
 118
 119         return 0;
 120 }
 121
 122 void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info)
 123 {
 124         atomic64_set(&kvm->arch.mmio_update, 0);
 125         kvm->arch.hpt = *info;
 126         kvm->arch.sdr1 = __pa(info->virt) | (info->order - 18);
 127
 128         pr_debug("KVM guest htab at %lx (order %ld), LPID %x\n",
 129                  info->virt, (long)info->order, kvm->arch.lpid);
 130 }
 131
 132 long kvmppc_alloc_reset_hpt(struct kvm *kvm, int order)
 133 {
 134         long err = -EBUSY;
 135         struct kvm_hpt_info info;
 136
 137         mutex_lock(&kvm->lock);
 138         if (kvm->arch.mmu_ready) {
 139                 kvm->arch.mmu_ready = 0;
 140                 /* order mmu_ready vs. vcpus_running */
 141                 smp_mb();
 142                 if (atomic_read(&kvm->arch.vcpus_running)) {
 143                         kvm->arch.mmu_ready = 1;
 144                         goto out;
 145                 }
 146         }
 147         if (kvm_is_radix(kvm)) {
 148                 err = kvmppc_switch_mmu_to_hpt(kvm);
 149                 if (err)
 150                         goto out;
 151         }
 152
 153         if (kvm->arch.hpt.order == order) {
 154                 /* We already have a suitable HPT */
 155
 156                 /* Set the entire HPT to 0, i.e. invalid HPTEs */
 157                 memset((void *)kvm->arch.hpt.virt, 0, 1ul << order);
 158                 /*
 159                  * Reset all the reverse-mapping chains for all memslots
 160                  */
 161                 kvmppc_rmap_reset(kvm);
 162                 /* Ensure that each vcpu will flush its TLB on next entry. */
 163                 cpumask_setall(&kvm->arch.need_tlb_flush);
 164                 err = 0;
 165                 goto out;
 166         }
 167
 168         if (kvm->arch.hpt.virt) {
 169                 kvmppc_free_hpt(&kvm->arch.hpt);
 170                 kvmppc_rmap_reset(kvm);
 171         }
 172
 173         err = kvmppc_allocate_hpt(&info, order);
 174         if (err < 0)
 175                 goto out;
 176         kvmppc_set_hpt(kvm, &info);
 177
 178 out:
 179         mutex_unlock(&kvm->lock);
 180         return err;
 181 }
 182
 183 void kvmppc_free_hpt(struct kvm_hpt_info *info)
 184 {
 185         vfree(info->rev);
 186         info->rev = NULL;
 187         if (info->cma)
 188                 kvm_free_hpt_cma(virt_to_page(info->virt),
 189                                  1 << (info->order - PAGE_SHIFT));
 190         else if (info->virt)
 191                 free_pages(info->virt, info->order - PAGE_SHIFT);
 192         info->virt = 0;
 193         info->order = 0;
 194 }
 195
 196 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
 197 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize)
 198 {
 199         return (pgsize > 0x1000) ? HPTE_V_LARGE : 0;
 200 }
 201
 202 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
 203 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize)
 204 {
 205         return (pgsize == 0x10000) ? 0x1000 : 0;
 206 }
 207
 208 void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 209                      unsigned long porder)
 210 {
 211         unsigned long i;
 212         unsigned long npages;
 213         unsigned long hp_v, hp_r;
 214         unsigned long addr, hash;
 215         unsigned long psize;
 216         unsigned long hp0, hp1;
 217         unsigned long idx_ret;
 218         long ret;
 219         struct kvm *kvm = vcpu->kvm;
 220
 221         psize = 1ul << porder;
 222         npages = memslot->npages >> (porder - PAGE_SHIFT);
 223
 224         /* VRMA can't be > 1TB */
 225         if (npages > 1ul << (40 - porder))
 226                 npages = 1ul << (40 - porder);
 227         /* Can't use more than 1 HPTE per HPTEG */
 228         if (npages > kvmppc_hpt_mask(&kvm->arch.hpt) + 1)
 229                 npages = kvmppc_hpt_mask(&kvm->arch.hpt) + 1;
 230
 231         hp0 = HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)) |
 232                 HPTE_V_BOLTED | hpte0_pgsize_encoding(psize);
 233         hp1 = hpte1_pgsize_encoding(psize) |
 234                 HPTE_R_R | HPTE_R_C | HPTE_R_M | PP_RWXX;
 235
 236         for (i = 0; i < npages; ++i) {
 237                 addr = i << porder;
 238                 /* can't use hpt_hash since va > 64 bits */
 239                 hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25)))
 240                         & kvmppc_hpt_mask(&kvm->arch.hpt);
 241                 /*
 242                  * We assume that the hash table is empty and no
 243                  * vcpus are using it at this stage.  Since we create
 244                  * at most one HPTE per HPTEG, we just assume entry 7
 245                  * is available and use it.
 246                  */
 247                 hash = (hash << 3) + 7;
 248                 hp_v = hp0 | ((addr >> 16) & ~0x7fUL);
 249                 hp_r = hp1 | addr;
 250                 ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,
 251                                                  &idx_ret);
 252                 if (ret != H_SUCCESS) {
 253                         pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
 254                                addr, ret);
 255                         break;
 256                 }
 257         }
 258 }
 259
 260 int kvmppc_mmu_hv_init(void)
 261 {
 262         unsigned long host_lpid, rsvd_lpid;
 263
 264         if (!cpu_has_feature(CPU_FTR_HVMODE))
 265                 return -EINVAL;
 266
 267         /* POWER7 has 10-bit LPIDs (12-bit in POWER8) */
 268         host_lpid = mfspr(SPRN_LPID);
 269         rsvd_lpid = LPID_RSVD;
 270
 271         kvmppc_init_lpid(rsvd_lpid + 1);
 272
 273         kvmppc_claim_lpid(host_lpid);
 274         /* rsvd_lpid is reserved for use in partition switching */
 275         kvmppc_claim_lpid(rsvd_lpid);
 276
 277         return 0;
 278 }
 279
 280 static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu)
 281 {
 282         unsigned long msr = vcpu->arch.intr_msr;
 283
 284         /* If transactional, change to suspend mode on IRQ delivery */
 285         if (MSR_TM_TRANSACTIONAL(vcpu->arch.shregs.msr))
 286                 msr |= MSR_TS_S;
 287         else
 288                 msr |= vcpu->arch.shregs.msr & MSR_TS_MASK;
 289         kvmppc_set_msr(vcpu, msr);
 290 }
 291
 292 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
 293                                 long pte_index, unsigned long pteh,
 294                                 unsigned long ptel, unsigned long *pte_idx_ret)
 295 {
 296         long ret;
 297
 298         /* Protect linux PTE lookup from page table destruction */
 299         rcu_read_lock_sched();  /* this disables preemption too */
 300         ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,
 301                                 current->mm->pgd, false, pte_idx_ret);
 302         rcu_read_unlock_sched();
 303         if (ret == H_TOO_HARD) {
 304                 /* this can't happen */
 305                 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
 306                 ret = H_RESOURCE;       /* or something */
 307         }
 308         return ret;
 309
 310 }
 311
 312 static struct kvmppc_slb *kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu *vcpu,
 313                                                          gva_t eaddr)
 314 {
 315         u64 mask;
 316         int i;
 317
 318         for (i = 0; i < vcpu->arch.slb_nr; i++) {
 319                 if (!(vcpu->arch.slb[i].orige & SLB_ESID_V))
 320                         continue;
 321
 322                 if (vcpu->arch.slb[i].origv & SLB_VSID_B_1T)
 323                         mask = ESID_MASK_1T;
 324                 else
 325                         mask = ESID_MASK;
 326
 327                 if (((vcpu->arch.slb[i].orige ^ eaddr) & mask) == 0)
 328                         return &vcpu->arch.slb[i];
 329         }
 330         return NULL;
 331 }
 332
 333 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v, unsigned long r,
 334                         unsigned long ea)
 335 {
 336         unsigned long ra_mask;
 337
 338         ra_mask = kvmppc_actual_pgsz(v, r) - 1;
 339         return (r & HPTE_R_RPN & ~ra_mask) | (ea & ra_mask);
 340 }
 341
 342 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 343                         struct kvmppc_pte *gpte, bool data, bool iswrite)
 344 {
 345         struct kvm *kvm = vcpu->kvm;
 346         struct kvmppc_slb *slbe;
 347         unsigned long slb_v;
 348         unsigned long pp, key;
 349         unsigned long v, orig_v, gr;
 350         __be64 *hptep;
 351         int index;
 352         int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR);
 353
 354         if (kvm_is_radix(vcpu->kvm))
 355                 return kvmppc_mmu_radix_xlate(vcpu, eaddr, gpte, data, iswrite);
 356
 357         /* Get SLB entry */
 358         if (virtmode) {
 359                 slbe = kvmppc_mmu_book3s_hv_find_slbe(vcpu, eaddr);
 360                 if (!slbe)
 361                         return -EINVAL;
 362                 slb_v = slbe->origv;
 363         } else {
 364                 /* real mode access */
 365                 slb_v = vcpu->kvm->arch.vrma_slb_v;
 366         }
 367
 368         preempt_disable();
 369         /* Find the HPTE in the hash table */
 370         index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,
 371                                          HPTE_V_VALID | HPTE_V_ABSENT);
 372         if (index < 0) {
 373                 preempt_enable();
 374                 return -ENOENT;
 375         }
 376         hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
 377         v = orig_v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 378         if (cpu_has_feature(CPU_FTR_ARCH_300))
 379                 v = hpte_new_to_old_v(v, be64_to_cpu(hptep[1]));
 380         gr = kvm->arch.hpt.rev[index].guest_rpte;
 381
 382         unlock_hpte(hptep, orig_v);
 383         preempt_enable();
 384
 385         gpte->eaddr = eaddr;
 386         gpte->vpage = ((v & HPTE_V_AVPN) << 4) | ((eaddr >> 12) & 0xfff);
 387
 388         /* Get PP bits and key for permission check */
 389         pp = gr & (HPTE_R_PP0 | HPTE_R_PP);
 390         key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS;
 391         key &= slb_v;
 392
 393         /* Calculate permissions */
 394         gpte->may_read = hpte_read_permission(pp, key);
 395         gpte->may_write = hpte_write_permission(pp, key);
 396         gpte->may_execute = gpte->may_read && !(gr & (HPTE_R_N | HPTE_R_G));
 397
 398         /* Storage key permission check for POWER7 */
 399         if (data && virtmode) {
 400                 int amrfield = hpte_get_skey_perm(gr, vcpu->arch.amr);
 401                 if (amrfield & 1)
 402                         gpte->may_read = 0;
 403                 if (amrfield & 2)
 404                         gpte->may_write = 0;
 405         }
 406
 407         /* Get the guest physical address */
 408         gpte->raddr = kvmppc_mmu_get_real_addr(v, gr, eaddr);
 409         return 0;
 410 }
 411
 412 /*
 413  * Quick test for whether an instruction is a load or a store.
 414  * If the instruction is a load or a store, then this will indicate
 415  * which it is, at least on server processors.  (Embedded processors
 416  * have some external PID instructions that don't follow the rule
 417  * embodied here.)  If the instruction isn't a load or store, then
 418  * this doesn't return anything useful.
 419  */
 420 static int instruction_is_store(unsigned int instr)
 421 {
 422         unsigned int mask;
 423
 424         mask = 0x10000000;
 425         if ((instr & 0xfc000000) == 0x7c000000)
 426                 mask = 0x100;           /* major opcode 31 */
 427         return (instr & mask) != 0;
 428 }
 429
 430 int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu,
 431                            unsigned long gpa, gva_t ea, int is_store)
 432 {
 433         u32 last_inst;
 434
 435         /*
 436          * If we fail, we just return to the guest and try executing it again.
 437          */
 438         if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
 439                 EMULATE_DONE)
 440                 return RESUME_GUEST;
 441
 442         /*
 443          * WARNING: We do not know for sure whether the instruction we just
 444          * read from memory is the same that caused the fault in the first
 445          * place.  If the instruction we read is neither an load or a store,
 446          * then it can't access memory, so we don't need to worry about
 447          * enforcing access permissions.  So, assuming it is a load or
 448          * store, we just check that its direction (load or store) is
 449          * consistent with the original fault, since that's what we
 450          * checked the access permissions against.  If there is a mismatch
 451          * we just return and retry the instruction.
 452          */
 453
 454         if (instruction_is_store(last_inst) != !!is_store)
 455                 return RESUME_GUEST;
 456
 457         /*
 458          * Emulated accesses are emulated by looking at the hash for
 459          * translation once, then performing the access later. The
 460          * translation could be invalidated in the meantime in which
 461          * point performing the subsequent memory access on the old
 462          * physical address could possibly be a security hole for the
 463          * guest (but not the host).
 464          *
 465          * This is less of an issue for MMIO stores since they aren't
 466          * globally visible. It could be an issue for MMIO loads to
 467          * a certain extent but we'll ignore it for now.
 468          */
 469
 470         vcpu->arch.paddr_accessed = gpa;
 471         vcpu->arch.vaddr_accessed = ea;
 472         return kvmppc_emulate_mmio(run, vcpu);
 473 }
 474
 475 int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 476                                 unsigned long ea, unsigned long dsisr)
 477 {
 478         struct kvm *kvm = vcpu->kvm;
 479         unsigned long hpte[3], r;
 480         unsigned long hnow_v, hnow_r;
 481         __be64 *hptep;
 482         unsigned long mmu_seq, psize, pte_size;
 483         unsigned long gpa_base, gfn_base;
 484         unsigned long gpa, gfn, hva, pfn;
 485         struct kvm_memory_slot *memslot;
 486         unsigned long *rmap;
 487         struct revmap_entry *rev;
 488         struct page *page, *pages[1];
 489         long index, ret, npages;
 490         bool is_ci;
 491         unsigned int writing, write_ok;
 492         struct vm_area_struct *vma;
 493         unsigned long rcbits;
 494         long mmio_update;
 495
 496         if (kvm_is_radix(kvm))
 497                 return kvmppc_book3s_radix_page_fault(run, vcpu, ea, dsisr);
 498
 499         /*
 500          * Real-mode code has already searched the HPT and found the
 501          * entry we're interested in.  Lock the entry and check that
 502          * it hasn't changed.  If it has, just return and re-execute the
 503          * instruction.
 504          */
 505         if (ea != vcpu->arch.pgfault_addr)
 506                 return RESUME_GUEST;
 507
 508         if (vcpu->arch.pgfault_cache) {
 509                 mmio_update = atomic64_read(&kvm->arch.mmio_update);
 510                 if (mmio_update == vcpu->arch.pgfault_cache->mmio_update) {
 511                         r = vcpu->arch.pgfault_cache->rpte;
 512                         psize = kvmppc_actual_pgsz(vcpu->arch.pgfault_hpte[0],
 513                                                    r);
 514                         gpa_base = r & HPTE_R_RPN & ~(psize - 1);
 515                         gfn_base = gpa_base >> PAGE_SHIFT;
 516                         gpa = gpa_base | (ea & (psize - 1));
 517                         return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
 518                                                 dsisr & DSISR_ISSTORE);
 519                 }
 520         }
 521         index = vcpu->arch.pgfault_index;
 522         hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
 523         rev = &kvm->arch.hpt.rev[index];
 524         preempt_disable();
 525         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 526                 cpu_relax();
 527         hpte[0] = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 528         hpte[1] = be64_to_cpu(hptep[1]);
 529         hpte[2] = r = rev->guest_rpte;
 530         unlock_hpte(hptep, hpte[0]);
 531         preempt_enable();
 532
 533         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 534                 hpte[0] = hpte_new_to_old_v(hpte[0], hpte[1]);
 535                 hpte[1] = hpte_new_to_old_r(hpte[1]);
 536         }
 537         if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
 538             hpte[1] != vcpu->arch.pgfault_hpte[1])
 539                 return RESUME_GUEST;
 540
 541         /* Translate the logical address and get the page */
 542         psize = kvmppc_actual_pgsz(hpte[0], r);
 543         gpa_base = r & HPTE_R_RPN & ~(psize - 1);
 544         gfn_base = gpa_base >> PAGE_SHIFT;
 545         gpa = gpa_base | (ea & (psize - 1));
 546         gfn = gpa >> PAGE_SHIFT;
 547         memslot = gfn_to_memslot(kvm, gfn);
 548
 549         trace_kvm_page_fault_enter(vcpu, hpte, memslot, ea, dsisr);
 550
 551         /* No memslot means it's an emulated MMIO region */
 552         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
 553                 return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
 554                                               dsisr & DSISR_ISSTORE);
 555
 556         /*
 557          * This should never happen, because of the slot_is_aligned()
 558          * check in kvmppc_do_h_enter().
 559          */
 560         if (gfn_base < memslot->base_gfn)
 561                 return -EFAULT;
 562
 563         /* used to check for invalidations in progress */
 564         mmu_seq = kvm->mmu_notifier_seq;
 565         smp_rmb();
 566
 567         ret = -EFAULT;
 568         is_ci = false;
 569         pfn = 0;
 570         page = NULL;
 571         pte_size = PAGE_SIZE;
 572         writing = (dsisr & DSISR_ISSTORE) != 0;
 573         /* If writing != 0, then the HPTE must allow writing, if we get here */
 574         write_ok = writing;
 575         hva = gfn_to_hva_memslot(memslot, gfn);
 576         npages = get_user_pages_fast(hva, 1, writing, pages);
 577         if (npages < 1) {
 578                 /* Check if it's an I/O mapping */
 579                 down_read(&current->mm->mmap_sem);
 580                 vma = find_vma(current->mm, hva);
 581                 if (vma && vma->vm_start <= hva && hva + psize <= vma->vm_end &&
 582                     (vma->vm_flags & VM_PFNMAP)) {
 583                         pfn = vma->vm_pgoff +
 584                                 ((hva - vma->vm_start) >> PAGE_SHIFT);
 585                         pte_size = psize;
 586                         is_ci = pte_ci(__pte((pgprot_val(vma->vm_page_prot))));
 587                         write_ok = vma->vm_flags & VM_WRITE;
 588                 }
 589                 up_read(&current->mm->mmap_sem);
 590                 if (!pfn)
 591                         goto out_put;
 592         } else {
 593                 page = pages[0];
 594                 pfn = page_to_pfn(page);
 595                 if (PageHuge(page)) {
 596                         page = compound_head(page);
 597                         pte_size <<= compound_order(page);
 598                 }
 599                 /* if the guest wants write access, see if that is OK */
 600                 if (!writing && hpte_is_writable(r)) {
 601                         pte_t *ptep, pte;
 602                         unsigned long flags;
 603                         /*
 604                          * We need to protect against page table destruction
 605                          * hugepage split and collapse.
 606                          */
 607                         local_irq_save(flags);
 608                         ptep = find_current_mm_pte(current->mm->pgd,
 609                                                    hva, NULL, NULL);
 610                         if (ptep) {
 611                                 pte = kvmppc_read_update_linux_pte(ptep, 1);
 612                                 if (__pte_write(pte))
 613                                         write_ok = 1;
 614                         }
 615                         local_irq_restore(flags);
 616                 }
 617         }
 618
 619         if (psize > pte_size)
 620                 goto out_put;
 621
 622         /* Check WIMG vs. the actual page we're accessing */
 623         if (!hpte_cache_flags_ok(r, is_ci)) {
 624                 if (is_ci)
 625                         goto out_put;
 626                 /*
 627                  * Allow guest to map emulated device memory as
 628                  * uncacheable, but actually make it cacheable.
 629                  */
 630                 r = (r & ~(HPTE_R_W|HPTE_R_I|HPTE_R_G)) | HPTE_R_M;
 631         }
 632
 633         /*
 634          * Set the HPTE to point to pfn.
 635          * Since the pfn is at PAGE_SIZE granularity, make sure we
 636          * don't mask out lower-order bits if psize < PAGE_SIZE.
 637          */
 638         if (psize < PAGE_SIZE)
 639                 psize = PAGE_SIZE;
 640         r = (r & HPTE_R_KEY_HI) | (r & ~(HPTE_R_PP0 - psize)) |
 641                                         ((pfn << PAGE_SHIFT) & ~(psize - 1));
 642         if (hpte_is_writable(r) && !write_ok)
 643                 r = hpte_make_readonly(r);
 644         ret = RESUME_GUEST;
 645         preempt_disable();
 646         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 647                 cpu_relax();
 648         hnow_v = be64_to_cpu(hptep[0]);
 649         hnow_r = be64_to_cpu(hptep[1]);
 650         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 651                 hnow_v = hpte_new_to_old_v(hnow_v, hnow_r);
 652                 hnow_r = hpte_new_to_old_r(hnow_r);
 653         }
 654
 655         /*
 656          * If the HPT is being resized, don't update the HPTE,
 657          * instead let the guest retry after the resize operation is complete.
 658          * The synchronization for mmu_ready test vs. set is provided
 659          * by the HPTE lock.
 660          */
 661         if (!kvm->arch.mmu_ready)
 662                 goto out_unlock;
 663
 664         if ((hnow_v & ~HPTE_V_HVLOCK) != hpte[0] || hnow_r != hpte[1] ||
 665             rev->guest_rpte != hpte[2])
 666                 /* HPTE has been changed under us; let the guest retry */
 667                 goto out_unlock;
 668         hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
 669
 670         /* Always put the HPTE in the rmap chain for the page base address */
 671         rmap = &memslot->arch.rmap[gfn_base - memslot->base_gfn];
 672         lock_rmap(rmap);
 673
 674         /* Check if we might have been invalidated; let the guest retry if so */
 675         ret = RESUME_GUEST;
 676         if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) {
 677                 unlock_rmap(rmap);
 678                 goto out_unlock;
 679         }
 680
 681         /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
 682         rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
 683         r &= rcbits | ~(HPTE_R_R | HPTE_R_C);
 684
 685         if (be64_to_cpu(hptep[0]) & HPTE_V_VALID) {
 686                 /* HPTE was previously valid, so we need to invalidate it */
 687                 unlock_rmap(rmap);
 688                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 689                 kvmppc_invalidate_hpte(kvm, hptep, index);
 690                 /* don't lose previous R and C bits */
 691                 r |= be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 692         } else {
 693                 kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
 694         }
 695
 696         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 697                 r = hpte_old_to_new_r(hpte[0], r);
 698                 hpte[0] = hpte_old_to_new_v(hpte[0]);
 699         }
 700         hptep[1] = cpu_to_be64(r);
 701         eieio();
 702         __unlock_hpte(hptep, hpte[0]);
 703         asm volatile("ptesync" : : : "memory");
 704         preempt_enable();
 705         if (page && hpte_is_writable(r))
 706                 SetPageDirty(page);
 707
 708  out_put:
 709         trace_kvm_page_fault_exit(vcpu, hpte, ret);
 710
 711         if (page) {
 712                 /*
 713                  * We drop pages[0] here, not page because page might
 714                  * have been set to the head page of a compound, but
 715                  * we have to drop the reference on the correct tail
 716                  * page to match the get inside gup()
 717                  */
 718                 put_page(pages[0]);
 719         }
 720         return ret;
 721
 722  out_unlock:
 723         __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 724         preempt_enable();
 725         goto out_put;
 726 }
 727
 728 void kvmppc_rmap_reset(struct kvm *kvm)
 729 {
 730         struct kvm_memslots *slots;
 731         struct kvm_memory_slot *memslot;
 732         int srcu_idx;
 733
 734         srcu_idx = srcu_read_lock(&kvm->srcu);
 735         slots = kvm_memslots(kvm);
 736         kvm_for_each_memslot(memslot, slots) {
 737                 /*
 738                  * This assumes it is acceptable to lose reference and
 739                  * change bits across a reset.
 740                  */
 741                 memset(memslot->arch.rmap, 0,
 742                        memslot->npages * sizeof(*memslot->arch.rmap));
 743         }
 744         srcu_read_unlock(&kvm->srcu, srcu_idx);
 745 }
 746
 747 typedef int (*hva_handler_fn)(struct kvm *kvm, struct kvm_memory_slot *memslot,
 748                               unsigned long gfn);
 749
 750 static int kvm_handle_hva_range(struct kvm *kvm,
 751                                 unsigned long start,
 752                                 unsigned long end,
 753                                 hva_handler_fn handler)
 754 {
 755         int ret;
 756         int retval = 0;
 757         struct kvm_memslots *slots;
 758         struct kvm_memory_slot *memslot;
 759
 760         slots = kvm_memslots(kvm);
 761         kvm_for_each_memslot(memslot, slots) {
 762                 unsigned long hva_start, hva_end;
 763                 gfn_t gfn, gfn_end;
 764
 765                 hva_start = max(start, memslot->userspace_addr);
 766                 hva_end = min(end, memslot->userspace_addr +
 767                                         (memslot->npages << PAGE_SHIFT));
 768                 if (hva_start >= hva_end)
 769                         continue;
 770                 /*
 771                  * {gfn(page) | page intersects with [hva_start, hva_end)} =
 772                  * {gfn, gfn+1, ..., gfn_end-1}.
 773                  */
 774                 gfn = hva_to_gfn_memslot(hva_start, memslot);
 775                 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
 776
 777                 for (; gfn < gfn_end; ++gfn) {
 778                         ret = handler(kvm, memslot, gfn);
 779                         retval |= ret;
 780                 }
 781         }
 782
 783         return retval;
 784 }
 785
 786 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 787                           hva_handler_fn handler)
 788 {
 789         return kvm_handle_hva_range(kvm, hva, hva + 1, handler);
 790 }
 791
 792 /* Must be called with both HPTE and rmap locked */
 793 static void kvmppc_unmap_hpte(struct kvm *kvm, unsigned long i,
 794                               struct kvm_memory_slot *memslot,
 795                               unsigned long *rmapp, unsigned long gfn)
 796 {
 797         __be64 *hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 798         struct revmap_entry *rev = kvm->arch.hpt.rev;
 799         unsigned long j, h;
 800         unsigned long ptel, psize, rcbits;
 801
 802         j = rev[i].forw;
 803         if (j == i) {
 804                 /* chain is now empty */
 805                 *rmapp &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX);
 806         } else {
 807                 /* remove i from chain */
 808                 h = rev[i].back;
 809                 rev[h].forw = j;
 810                 rev[j].back = h;
 811                 rev[i].forw = rev[i].back = i;
 812                 *rmapp = (*rmapp & ~KVMPPC_RMAP_INDEX) | j;
 813         }
 814
 815         /* Now check and modify the HPTE */
 816         ptel = rev[i].guest_rpte;
 817         psize = kvmppc_actual_pgsz(be64_to_cpu(hptep[0]), ptel);
 818         if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 819             hpte_rpn(ptel, psize) == gfn) {
 820                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 821                 kvmppc_invalidate_hpte(kvm, hptep, i);
 822                 hptep[1] &= ~cpu_to_be64(HPTE_R_KEY_HI | HPTE_R_KEY_LO);
 823                 /* Harvest R and C */
 824                 rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 825                 *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT;
 826                 if ((rcbits & HPTE_R_C) && memslot->dirty_bitmap)
 827                         kvmppc_update_dirty_map(memslot, gfn, psize);
 828                 if (rcbits & ~rev[i].guest_rpte) {
 829                         rev[i].guest_rpte = ptel | rcbits;
 830                         note_hpte_modification(kvm, &rev[i]);
 831                 }
 832         }
 833 }
 834
 835 static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 836                            unsigned long gfn)
 837 {
 838         unsigned long i;
 839         __be64 *hptep;
 840         unsigned long *rmapp;
 841
 842         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 843         for (;;) {
 844                 lock_rmap(rmapp);
 845                 if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 846                         unlock_rmap(rmapp);
 847                         break;
 848                 }
 849
 850                 /*
 851                  * To avoid an ABBA deadlock with the HPTE lock bit,
 852                  * we can't spin on the HPTE lock while holding the
 853                  * rmap chain lock.
 854                  */
 855                 i = *rmapp & KVMPPC_RMAP_INDEX;
 856                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 857                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 858                         /* unlock rmap before spinning on the HPTE lock */
 859                         unlock_rmap(rmapp);
 860                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 861                                 cpu_relax();
 862                         continue;
 863                 }
 864
 865                 kvmppc_unmap_hpte(kvm, i, memslot, rmapp, gfn);
 866                 unlock_rmap(rmapp);
 867                 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 868         }
 869         return 0;
 870 }
 871
 872 int kvm_unmap_hva_hv(struct kvm *kvm, unsigned long hva)
 873 {
 874         hva_handler_fn handler;
 875
 876         handler = kvm_is_radix(kvm) ? kvm_unmap_radix : kvm_unmap_rmapp;
 877         kvm_handle_hva(kvm, hva, handler);
 878         return 0;
 879 }
 880
 881 int kvm_unmap_hva_range_hv(struct kvm *kvm, unsigned long start, unsigned long end)
 882 {
 883         hva_handler_fn handler;
 884
 885         handler = kvm_is_radix(kvm) ? kvm_unmap_radix : kvm_unmap_rmapp;
 886         kvm_handle_hva_range(kvm, start, end, handler);
 887         return 0;
 888 }
 889
 890 void kvmppc_core_flush_memslot_hv(struct kvm *kvm,
 891                                   struct kvm_memory_slot *memslot)
 892 {
 893         unsigned long gfn;
 894         unsigned long n;
 895         unsigned long *rmapp;
 896
 897         gfn = memslot->base_gfn;
 898         rmapp = memslot->arch.rmap;
 899         for (n = memslot->npages; n; --n, ++gfn) {
 900                 if (kvm_is_radix(kvm)) {
 901                         kvm_unmap_radix(kvm, memslot, gfn);
 902                         continue;
 903                 }
 904                 /*
 905                  * Testing the present bit without locking is OK because
 906                  * the memslot has been marked invalid already, and hence
 907                  * no new HPTEs referencing this page can be created,
 908                  * thus the present bit can't go from 0 to 1.
 909                  */
 910                 if (*rmapp & KVMPPC_RMAP_PRESENT)
 911                         kvm_unmap_rmapp(kvm, memslot, gfn);
 912                 ++rmapp;
 913         }
 914 }
 915
 916 static int kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 917                          unsigned long gfn)
 918 {
 919         struct revmap_entry *rev = kvm->arch.hpt.rev;
 920         unsigned long head, i, j;
 921         __be64 *hptep;
 922         int ret = 0;
 923         unsigned long *rmapp;
 924
 925         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 926  retry:
 927         lock_rmap(rmapp);
 928         if (*rmapp & KVMPPC_RMAP_REFERENCED) {
 929                 *rmapp &= ~KVMPPC_RMAP_REFERENCED;
 930                 ret = 1;
 931         }
 932         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 933                 unlock_rmap(rmapp);
 934                 return ret;
 935         }
 936
 937         i = head = *rmapp & KVMPPC_RMAP_INDEX;
 938         do {
 939                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 940                 j = rev[i].forw;
 941
 942                 /* If this HPTE isn't referenced, ignore it */
 943                 if (!(be64_to_cpu(hptep[1]) & HPTE_R_R))
 944                         continue;
 945
 946                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 947                         /* unlock rmap before spinning on the HPTE lock */
 948                         unlock_rmap(rmapp);
 949                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 950                                 cpu_relax();
 951                         goto retry;
 952                 }
 953
 954                 /* Now check and modify the HPTE */
 955                 if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 956                     (be64_to_cpu(hptep[1]) & HPTE_R_R)) {
 957                         kvmppc_clear_ref_hpte(kvm, hptep, i);
 958                         if (!(rev[i].guest_rpte & HPTE_R_R)) {
 959                                 rev[i].guest_rpte |= HPTE_R_R;
 960                                 note_hpte_modification(kvm, &rev[i]);
 961                         }
 962                         ret = 1;
 963                 }
 964                 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 965         } while ((i = j) != head);
 966
 967         unlock_rmap(rmapp);
 968         return ret;
 969 }
 970
 971 int kvm_age_hva_hv(struct kvm *kvm, unsigned long start, unsigned long end)
 972 {
 973         hva_handler_fn handler;
 974
 975         handler = kvm_is_radix(kvm) ? kvm_age_radix : kvm_age_rmapp;
 976         return kvm_handle_hva_range(kvm, start, end, handler);
 977 }
 978
 979 static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 980                               unsigned long gfn)
 981 {
 982         struct revmap_entry *rev = kvm->arch.hpt.rev;
 983         unsigned long head, i, j;
 984         unsigned long *hp;
 985         int ret = 1;
 986         unsigned long *rmapp;
 987
 988         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 989         if (*rmapp & KVMPPC_RMAP_REFERENCED)
 990                 return 1;
 991
 992         lock_rmap(rmapp);
 993         if (*rmapp & KVMPPC_RMAP_REFERENCED)
 994                 goto out;
 995
 996         if (*rmapp & KVMPPC_RMAP_PRESENT) {
 997                 i = head = *rmapp & KVMPPC_RMAP_INDEX;
 998                 do {
 999                         hp = (unsigned long *)(kvm->arch.hpt.virt + (i << 4));
1000                         j = rev[i].forw;
1001                         if (be64_to_cpu(hp[1]) & HPTE_R_R)
1002                                 goto out;
1003                 } while ((i = j) != head);
1004         }
1005         ret = 0;
1006
1007  out:
1008         unlock_rmap(rmapp);
1009         return ret;
1010 }
1011
1012 int kvm_test_age_hva_hv(struct kvm *kvm, unsigned long hva)
1013 {
1014         hva_handler_fn handler;
1015
1016         handler = kvm_is_radix(kvm) ? kvm_test_age_radix : kvm_test_age_rmapp;
1017         return kvm_handle_hva(kvm, hva, handler);
1018 }
1019
1020 void kvm_set_spte_hva_hv(struct kvm *kvm, unsigned long hva, pte_t pte)
1021 {
1022         hva_handler_fn handler;
1023
1024         handler = kvm_is_radix(kvm) ? kvm_unmap_radix : kvm_unmap_rmapp;
1025         kvm_handle_hva(kvm, hva, handler);
1026 }
1027
1028 static int vcpus_running(struct kvm *kvm)
1029 {
1030         return atomic_read(&kvm->arch.vcpus_running) != 0;
1031 }
1032
1033 /*
1034  * Returns the number of system pages that are dirty.
1035  * This can be more than 1 if we find a huge-page HPTE.
1036  */
1037 static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
1038 {
1039         struct revmap_entry *rev = kvm->arch.hpt.rev;
1040         unsigned long head, i, j;
1041         unsigned long n;
1042         unsigned long v, r;
1043         __be64 *hptep;
1044         int npages_dirty = 0;
1045
1046  retry:
1047         lock_rmap(rmapp);
1048         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
1049                 unlock_rmap(rmapp);
1050                 return npages_dirty;
1051         }
1052
1053         i = head = *rmapp & KVMPPC_RMAP_INDEX;
1054         do {
1055                 unsigned long hptep1;
1056                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
1057                 j = rev[i].forw;
1058
1059                 /*
1060                  * Checking the C (changed) bit here is racy since there
1061                  * is no guarantee about when the hardware writes it back.
1062                  * If the HPTE is not writable then it is stable since the
1063                  * page can't be written to, and we would have done a tlbie
1064                  * (which forces the hardware to complete any writeback)
1065                  * when making the HPTE read-only.
1066                  * If vcpus are running then this call is racy anyway
1067                  * since the page could get dirtied subsequently, so we
1068                  * expect there to be a further call which would pick up
1069                  * any delayed C bit writeback.
1070                  * Otherwise we need to do the tlbie even if C==0 in
1071                  * order to pick up any delayed writeback of C.
1072                  */
1073                 hptep1 = be64_to_cpu(hptep[1]);
1074                 if (!(hptep1 & HPTE_R_C) &&
1075                     (!hpte_is_writable(hptep1) || vcpus_running(kvm)))
1076                         continue;
1077
1078                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
1079                         /* unlock rmap before spinning on the HPTE lock */
1080                         unlock_rmap(rmapp);
1081                         while (hptep[0] & cpu_to_be64(HPTE_V_HVLOCK))
1082                                 cpu_relax();
1083                         goto retry;
1084                 }
1085
1086                 /* Now check and modify the HPTE */
1087                 if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID))) {
1088                         __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
1089                         continue;
1090                 }
1091
1092                 /* need to make it temporarily absent so C is stable */
1093                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
1094                 kvmppc_invalidate_hpte(kvm, hptep, i);
1095                 v = be64_to_cpu(hptep[0]);
1096                 r = be64_to_cpu(hptep[1]);
1097                 if (r & HPTE_R_C) {
1098                         hptep[1] = cpu_to_be64(r & ~HPTE_R_C);
1099                         if (!(rev[i].guest_rpte & HPTE_R_C)) {
1100                                 rev[i].guest_rpte |= HPTE_R_C;
1101                                 note_hpte_modification(kvm, &rev[i]);
1102                         }
1103                         n = kvmppc_actual_pgsz(v, r);
1104                         n = (n + PAGE_SIZE - 1) >> PAGE_SHIFT;
1105                         if (n > npages_dirty)
1106                                 npages_dirty = n;
1107                         eieio();
1108                 }
1109                 v &= ~HPTE_V_ABSENT;
1110                 v |= HPTE_V_VALID;
1111                 __unlock_hpte(hptep, v);
1112         } while ((i = j) != head);
1113
1114         unlock_rmap(rmapp);
1115         return npages_dirty;
1116 }
1117
1118 void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa,
1119                               struct kvm_memory_slot *memslot,
1120                               unsigned long *map)
1121 {
1122         unsigned long gfn;
1123
1124         if (!vpa->dirty || !vpa->pinned_addr)
1125                 return;
1126         gfn = vpa->gpa >> PAGE_SHIFT;
1127         if (gfn < memslot->base_gfn ||
1128             gfn >= memslot->base_gfn + memslot->npages)
1129                 return;
1130
1131         vpa->dirty = false;
1132         if (map)
1133                 __set_bit_le(gfn - memslot->base_gfn, map);
1134 }
1135
1136 long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm,
1137                         struct kvm_memory_slot *memslot, unsigned long *map)
1138 {
1139         unsigned long i;
1140         unsigned long *rmapp;
1141
1142         preempt_disable();
1143         rmapp = memslot->arch.rmap;
1144         for (i = 0; i < memslot->npages; ++i) {
1145                 int npages = kvm_test_clear_dirty_npages(kvm, rmapp);
1146                 /*
1147                  * Note that if npages > 0 then i must be a multiple of npages,
1148                  * since we always put huge-page HPTEs in the rmap chain
1149                  * corresponding to their page base address.
1150                  */
1151                 if (npages)
1152                         set_dirty_bits(map, i, npages);
1153                 ++rmapp;
1154         }
1155         preempt_enable();
1156         return 0;
1157 }
1158
1159 void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,
1160                             unsigned long *nb_ret)
1161 {
1162         struct kvm_memory_slot *memslot;
1163         unsigned long gfn = gpa >> PAGE_SHIFT;
1164         struct page *page, *pages[1];
1165         int npages;
1166         unsigned long hva, offset;
1167         int srcu_idx;
1168
1169         srcu_idx = srcu_read_lock(&kvm->srcu);
1170         memslot = gfn_to_memslot(kvm, gfn);
1171         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
1172                 goto err;
1173         hva = gfn_to_hva_memslot(memslot, gfn);
1174         npages = get_user_pages_fast(hva, 1, 1, pages);
1175         if (npages < 1)
1176                 goto err;
1177         page = pages[0];
1178         srcu_read_unlock(&kvm->srcu, srcu_idx);
1179
1180         offset = gpa & (PAGE_SIZE - 1);
1181         if (nb_ret)
1182                 *nb_ret = PAGE_SIZE - offset;
1183         return page_address(page) + offset;
1184
1185  err:
1186         srcu_read_unlock(&kvm->srcu, srcu_idx);
1187         return NULL;
1188 }
1189
1190 void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
1191                              bool dirty)
1192 {
1193         struct page *page = virt_to_page(va);
1194         struct kvm_memory_slot *memslot;
1195         unsigned long gfn;
1196         int srcu_idx;
1197
1198         put_page(page);
1199
1200         if (!dirty)
1201                 return;
1202
1203         /* We need to mark this page dirty in the memslot dirty_bitmap, if any */
1204         gfn = gpa >> PAGE_SHIFT;
1205         srcu_idx = srcu_read_lock(&kvm->srcu);
1206         memslot = gfn_to_memslot(kvm, gfn);
1207         if (memslot && memslot->dirty_bitmap)
1208                 set_bit_le(gfn - memslot->base_gfn, memslot->dirty_bitmap);
1209         srcu_read_unlock(&kvm->srcu, srcu_idx);
1210 }
1211
1212 /*
1213  * HPT resizing
1214  */
1215 static int resize_hpt_allocate(struct kvm_resize_hpt *resize)
1216 {
1217         int rc;
1218
1219         rc = kvmppc_allocate_hpt(&resize->hpt, resize->order);
1220         if (rc < 0)
1221                 return rc;
1222
1223         resize_hpt_debug(resize, "resize_hpt_allocate(): HPT @ 0x%lx\n",
1224                          resize->hpt.virt);
1225
1226         return 0;
1227 }
1228
1229 static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
1230                                             unsigned long idx)
1231 {
1232         struct kvm *kvm = resize->kvm;
1233         struct kvm_hpt_info *old = &kvm->arch.hpt;
1234         struct kvm_hpt_info *new = &resize->hpt;
1235         unsigned long old_hash_mask = (1ULL << (old->order - 7)) - 1;
1236         unsigned long new_hash_mask = (1ULL << (new->order - 7)) - 1;
1237         __be64 *hptep, *new_hptep;
1238         unsigned long vpte, rpte, guest_rpte;
1239         int ret;
1240         struct revmap_entry *rev;
1241         unsigned long apsize, avpn, pteg, hash;
1242         unsigned long new_idx, new_pteg, replace_vpte;
1243         int pshift;
1244
1245         hptep = (__be64 *)(old->virt + (idx << 4));
1246
1247         /* Guest is stopped, so new HPTEs can't be added or faulted
1248          * in, only unmapped or altered by host actions.  So, it's
1249          * safe to check this before we take the HPTE lock */
1250         vpte = be64_to_cpu(hptep[0]);
1251         if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
1252                 return 0; /* nothing to do */
1253
1254         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
1255                 cpu_relax();
1256
1257         vpte = be64_to_cpu(hptep[0]);
1258
1259         ret = 0;
1260         if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
1261                 /* Nothing to do */
1262                 goto out;
1263
1264         /* Unmap */
1265         rev = &old->rev[idx];
1266         guest_rpte = rev->guest_rpte;
1267
1268         ret = -EIO;
1269         apsize = kvmppc_actual_pgsz(vpte, guest_rpte);
1270         if (!apsize)
1271                 goto out;
1272
1273         if (vpte & HPTE_V_VALID) {
1274                 unsigned long gfn = hpte_rpn(guest_rpte, apsize);
1275                 int srcu_idx = srcu_read_lock(&kvm->srcu);
1276                 struct kvm_memory_slot *memslot =
1277                         __gfn_to_memslot(kvm_memslots(kvm), gfn);
1278
1279                 if (memslot) {
1280                         unsigned long *rmapp;
1281                         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
1282
1283                         lock_rmap(rmapp);
1284                         kvmppc_unmap_hpte(kvm, idx, memslot, rmapp, gfn);
1285                         unlock_rmap(rmapp);
1286                 }
1287
1288                 srcu_read_unlock(&kvm->srcu, srcu_idx);
1289         }
1290
1291         /* Reload PTE after unmap */
1292         vpte = be64_to_cpu(hptep[0]);
1293
1294         BUG_ON(vpte & HPTE_V_VALID);
1295         BUG_ON(!(vpte & HPTE_V_ABSENT));
1296
1297         ret = 0;
1298         if (!(vpte & HPTE_V_BOLTED))
1299                 goto out;
1300
1301         rpte = be64_to_cpu(hptep[1]);
1302         pshift = kvmppc_hpte_base_page_shift(vpte, rpte);
1303         avpn = HPTE_V_AVPN_VAL(vpte) & ~(((1ul << pshift) - 1) >> 23);
1304         pteg = idx / HPTES_PER_GROUP;
1305         if (vpte & HPTE_V_SECONDARY)
1306                 pteg = ~pteg;
1307
1308         if (!(vpte & HPTE_V_1TB_SEG)) {
1309                 unsigned long offset, vsid;
1310
1311                 /* We only have 28 - 23 bits of offset in avpn */
1312                 offset = (avpn & 0x1f) << 23;
1313                 vsid = avpn >> 5;
1314                 /* We can find more bits from the pteg value */
1315                 if (pshift < 23)
1316                         offset |= ((vsid ^ pteg) & old_hash_mask) << pshift;
1317
1318                 hash = vsid ^ (offset >> pshift);
1319         } else {
1320                 unsigned long offset, vsid;
1321
1322                 /* We only have 40 - 23 bits of seg_off in avpn */
1323                 offset = (avpn & 0x1ffff) << 23;
1324                 vsid = avpn >> 17;
1325                 if (pshift < 23)
1326                         offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask) << pshift;
1327
1328                 hash = vsid ^ (vsid << 25) ^ (offset >> pshift);
1329         }
1330
1331         new_pteg = hash & new_hash_mask;
1332         if (vpte & HPTE_V_SECONDARY) {
1333                 BUG_ON(~pteg != (hash & old_hash_mask));
1334                 new_pteg = ~new_pteg;
1335         } else {
1336                 BUG_ON(pteg != (hash & old_hash_mask));
1337         }
1338
1339         new_idx = new_pteg * HPTES_PER_GROUP + (idx % HPTES_PER_GROUP);
1340         new_hptep = (__be64 *)(new->virt + (new_idx << 4));
1341
1342         replace_vpte = be64_to_cpu(new_hptep[0]);
1343
1344         if (replace_vpte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1345                 BUG_ON(new->order >= old->order);
1346
1347                 if (replace_vpte & HPTE_V_BOLTED) {
1348                         if (vpte & HPTE_V_BOLTED)
1349                                 /* Bolted collision, nothing we can do */
1350                                 ret = -ENOSPC;
1351                         /* Discard the new HPTE */
1352                         goto out;
1353                 }
1354
1355                 /* Discard the previous HPTE */
1356         }
1357
1358         new_hptep[1] = cpu_to_be64(rpte);
1359         new->rev[new_idx].guest_rpte = guest_rpte;
1360         /* No need for a barrier, since new HPT isn't active */
1361         new_hptep[0] = cpu_to_be64(vpte);
1362         unlock_hpte(new_hptep, vpte);
1363
1364 out:
1365         unlock_hpte(hptep, vpte);
1366         return ret;
1367 }
1368
1369 static int resize_hpt_rehash(struct kvm_resize_hpt *resize)
1370 {
1371         struct kvm *kvm = resize->kvm;
1372         unsigned  long i;
1373         int rc;
1374
1375         /*
1376          * resize_hpt_rehash_hpte() doesn't handle the new-format HPTEs
1377          * that POWER9 uses, and could well hit a BUG_ON on POWER9.
1378          */
1379         if (cpu_has_feature(CPU_FTR_ARCH_300))
1380                 return -EIO;
1381         for (i = 0; i < kvmppc_hpt_npte(&kvm->arch.hpt); i++) {
1382                 rc = resize_hpt_rehash_hpte(resize, i);
1383                 if (rc != 0)
1384                         return rc;
1385         }
1386
1387         return 0;
1388 }
1389
1390 static void resize_hpt_pivot(struct kvm_resize_hpt *resize)
1391 {
1392         struct kvm *kvm = resize->kvm;
1393         struct kvm_hpt_info hpt_tmp;
1394
1395         /* Exchange the pending tables in the resize structure with
1396          * the active tables */
1397
1398         resize_hpt_debug(resize, "resize_hpt_pivot()\n");
1399
1400         spin_lock(&kvm->mmu_lock);
1401         asm volatile("ptesync" : : : "memory");
1402
1403         hpt_tmp = kvm->arch.hpt;
1404         kvmppc_set_hpt(kvm, &resize->hpt);
1405         resize->hpt = hpt_tmp;
1406
1407         spin_unlock(&kvm->mmu_lock);
1408
1409         synchronize_srcu_expedited(&kvm->srcu);
1410
1411         resize_hpt_debug(resize, "resize_hpt_pivot() done\n");
1412 }
1413
1414 static void resize_hpt_release(struct kvm *kvm, struct kvm_resize_hpt *resize)
1415 {
1416         BUG_ON(kvm->arch.resize_hpt != resize);
1417
1418         if (!resize)
1419                 return;
1420
1421         if (resize->hpt.virt)
1422                 kvmppc_free_hpt(&resize->hpt);
1423
1424         kvm->arch.resize_hpt = NULL;
1425         kfree(resize);
1426 }
1427
1428 static void resize_hpt_prepare_work(struct work_struct *work)
1429 {
1430         struct kvm_resize_hpt *resize = container_of(work,
1431                                                      struct kvm_resize_hpt,
1432                                                      work);
1433         struct kvm *kvm = resize->kvm;
1434         int err;
1435
1436         resize_hpt_debug(resize, "resize_hpt_prepare_work(): order = %d\n",
1437                          resize->order);
1438
1439         err = resize_hpt_allocate(resize);
1440
1441         mutex_lock(&kvm->lock);
1442
1443         resize->error = err;
1444         resize->prepare_done = true;
1445
1446         mutex_unlock(&kvm->lock);
1447 }
1448
1449 long kvm_vm_ioctl_resize_hpt_prepare(struct kvm *kvm,
1450                                      struct kvm_ppc_resize_hpt *rhpt)
1451 {
1452         unsigned long flags = rhpt->flags;
1453         unsigned long shift = rhpt->shift;
1454         struct kvm_resize_hpt *resize;
1455         int ret;
1456
1457         if (flags != 0 || kvm_is_radix(kvm))
1458                 return -EINVAL;
1459
1460         if (shift && ((shift < 18) || (shift > 46)))
1461                 return -EINVAL;
1462
1463         mutex_lock(&kvm->lock);
1464
1465         resize = kvm->arch.resize_hpt;
1466
1467         if (resize) {
1468                 if (resize->order == shift) {
1469                         /* Suitable resize in progress */
1470                         if (resize->prepare_done) {
1471                                 ret = resize->error;
1472                                 if (ret != 0)
1473                                         resize_hpt_release(kvm, resize);
1474                         } else {
1475                                 ret = 100; /* estimated time in ms */
1476                         }
1477
1478                         goto out;
1479                 }
1480
1481                 /* not suitable, cancel it */
1482                 resize_hpt_release(kvm, resize);
1483         }
1484
1485         ret = 0;
1486         if (!shift)
1487                 goto out; /* nothing to do */
1488
1489         /* start new resize */
1490
1491         resize = kzalloc(sizeof(*resize), GFP_KERNEL);
1492         if (!resize) {
1493                 ret = -ENOMEM;
1494                 goto out;
1495         }
1496         resize->order = shift;
1497         resize->kvm = kvm;
1498         INIT_WORK(&resize->work, resize_hpt_prepare_work);
1499         kvm->arch.resize_hpt = resize;
1500
1501         schedule_work(&resize->work);
1502
1503         ret = 100; /* estimated time in ms */
1504
1505 out:
1506         mutex_unlock(&kvm->lock);
1507         return ret;
1508 }
1509
1510 static void resize_hpt_boot_vcpu(void *opaque)
1511 {
1512         /* Nothing to do, just force a KVM exit */
1513 }
1514
1515 long kvm_vm_ioctl_resize_hpt_commit(struct kvm *kvm,
1516                                     struct kvm_ppc_resize_hpt *rhpt)
1517 {
1518         unsigned long flags = rhpt->flags;
1519         unsigned long shift = rhpt->shift;
1520         struct kvm_resize_hpt *resize;
1521         long ret;
1522
1523         if (flags != 0 || kvm_is_radix(kvm))
1524                 return -EINVAL;
1525
1526         if (shift && ((shift < 18) || (shift > 46)))
1527                 return -EINVAL;
1528
1529         mutex_lock(&kvm->lock);
1530
1531         resize = kvm->arch.resize_hpt;
1532
1533         /* This shouldn't be possible */
1534         ret = -EIO;
1535         if (WARN_ON(!kvm->arch.mmu_ready))
1536                 goto out_no_hpt;
1537
1538         /* Stop VCPUs from running while we mess with the HPT */
1539         kvm->arch.mmu_ready = 0;
1540         smp_mb();
1541
1542         /* Boot all CPUs out of the guest so they re-read
1543          * mmu_ready */
1544         on_each_cpu(resize_hpt_boot_vcpu, NULL, 1);
1545
1546         ret = -ENXIO;
1547         if (!resize || (resize->order != shift))
1548                 goto out;
1549
1550         ret = -EBUSY;
1551         if (!resize->prepare_done)
1552                 goto out;
1553
1554         ret = resize->error;
1555         if (ret != 0)
1556                 goto out;
1557
1558         ret = resize_hpt_rehash(resize);
1559         if (ret != 0)
1560                 goto out;
1561
1562         resize_hpt_pivot(resize);
1563
1564 out:
1565         /* Let VCPUs run again */
1566         kvm->arch.mmu_ready = 1;
1567         smp_mb();
1568 out_no_hpt:
1569         resize_hpt_release(kvm, resize);
1570         mutex_unlock(&kvm->lock);
1571         return ret;
1572 }
1573
1574 /*
1575  * Functions for reading and writing the hash table via reads and
1576  * writes on a file descriptor.
1577  *
1578  * Reads return the guest view of the hash table, which has to be
1579  * pieced together from the real hash table and the guest_rpte
1580  * values in the revmap array.
1581  *
1582  * On writes, each HPTE written is considered in turn, and if it
1583  * is valid, it is written to the HPT as if an H_ENTER with the
1584  * exact flag set was done.  When the invalid count is non-zero
1585  * in the header written to the stream, the kernel will make
1586  * sure that that many HPTEs are invalid, and invalidate them
1587  * if not.
1588  */
1589
1590 struct kvm_htab_ctx {
1591         unsigned long   index;
1592         unsigned long   flags;
1593         struct kvm      *kvm;
1594         int             first_pass;
1595 };
1596
1597 #define HPTE_SIZE       (2 * sizeof(unsigned long))
1598
1599 /*
1600  * Returns 1 if this HPT entry has been modified or has pending
1601  * R/C bit changes.
1602  */
1603 static int hpte_dirty(struct revmap_entry *revp, __be64 *hptp)
1604 {
1605         unsigned long rcbits_unset;
1606
1607         if (revp->guest_rpte & HPTE_GR_MODIFIED)
1608                 return 1;
1609
1610         /* Also need to consider changes in reference and changed bits */
1611         rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1612         if ((be64_to_cpu(hptp[0]) & HPTE_V_VALID) &&
1613             (be64_to_cpu(hptp[1]) & rcbits_unset))
1614                 return 1;
1615
1616         return 0;
1617 }
1618
1619 static long record_hpte(unsigned long flags, __be64 *hptp,
1620                         unsigned long *hpte, struct revmap_entry *revp,
1621                         int want_valid, int first_pass)
1622 {
1623         unsigned long v, r, hr;
1624         unsigned long rcbits_unset;
1625         int ok = 1;
1626         int valid, dirty;
1627
1628         /* Unmodified entries are uninteresting except on the first pass */
1629         dirty = hpte_dirty(revp, hptp);
1630         if (!first_pass && !dirty)
1631                 return 0;
1632
1633         valid = 0;
1634         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1635                 valid = 1;
1636                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) &&
1637                     !(be64_to_cpu(hptp[0]) & HPTE_V_BOLTED))
1638                         valid = 0;
1639         }
1640         if (valid != want_valid)
1641                 return 0;
1642
1643         v = r = 0;
1644         if (valid || dirty) {
1645                 /* lock the HPTE so it's stable and read it */
1646                 preempt_disable();
1647                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
1648                         cpu_relax();
1649                 v = be64_to_cpu(hptp[0]);
1650                 hr = be64_to_cpu(hptp[1]);
1651                 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1652                         v = hpte_new_to_old_v(v, hr);
1653                         hr = hpte_new_to_old_r(hr);
1654                 }
1655
1656                 /* re-evaluate valid and dirty from synchronized HPTE value */
1657                 valid = !!(v & HPTE_V_VALID);
1658                 dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED);
1659
1660                 /* Harvest R and C into guest view if necessary */
1661                 rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1662                 if (valid && (rcbits_unset & hr)) {
1663                         revp->guest_rpte |= (hr &
1664                                 (HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED;
1665                         dirty = 1;
1666                 }
1667
1668                 if (v & HPTE_V_ABSENT) {
1669                         v &= ~HPTE_V_ABSENT;
1670                         v |= HPTE_V_VALID;
1671                         valid = 1;
1672                 }
1673                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED))
1674                         valid = 0;
1675
1676                 r = revp->guest_rpte;
1677                 /* only clear modified if this is the right sort of entry */
1678                 if (valid == want_valid && dirty) {
1679                         r &= ~HPTE_GR_MODIFIED;
1680                         revp->guest_rpte = r;
1681                 }
1682                 unlock_hpte(hptp, be64_to_cpu(hptp[0]));
1683                 preempt_enable();
1684                 if (!(valid == want_valid && (first_pass || dirty)))
1685                         ok = 0;
1686         }
1687         hpte[0] = cpu_to_be64(v);
1688         hpte[1] = cpu_to_be64(r);
1689         return ok;
1690 }
1691
1692 static ssize_t kvm_htab_read(struct file *file, char __user *buf,
1693                              size_t count, loff_t *ppos)
1694 {
1695         struct kvm_htab_ctx *ctx = file->private_data;
1696         struct kvm *kvm = ctx->kvm;
1697         struct kvm_get_htab_header hdr;
1698         __be64 *hptp;
1699         struct revmap_entry *revp;
1700         unsigned long i, nb, nw;
1701         unsigned long __user *lbuf;
1702         struct kvm_get_htab_header __user *hptr;
1703         unsigned long flags;
1704         int first_pass;
1705         unsigned long hpte[2];
1706
1707         if (!access_ok(VERIFY_WRITE, buf, count))
1708                 return -EFAULT;
1709         if (kvm_is_radix(kvm))
1710                 return 0;
1711
1712         first_pass = ctx->first_pass;
1713         flags = ctx->flags;
1714
1715         i = ctx->index;
1716         hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
1717         revp = kvm->arch.hpt.rev + i;
1718         lbuf = (unsigned long __user *)buf;
1719
1720         nb = 0;
1721         while (nb + sizeof(hdr) + HPTE_SIZE < count) {
1722                 /* Initialize header */
1723                 hptr = (struct kvm_get_htab_header __user *)buf;
1724                 hdr.n_valid = 0;
1725                 hdr.n_invalid = 0;
1726                 nw = nb;
1727                 nb += sizeof(hdr);
1728                 lbuf = (unsigned long __user *)(buf + sizeof(hdr));
1729
1730                 /* Skip uninteresting entries, i.e. clean on not-first pass */
1731                 if (!first_pass) {
1732                         while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1733                                !hpte_dirty(revp, hptp)) {
1734                                 ++i;
1735                                 hptp += 2;
1736                                 ++revp;
1737                         }
1738                 }
1739                 hdr.index = i;
1740
1741                 /* Grab a series of valid entries */
1742                 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1743                        hdr.n_valid < 0xffff &&
1744                        nb + HPTE_SIZE < count &&
1745                        record_hpte(flags, hptp, hpte, revp, 1, first_pass)) {
1746                         /* valid entry, write it out */
1747                         ++hdr.n_valid;
1748                         if (__put_user(hpte[0], lbuf) ||
1749                             __put_user(hpte[1], lbuf + 1))
1750                                 return -EFAULT;
1751                         nb += HPTE_SIZE;
1752                         lbuf += 2;
1753                         ++i;
1754                         hptp += 2;
1755                         ++revp;
1756                 }
1757                 /* Now skip invalid entries while we can */
1758                 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1759                        hdr.n_invalid < 0xffff &&
1760                        record_hpte(flags, hptp, hpte, revp, 0, first_pass)) {
1761                         /* found an invalid entry */
1762                         ++hdr.n_invalid;
1763                         ++i;
1764                         hptp += 2;
1765                         ++revp;
1766                 }
1767
1768                 if (hdr.n_valid || hdr.n_invalid) {
1769                         /* write back the header */
1770                         if (__copy_to_user(hptr, &hdr, sizeof(hdr)))
1771                                 return -EFAULT;
1772                         nw = nb;
1773                         buf = (char __user *)lbuf;
1774                 } else {
1775                         nb = nw;
1776                 }
1777
1778                 /* Check if we've wrapped around the hash table */
1779                 if (i >= kvmppc_hpt_npte(&kvm->arch.hpt)) {
1780                         i = 0;
1781                         ctx->first_pass = 0;
1782                         break;
1783                 }
1784         }
1785
1786         ctx->index = i;
1787
1788         return nb;
1789 }
1790
1791 static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1792                               size_t count, loff_t *ppos)
1793 {
1794         struct kvm_htab_ctx *ctx = file->private_data;
1795         struct kvm *kvm = ctx->kvm;
1796         struct kvm_get_htab_header hdr;
1797         unsigned long i, j;
1798         unsigned long v, r;
1799         unsigned long __user *lbuf;
1800         __be64 *hptp;
1801         unsigned long tmp[2];
1802         ssize_t nb;
1803         long int err, ret;
1804         int mmu_ready;
1805         int pshift;
1806
1807         if (!access_ok(VERIFY_READ, buf, count))
1808                 return -EFAULT;
1809         if (kvm_is_radix(kvm))
1810                 return -EINVAL;
1811
1812         /* lock out vcpus from running while we're doing this */
1813         mutex_lock(&kvm->lock);
1814         mmu_ready = kvm->arch.mmu_ready;
1815         if (mmu_ready) {
1816                 kvm->arch.mmu_ready = 0;        /* temporarily */
1817                 /* order mmu_ready vs. vcpus_running */
1818                 smp_mb();
1819                 if (atomic_read(&kvm->arch.vcpus_running)) {
1820                         kvm->arch.mmu_ready = 1;
1821                         mutex_unlock(&kvm->lock);
1822                         return -EBUSY;
1823                 }
1824         }
1825
1826         err = 0;
1827         for (nb = 0; nb + sizeof(hdr) <= count; ) {
1828                 err = -EFAULT;
1829                 if (__copy_from_user(&hdr, buf, sizeof(hdr)))
1830                         break;
1831
1832                 err = 0;
1833                 if (nb + hdr.n_valid * HPTE_SIZE > count)
1834                         break;
1835
1836                 nb += sizeof(hdr);
1837                 buf += sizeof(hdr);
1838
1839                 err = -EINVAL;
1840                 i = hdr.index;
1841                 if (i >= kvmppc_hpt_npte(&kvm->arch.hpt) ||
1842                     i + hdr.n_valid + hdr.n_invalid > kvmppc_hpt_npte(&kvm->arch.hpt))
1843                         break;
1844
1845                 hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
1846                 lbuf = (unsigned long __user *)buf;
1847                 for (j = 0; j < hdr.n_valid; ++j) {
1848                         __be64 hpte_v;
1849                         __be64 hpte_r;
1850
1851                         err = -EFAULT;
1852                         if (__get_user(hpte_v, lbuf) ||
1853                             __get_user(hpte_r, lbuf + 1))
1854                                 goto out;
1855                         v = be64_to_cpu(hpte_v);
1856                         r = be64_to_cpu(hpte_r);
1857                         err = -EINVAL;
1858                         if (!(v & HPTE_V_VALID))
1859                                 goto out;
1860                         pshift = kvmppc_hpte_base_page_shift(v, r);
1861                         if (pshift <= 0)
1862                                 goto out;
1863                         lbuf += 2;
1864                         nb += HPTE_SIZE;
1865
1866                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1867                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1868                         err = -EIO;
1869                         ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
1870                                                          tmp);
1871                         if (ret != H_SUCCESS) {
1872                                 pr_err("kvm_htab_write ret %ld i=%ld v=%lx "
1873                                        "r=%lx\n", ret, i, v, r);
1874                                 goto out;
1875                         }
1876                         if (!mmu_ready && is_vrma_hpte(v)) {
1877                                 unsigned long senc, lpcr;
1878
1879                                 senc = slb_pgsize_encoding(1ul << pshift);
1880                                 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
1881                                         (VRMA_VSID << SLB_VSID_SHIFT_1T);
1882                                 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
1883                                         lpcr = senc << (LPCR_VRMASD_SH - 4);
1884                                         kvmppc_update_lpcr(kvm, lpcr,
1885                                                            LPCR_VRMASD);
1886                                 } else {
1887                                         kvmppc_setup_partition_table(kvm);
1888                                 }
1889                                 mmu_ready = 1;
1890                         }
1891                         ++i;
1892                         hptp += 2;
1893                 }
1894
1895                 for (j = 0; j < hdr.n_invalid; ++j) {
1896                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1897                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1898                         ++i;
1899                         hptp += 2;
1900                 }
1901                 err = 0;
1902         }
1903
1904  out:
1905         /* Order HPTE updates vs. mmu_ready */
1906         smp_wmb();
1907         kvm->arch.mmu_ready = mmu_ready;
1908         mutex_unlock(&kvm->lock);
1909
1910         if (err)
1911                 return err;
1912         return nb;
1913 }
1914
1915 static int kvm_htab_release(struct inode *inode, struct file *filp)
1916 {
1917         struct kvm_htab_ctx *ctx = filp->private_data;
1918
1919         filp->private_data = NULL;
1920         if (!(ctx->flags & KVM_GET_HTAB_WRITE))
1921                 atomic_dec(&ctx->kvm->arch.hpte_mod_interest);
1922         kvm_put_kvm(ctx->kvm);
1923         kfree(ctx);
1924         return 0;
1925 }
1926
1927 static const struct file_operations kvm_htab_fops = {
1928         .read           = kvm_htab_read,
1929         .write          = kvm_htab_write,
1930         .llseek         = default_llseek,
1931         .release        = kvm_htab_release,
1932 };
1933
1934 int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)
1935 {
1936         int ret;
1937         struct kvm_htab_ctx *ctx;
1938         int rwflag;
1939
1940         /* reject flags we don't recognize */
1941         if (ghf->flags & ~(KVM_GET_HTAB_BOLTED_ONLY | KVM_GET_HTAB_WRITE))
1942                 return -EINVAL;
1943         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1944         if (!ctx)
1945                 return -ENOMEM;
1946         kvm_get_kvm(kvm);
1947         ctx->kvm = kvm;
1948         ctx->index = ghf->start_index;
1949         ctx->flags = ghf->flags;
1950         ctx->first_pass = 1;
1951
1952         rwflag = (ghf->flags & KVM_GET_HTAB_WRITE) ? O_WRONLY : O_RDONLY;
1953         ret = anon_inode_getfd("kvm-htab", &kvm_htab_fops, ctx, rwflag | O_CLOEXEC);
1954         if (ret < 0) {
1955                 kfree(ctx);
1956                 kvm_put_kvm(kvm);
1957                 return ret;
1958         }
1959
1960         if (rwflag == O_RDONLY) {
1961                 mutex_lock(&kvm->slots_lock);
1962                 atomic_inc(&kvm->arch.hpte_mod_interest);
1963                 /* make sure kvmppc_do_h_enter etc. see the increment */
1964                 synchronize_srcu_expedited(&kvm->srcu);
1965                 mutex_unlock(&kvm->slots_lock);
1966         }
1967
1968         return ret;
1969 }
1970
1971 struct debugfs_htab_state {
1972         struct kvm      *kvm;
1973         struct mutex    mutex;
1974         unsigned long   hpt_index;
1975         int             chars_left;
1976         int             buf_index;
1977         char            buf[64];
1978 };
1979
1980 static int debugfs_htab_open(struct inode *inode, struct file *file)
1981 {
1982         struct kvm *kvm = inode->i_private;
1983         struct debugfs_htab_state *p;
1984
1985         p = kzalloc(sizeof(*p), GFP_KERNEL);
1986         if (!p)
1987                 return -ENOMEM;
1988
1989         kvm_get_kvm(kvm);
1990         p->kvm = kvm;
1991         mutex_init(&p->mutex);
1992         file->private_data = p;
1993
1994         return nonseekable_open(inode, file);
1995 }
1996
1997 static int debugfs_htab_release(struct inode *inode, struct file *file)
1998 {
1999         struct debugfs_htab_state *p = file->private_data;
2000
2001         kvm_put_kvm(p->kvm);
2002         kfree(p);
2003         return 0;
2004 }
2005
2006 static ssize_t debugfs_htab_read(struct file *file, char __user *buf,
2007                                  size_t len, loff_t *ppos)
2008 {
2009         struct debugfs_htab_state *p = file->private_data;
2010         ssize_t ret, r;
2011         unsigned long i, n;
2012         unsigned long v, hr, gr;
2013         struct kvm *kvm;
2014         __be64 *hptp;
2015
2016         kvm = p->kvm;
2017         if (kvm_is_radix(kvm))
2018                 return 0;
2019
2020         ret = mutex_lock_interruptible(&p->mutex);
2021         if (ret)
2022                 return ret;
2023
2024         if (p->chars_left) {
2025                 n = p->chars_left;
2026                 if (n > len)
2027                         n = len;
2028                 r = copy_to_user(buf, p->buf + p->buf_index, n);
2029                 n -= r;
2030                 p->chars_left -= n;
2031                 p->buf_index += n;
2032                 buf += n;
2033                 len -= n;
2034                 ret = n;
2035                 if (r) {
2036                         if (!n)
2037                                 ret = -EFAULT;
2038                         goto out;
2039                 }
2040         }
2041
2042         i = p->hpt_index;
2043         hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
2044         for (; len != 0 && i < kvmppc_hpt_npte(&kvm->arch.hpt);
2045              ++i, hptp += 2) {
2046                 if (!(be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)))
2047                         continue;
2048
2049                 /* lock the HPTE so it's stable and read it */
2050                 preempt_disable();
2051                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
2052                         cpu_relax();
2053                 v = be64_to_cpu(hptp[0]) & ~HPTE_V_HVLOCK;
2054                 hr = be64_to_cpu(hptp[1]);
2055                 gr = kvm->arch.hpt.rev[i].guest_rpte;
2056                 unlock_hpte(hptp, v);
2057                 preempt_enable();
2058
2059                 if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT)))
2060                         continue;
2061
2062                 n = scnprintf(p->buf, sizeof(p->buf),
2063                               "%6lx %.16lx %.16lx %.16lx\n",
2064                               i, v, hr, gr);
2065                 p->chars_left = n;
2066                 if (n > len)
2067                         n = len;
2068                 r = copy_to_user(buf, p->buf, n);
2069                 n -= r;
2070                 p->chars_left -= n;
2071                 p->buf_index = n;
2072                 buf += n;
2073                 len -= n;
2074                 ret += n;
2075                 if (r) {
2076                         if (!ret)
2077                                 ret = -EFAULT;
2078                         goto out;
2079                 }
2080         }
2081         p->hpt_index = i;
2082
2083  out:
2084         mutex_unlock(&p->mutex);
2085         return ret;
2086 }
2087
2088 static ssize_t debugfs_htab_write(struct file *file, const char __user *buf,
2089                            size_t len, loff_t *ppos)
2090 {
2091         return -EACCES;
2092 }
2093
2094 static const struct file_operations debugfs_htab_fops = {
2095         .owner   = THIS_MODULE,
2096         .open    = debugfs_htab_open,
2097         .release = debugfs_htab_release,
2098         .read    = debugfs_htab_read,
2099         .write   = debugfs_htab_write,
2100         .llseek  = generic_file_llseek,
2101 };
2102
2103 void kvmppc_mmu_debugfs_init(struct kvm *kvm)
2104 {
2105         kvm->arch.htab_dentry = debugfs_create_file("htab", 0400,
2106                                                     kvm->arch.debugfs_dir, kvm,
2107                                                     &debugfs_htab_fops);
2108 }
2109
2110 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu)
2111 {
2112         struct kvmppc_mmu *mmu = &vcpu->arch.mmu;
2113
2114         vcpu->arch.slb_nr = 32;         /* POWER7/POWER8 */
2115
2116         mmu->xlate = kvmppc_mmu_book3s_64_hv_xlate;
2117         mmu->reset_msr = kvmppc_mmu_book3s_64_hv_reset_msr;
2118
2119         vcpu->arch.hflags |= BOOK3S_HFLAG_SLB;
2120 }