arch/arm64/kvm/hyp/nvhe/pkvm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2021 Google LLC
   4  * Author: Fuad Tabba <tabba@google.com>
   5  */
   6
   7 #include <linux/kvm_host.h>
   8 #include <linux/mm.h>
   9 #include <nvhe/fixed_config.h>
  10 #include <nvhe/mem_protect.h>
  11 #include <nvhe/memory.h>
  12 #include <nvhe/pkvm.h>
  13 #include <nvhe/trap_handler.h>
  14
  15 /* Used by icache_is_vpipt(). */
  16 unsigned long __icache_flags;
  17
  18 /* Used by kvm_get_vttbr(). */
  19 unsigned int kvm_arm_vmid_bits;
  20
  21 /*
  22  * Set trap register values based on features in ID_AA64PFR0.
  23  */
  24 static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu)
  25 {
  26         const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR0_EL1);
  27         u64 hcr_set = HCR_RW;
  28         u64 hcr_clear = 0;
  29         u64 cptr_set = 0;
  30         u64 cptr_clear = 0;
  31
  32         /* Protected KVM does not support AArch32 guests. */
  33         BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0),
  34                 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY);
  35         BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1),
  36                 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY);
  37
  38         /*
  39          * Linux guests assume support for floating-point and Advanced SIMD. Do
  40          * not change the trapping behavior for these from the KVM default.
  41          */
  42         BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP),
  43                                 PVM_ID_AA64PFR0_ALLOW));
  44         BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD),
  45                                 PVM_ID_AA64PFR0_ALLOW));
  46
  47         if (has_hvhe())
  48                 hcr_set |= HCR_E2H;
  49
  50         /* Trap RAS unless all current versions are supported */
  51         if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), feature_ids) <
  52             ID_AA64PFR0_EL1_RAS_V1P1) {
  53                 hcr_set |= HCR_TERR | HCR_TEA;
  54                 hcr_clear |= HCR_FIEN;
  55         }
  56
  57         /* Trap AMU */
  58         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU), feature_ids)) {
  59                 hcr_clear |= HCR_AMVOFFEN;
  60                 cptr_set |= CPTR_EL2_TAM;
  61         }
  62
  63         /* Trap SVE */
  64         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids)) {
  65                 if (has_hvhe())
  66                         cptr_clear |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN;
  67                 else
  68                         cptr_set |= CPTR_EL2_TZ;
  69         }
  70
  71         vcpu->arch.hcr_el2 |= hcr_set;
  72         vcpu->arch.hcr_el2 &= ~hcr_clear;
  73         vcpu->arch.cptr_el2 |= cptr_set;
  74         vcpu->arch.cptr_el2 &= ~cptr_clear;
  75 }
  76
  77 /*
  78  * Set trap register values based on features in ID_AA64PFR1.
  79  */
  80 static void pvm_init_traps_aa64pfr1(struct kvm_vcpu *vcpu)
  81 {
  82         const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR1_EL1);
  83         u64 hcr_set = 0;
  84         u64 hcr_clear = 0;
  85
  86         /* Memory Tagging: Trap and Treat as Untagged if not supported. */
  87         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE), feature_ids)) {
  88                 hcr_set |= HCR_TID5;
  89                 hcr_clear |= HCR_DCT | HCR_ATA;
  90         }
  91
  92         vcpu->arch.hcr_el2 |= hcr_set;
  93         vcpu->arch.hcr_el2 &= ~hcr_clear;
  94 }
  95
  96 /*
  97  * Set trap register values based on features in ID_AA64DFR0.
  98  */
  99 static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu)
 100 {
 101         const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64DFR0_EL1);
 102         u64 mdcr_set = 0;
 103         u64 mdcr_clear = 0;
 104         u64 cptr_set = 0;
 105
 106         /* Trap/constrain PMU */
 107         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), feature_ids)) {
 108                 mdcr_set |= MDCR_EL2_TPM | MDCR_EL2_TPMCR;
 109                 mdcr_clear |= MDCR_EL2_HPME | MDCR_EL2_MTPME |
 110                               MDCR_EL2_HPMN_MASK;
 111         }
 112
 113         /* Trap Debug */
 114         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), feature_ids))
 115                 mdcr_set |= MDCR_EL2_TDRA | MDCR_EL2_TDA | MDCR_EL2_TDE;
 116
 117         /* Trap OS Double Lock */
 118         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DoubleLock), feature_ids))
 119                 mdcr_set |= MDCR_EL2_TDOSA;
 120
 121         /* Trap SPE */
 122         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer), feature_ids)) {
 123                 mdcr_set |= MDCR_EL2_TPMS;
 124                 mdcr_clear |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
 125         }
 126
 127         /* Trap Trace Filter */
 128         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceFilt), feature_ids))
 129                 mdcr_set |= MDCR_EL2_TTRF;
 130
 131         /* Trap Trace */
 132         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids)) {
 133                 if (has_hvhe())
 134                         cptr_set |= CPACR_EL1_TTA;
 135                 else
 136                         cptr_set |= CPTR_EL2_TTA;
 137         }
 138
 139         vcpu->arch.mdcr_el2 |= mdcr_set;
 140         vcpu->arch.mdcr_el2 &= ~mdcr_clear;
 141         vcpu->arch.cptr_el2 |= cptr_set;
 142 }
 143
 144 /*
 145  * Set trap register values based on features in ID_AA64MMFR0.
 146  */
 147 static void pvm_init_traps_aa64mmfr0(struct kvm_vcpu *vcpu)
 148 {
 149         const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64MMFR0_EL1);
 150         u64 mdcr_set = 0;
 151
 152         /* Trap Debug Communications Channel registers */
 153         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_FGT), feature_ids))
 154                 mdcr_set |= MDCR_EL2_TDCC;
 155
 156         vcpu->arch.mdcr_el2 |= mdcr_set;
 157 }
 158
 159 /*
 160  * Set trap register values based on features in ID_AA64MMFR1.
 161  */
 162 static void pvm_init_traps_aa64mmfr1(struct kvm_vcpu *vcpu)
 163 {
 164         const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64MMFR1_EL1);
 165         u64 hcr_set = 0;
 166
 167         /* Trap LOR */
 168         if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_LO), feature_ids))
 169                 hcr_set |= HCR_TLOR;
 170
 171         vcpu->arch.hcr_el2 |= hcr_set;
 172 }
 173
 174 /*
 175  * Set baseline trap register values.
 176  */
 177 static void pvm_init_trap_regs(struct kvm_vcpu *vcpu)
 178 {
 179         const u64 hcr_trap_feat_regs = HCR_TID3;
 180         const u64 hcr_trap_impdef = HCR_TACR | HCR_TIDCP | HCR_TID1;
 181
 182         /*
 183          * Always trap:
 184          * - Feature id registers: to control features exposed to guests
 185          * - Implementation-defined features
 186          */
 187         vcpu->arch.hcr_el2 |= hcr_trap_feat_regs | hcr_trap_impdef;
 188
 189         /* Clear res0 and set res1 bits to trap potential new features. */
 190         vcpu->arch.hcr_el2 &= ~(HCR_RES0);
 191         vcpu->arch.mdcr_el2 &= ~(MDCR_EL2_RES0);
 192         if (!has_hvhe()) {
 193                 vcpu->arch.cptr_el2 |= CPTR_NVHE_EL2_RES1;
 194                 vcpu->arch.cptr_el2 &= ~(CPTR_NVHE_EL2_RES0);
 195         }
 196 }
 197
 198 /*
 199  * Initialize trap register values for protected VMs.
 200  */
 201 void __pkvm_vcpu_init_traps(struct kvm_vcpu *vcpu)
 202 {
 203         pvm_init_trap_regs(vcpu);
 204         pvm_init_traps_aa64pfr0(vcpu);
 205         pvm_init_traps_aa64pfr1(vcpu);
 206         pvm_init_traps_aa64dfr0(vcpu);
 207         pvm_init_traps_aa64mmfr0(vcpu);
 208         pvm_init_traps_aa64mmfr1(vcpu);
 209 }
 210
 211 /*
 212  * Start the VM table handle at the offset defined instead of at 0.
 213  * Mainly for sanity checking and debugging.
 214  */
 215 #define HANDLE_OFFSET 0x1000
 216
 217 static unsigned int vm_handle_to_idx(pkvm_handle_t handle)
 218 {
 219         return handle - HANDLE_OFFSET;
 220 }
 221
 222 static pkvm_handle_t idx_to_vm_handle(unsigned int idx)
 223 {
 224         return idx + HANDLE_OFFSET;
 225 }
 226
 227 /*
 228  * Spinlock for protecting state related to the VM table. Protects writes
 229  * to 'vm_table' and 'nr_table_entries' as well as reads and writes to
 230  * 'last_hyp_vcpu_lookup'.
 231  */
 232 static DEFINE_HYP_SPINLOCK(vm_table_lock);
 233
 234 /*
 235  * The table of VM entries for protected VMs in hyp.
 236  * Allocated at hyp initialization and setup.
 237  */
 238 static struct pkvm_hyp_vm **vm_table;
 239
 240 void pkvm_hyp_vm_table_init(void *tbl)
 241 {
 242         WARN_ON(vm_table);
 243         vm_table = tbl;
 244 }
 245
 246 /*
 247  * Return the hyp vm structure corresponding to the handle.
 248  */
 249 static struct pkvm_hyp_vm *get_vm_by_handle(pkvm_handle_t handle)
 250 {
 251         unsigned int idx = vm_handle_to_idx(handle);
 252
 253         if (unlikely(idx >= KVM_MAX_PVMS))
 254                 return NULL;
 255
 256         return vm_table[idx];
 257 }
 258
 259 struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
 260                                          unsigned int vcpu_idx)
 261 {
 262         struct pkvm_hyp_vcpu *hyp_vcpu = NULL;
 263         struct pkvm_hyp_vm *hyp_vm;
 264
 265         hyp_spin_lock(&vm_table_lock);
 266         hyp_vm = get_vm_by_handle(handle);
 267         if (!hyp_vm || hyp_vm->nr_vcpus <= vcpu_idx)
 268                 goto unlock;
 269
 270         hyp_vcpu = hyp_vm->vcpus[vcpu_idx];
 271         hyp_page_ref_inc(hyp_virt_to_page(hyp_vm));
 272 unlock:
 273         hyp_spin_unlock(&vm_table_lock);
 274         return hyp_vcpu;
 275 }
 276
 277 void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
 278 {
 279         struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
 280
 281         hyp_spin_lock(&vm_table_lock);
 282         hyp_page_ref_dec(hyp_virt_to_page(hyp_vm));
 283         hyp_spin_unlock(&vm_table_lock);
 284 }
 285
 286 static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu)
 287 {
 288         if (host_vcpu)
 289                 hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1);
 290 }
 291
 292 static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[],
 293                              unsigned int nr_vcpus)
 294 {
 295         int i;
 296
 297         for (i = 0; i < nr_vcpus; i++)
 298                 unpin_host_vcpu(hyp_vcpus[i]->host_vcpu);
 299 }
 300
 301 static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
 302                              unsigned int nr_vcpus)
 303 {
 304         hyp_vm->host_kvm = host_kvm;
 305         hyp_vm->kvm.created_vcpus = nr_vcpus;
 306         hyp_vm->kvm.arch.vtcr = host_mmu.arch.vtcr;
 307 }
 308
 309 static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu,
 310                               struct pkvm_hyp_vm *hyp_vm,
 311                               struct kvm_vcpu *host_vcpu,
 312                               unsigned int vcpu_idx)
 313 {
 314         int ret = 0;
 315
 316         if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1))
 317                 return -EBUSY;
 318
 319         if (host_vcpu->vcpu_idx != vcpu_idx) {
 320                 ret = -EINVAL;
 321                 goto done;
 322         }
 323
 324         hyp_vcpu->host_vcpu = host_vcpu;
 325
 326         hyp_vcpu->vcpu.kvm = &hyp_vm->kvm;
 327         hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id);
 328         hyp_vcpu->vcpu.vcpu_idx = vcpu_idx;
 329
 330         hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu;
 331         hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags);
 332 done:
 333         if (ret)
 334                 unpin_host_vcpu(host_vcpu);
 335         return ret;
 336 }
 337
 338 static int find_free_vm_table_entry(struct kvm *host_kvm)
 339 {
 340         int i;
 341
 342         for (i = 0; i < KVM_MAX_PVMS; ++i) {
 343                 if (!vm_table[i])
 344                         return i;
 345         }
 346
 347         return -ENOMEM;
 348 }
 349
 350 /*
 351  * Allocate a VM table entry and insert a pointer to the new vm.
 352  *
 353  * Return a unique handle to the protected VM on success,
 354  * negative error code on failure.
 355  */
 356 static pkvm_handle_t insert_vm_table_entry(struct kvm *host_kvm,
 357                                            struct pkvm_hyp_vm *hyp_vm)
 358 {
 359         struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu;
 360         int idx;
 361
 362         hyp_assert_lock_held(&vm_table_lock);
 363
 364         /*
 365          * Initializing protected state might have failed, yet a malicious
 366          * host could trigger this function. Thus, ensure that 'vm_table'
 367          * exists.
 368          */
 369         if (unlikely(!vm_table))
 370                 return -EINVAL;
 371
 372         idx = find_free_vm_table_entry(host_kvm);
 373         if (idx < 0)
 374                 return idx;
 375
 376         hyp_vm->kvm.arch.pkvm.handle = idx_to_vm_handle(idx);
 377
 378         /* VMID 0 is reserved for the host */
 379         atomic64_set(&mmu->vmid.id, idx + 1);
 380
 381         mmu->arch = &hyp_vm->kvm.arch;
 382         mmu->pgt = &hyp_vm->pgt;
 383
 384         vm_table[idx] = hyp_vm;
 385         return hyp_vm->kvm.arch.pkvm.handle;
 386 }
 387
 388 /*
 389  * Deallocate and remove the VM table entry corresponding to the handle.
 390  */
 391 static void remove_vm_table_entry(pkvm_handle_t handle)
 392 {
 393         hyp_assert_lock_held(&vm_table_lock);
 394         vm_table[vm_handle_to_idx(handle)] = NULL;
 395 }
 396
 397 static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus)
 398 {
 399         return size_add(sizeof(struct pkvm_hyp_vm),
 400                 size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus));
 401 }
 402
 403 static void *map_donated_memory_noclear(unsigned long host_va, size_t size)
 404 {
 405         void *va = (void *)kern_hyp_va(host_va);
 406
 407         if (!PAGE_ALIGNED(va))
 408                 return NULL;
 409
 410         if (__pkvm_host_donate_hyp(hyp_virt_to_pfn(va),
 411                                    PAGE_ALIGN(size) >> PAGE_SHIFT))
 412                 return NULL;
 413
 414         return va;
 415 }
 416
 417 static void *map_donated_memory(unsigned long host_va, size_t size)
 418 {
 419         void *va = map_donated_memory_noclear(host_va, size);
 420
 421         if (va)
 422                 memset(va, 0, size);
 423
 424         return va;
 425 }
 426
 427 static void __unmap_donated_memory(void *va, size_t size)
 428 {
 429         WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(va),
 430                                        PAGE_ALIGN(size) >> PAGE_SHIFT));
 431 }
 432
 433 static void unmap_donated_memory(void *va, size_t size)
 434 {
 435         if (!va)
 436                 return;
 437
 438         memset(va, 0, size);
 439         __unmap_donated_memory(va, size);
 440 }
 441
 442 static void unmap_donated_memory_noclear(void *va, size_t size)
 443 {
 444         if (!va)
 445                 return;
 446
 447         __unmap_donated_memory(va, size);
 448 }
 449
 450 /*
 451  * Initialize the hypervisor copy of the protected VM state using the
 452  * memory donated by the host.
 453  *
 454  * Unmaps the donated memory from the host at stage 2.
 455  *
 456  * host_kvm: A pointer to the host's struct kvm.
 457  * vm_hva: The host va of the area being donated for the VM state.
 458  *         Must be page aligned.
 459  * pgd_hva: The host va of the area being donated for the stage-2 PGD for
 460  *          the VM. Must be page aligned. Its size is implied by the VM's
 461  *          VTCR.
 462  *
 463  * Return a unique handle to the protected VM on success,
 464  * negative error code on failure.
 465  */
 466 int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
 467                    unsigned long pgd_hva)
 468 {
 469         struct pkvm_hyp_vm *hyp_vm = NULL;
 470         size_t vm_size, pgd_size;
 471         unsigned int nr_vcpus;
 472         void *pgd = NULL;
 473         int ret;
 474
 475         ret = hyp_pin_shared_mem(host_kvm, host_kvm + 1);
 476         if (ret)
 477                 return ret;
 478
 479         nr_vcpus = READ_ONCE(host_kvm->created_vcpus);
 480         if (nr_vcpus < 1) {
 481                 ret = -EINVAL;
 482                 goto err_unpin_kvm;
 483         }
 484
 485         vm_size = pkvm_get_hyp_vm_size(nr_vcpus);
 486         pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.vtcr);
 487
 488         ret = -ENOMEM;
 489
 490         hyp_vm = map_donated_memory(vm_hva, vm_size);
 491         if (!hyp_vm)
 492                 goto err_remove_mappings;
 493
 494         pgd = map_donated_memory_noclear(pgd_hva, pgd_size);
 495         if (!pgd)
 496                 goto err_remove_mappings;
 497
 498         init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus);
 499
 500         hyp_spin_lock(&vm_table_lock);
 501         ret = insert_vm_table_entry(host_kvm, hyp_vm);
 502         if (ret < 0)
 503                 goto err_unlock;
 504
 505         ret = kvm_guest_prepare_stage2(hyp_vm, pgd);
 506         if (ret)
 507                 goto err_remove_vm_table_entry;
 508         hyp_spin_unlock(&vm_table_lock);
 509
 510         return hyp_vm->kvm.arch.pkvm.handle;
 511
 512 err_remove_vm_table_entry:
 513         remove_vm_table_entry(hyp_vm->kvm.arch.pkvm.handle);
 514 err_unlock:
 515         hyp_spin_unlock(&vm_table_lock);
 516 err_remove_mappings:
 517         unmap_donated_memory(hyp_vm, vm_size);
 518         unmap_donated_memory(pgd, pgd_size);
 519 err_unpin_kvm:
 520         hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
 521         return ret;
 522 }
 523
 524 /*
 525  * Initialize the hypervisor copy of the protected vCPU state using the
 526  * memory donated by the host.
 527  *
 528  * handle: The handle for the protected vm.
 529  * host_vcpu: A pointer to the corresponding host vcpu.
 530  * vcpu_hva: The host va of the area being donated for the vcpu state.
 531  *           Must be page aligned. The size of the area must be equal to
 532  *           the page-aligned size of 'struct pkvm_hyp_vcpu'.
 533  * Return 0 on success, negative error code on failure.
 534  */
 535 int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
 536                      unsigned long vcpu_hva)
 537 {
 538         struct pkvm_hyp_vcpu *hyp_vcpu;
 539         struct pkvm_hyp_vm *hyp_vm;
 540         unsigned int idx;
 541         int ret;
 542
 543         hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu));
 544         if (!hyp_vcpu)
 545                 return -ENOMEM;
 546
 547         hyp_spin_lock(&vm_table_lock);
 548
 549         hyp_vm = get_vm_by_handle(handle);
 550         if (!hyp_vm) {
 551                 ret = -ENOENT;
 552                 goto unlock;
 553         }
 554
 555         idx = hyp_vm->nr_vcpus;
 556         if (idx >= hyp_vm->kvm.created_vcpus) {
 557                 ret = -EINVAL;
 558                 goto unlock;
 559         }
 560
 561         ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx);
 562         if (ret)
 563                 goto unlock;
 564
 565         hyp_vm->vcpus[idx] = hyp_vcpu;
 566         hyp_vm->nr_vcpus++;
 567 unlock:
 568         hyp_spin_unlock(&vm_table_lock);
 569
 570         if (ret)
 571                 unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu));
 572
 573         return ret;
 574 }
 575
 576 static void
 577 teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size)
 578 {
 579         size = PAGE_ALIGN(size);
 580         memset(addr, 0, size);
 581
 582         for (void *start = addr; start < addr + size; start += PAGE_SIZE)
 583                 push_hyp_memcache(mc, start, hyp_virt_to_phys);
 584
 585         unmap_donated_memory_noclear(addr, size);
 586 }
 587
 588 int __pkvm_teardown_vm(pkvm_handle_t handle)
 589 {
 590         struct kvm_hyp_memcache *mc;
 591         struct pkvm_hyp_vm *hyp_vm;
 592         struct kvm *host_kvm;
 593         unsigned int idx;
 594         size_t vm_size;
 595         int err;
 596
 597         hyp_spin_lock(&vm_table_lock);
 598         hyp_vm = get_vm_by_handle(handle);
 599         if (!hyp_vm) {
 600                 err = -ENOENT;
 601                 goto err_unlock;
 602         }
 603
 604         if (WARN_ON(hyp_page_count(hyp_vm))) {
 605                 err = -EBUSY;
 606                 goto err_unlock;
 607         }
 608
 609         host_kvm = hyp_vm->host_kvm;
 610
 611         /* Ensure the VMID is clean before it can be reallocated */
 612         __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu);
 613         remove_vm_table_entry(handle);
 614         hyp_spin_unlock(&vm_table_lock);
 615
 616         /* Reclaim guest pages (including page-table pages) */
 617         mc = &host_kvm->arch.pkvm.teardown_mc;
 618         reclaim_guest_pages(hyp_vm, mc);
 619         unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus);
 620
 621         /* Push the metadata pages to the teardown memcache */
 622         for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) {
 623                 struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx];
 624
 625                 teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu));
 626         }
 627
 628         vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus);
 629         teardown_donated_memory(mc, hyp_vm, vm_size);
 630         hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
 631         return 0;
 632
 633 err_unlock:
 634         hyp_spin_unlock(&vm_table_lock);
 635         return err;
 636 }