arch/x86/kvm/svm/avic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * AMD SVM support
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *   Avi Kivity   <avi@qumranet.com>
  13  */
  14
  15 #define pr_fmt(fmt) "SVM: " fmt
  16
  17 #include <linux/kvm_types.h>
  18 #include <linux/hashtable.h>
  19 #include <linux/amd-iommu.h>
  20 #include <linux/kvm_host.h>
  21
  22 #include <asm/irq_remapping.h>
  23
  24 #include "trace.h"
  25 #include "lapic.h"
  26 #include "x86.h"
  27 #include "irq.h"
  28 #include "svm.h"
  29
  30 /* AVIC GATAG is encoded using VM and VCPU IDs */
  31 #define AVIC_VCPU_ID_BITS               8
  32 #define AVIC_VCPU_ID_MASK               ((1 << AVIC_VCPU_ID_BITS) - 1)
  33
  34 #define AVIC_VM_ID_BITS                 24
  35 #define AVIC_VM_ID_NR                   (1 << AVIC_VM_ID_BITS)
  36 #define AVIC_VM_ID_MASK                 ((1 << AVIC_VM_ID_BITS) - 1)
  37
  38 #define AVIC_GATAG(x, y)                (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
  39                                                 (y & AVIC_VCPU_ID_MASK))
  40 #define AVIC_GATAG_TO_VMID(x)           ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
  41 #define AVIC_GATAG_TO_VCPUID(x)         (x & AVIC_VCPU_ID_MASK)
  42
  43 /* Note:
  44  * This hash table is used to map VM_ID to a struct kvm_svm,
  45  * when handling AMD IOMMU GALOG notification to schedule in
  46  * a particular vCPU.
  47  */
  48 #define SVM_VM_DATA_HASH_BITS   8
  49 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
  50 static u32 next_vm_id = 0;
  51 static bool next_vm_id_wrapped = 0;
  52 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
  53
  54 /*
  55  * This is a wrapper of struct amd_iommu_ir_data.
  56  */
  57 struct amd_svm_iommu_ir {
  58         struct list_head node;  /* Used by SVM for per-vcpu ir_list */
  59         void *data;             /* Storing pointer to struct amd_ir_data */
  60 };
  61
  62
  63 /* Note:
  64  * This function is called from IOMMU driver to notify
  65  * SVM to schedule in a particular vCPU of a particular VM.
  66  */
  67 int avic_ga_log_notifier(u32 ga_tag)
  68 {
  69         unsigned long flags;
  70         struct kvm_svm *kvm_svm;
  71         struct kvm_vcpu *vcpu = NULL;
  72         u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
  73         u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
  74
  75         pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
  76         trace_kvm_avic_ga_log(vm_id, vcpu_id);
  77
  78         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
  79         hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
  80                 if (kvm_svm->avic_vm_id != vm_id)
  81                         continue;
  82                 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
  83                 break;
  84         }
  85         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
  86
  87         /* Note:
  88          * At this point, the IOMMU should have already set the pending
  89          * bit in the vAPIC backing page. So, we just need to schedule
  90          * in the vcpu.
  91          */
  92         if (vcpu)
  93                 kvm_vcpu_wake_up(vcpu);
  94
  95         return 0;
  96 }
  97
  98 void avic_vm_destroy(struct kvm *kvm)
  99 {
 100         unsigned long flags;
 101         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 102
 103         if (!enable_apicv)
 104                 return;
 105
 106         if (kvm_svm->avic_logical_id_table_page)
 107                 __free_page(kvm_svm->avic_logical_id_table_page);
 108         if (kvm_svm->avic_physical_id_table_page)
 109                 __free_page(kvm_svm->avic_physical_id_table_page);
 110
 111         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
 112         hash_del(&kvm_svm->hnode);
 113         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
 114 }
 115
 116 int avic_vm_init(struct kvm *kvm)
 117 {
 118         unsigned long flags;
 119         int err = -ENOMEM;
 120         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 121         struct kvm_svm *k2;
 122         struct page *p_page;
 123         struct page *l_page;
 124         u32 vm_id;
 125
 126         if (!enable_apicv)
 127                 return 0;
 128
 129         /* Allocating physical APIC ID table (4KB) */
 130         p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 131         if (!p_page)
 132                 goto free_avic;
 133
 134         kvm_svm->avic_physical_id_table_page = p_page;
 135
 136         /* Allocating logical APIC ID table (4KB) */
 137         l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 138         if (!l_page)
 139                 goto free_avic;
 140
 141         kvm_svm->avic_logical_id_table_page = l_page;
 142
 143         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
 144  again:
 145         vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
 146         if (vm_id == 0) { /* id is 1-based, zero is not okay */
 147                 next_vm_id_wrapped = 1;
 148                 goto again;
 149         }
 150         /* Is it still in use? Only possible if wrapped at least once */
 151         if (next_vm_id_wrapped) {
 152                 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
 153                         if (k2->avic_vm_id == vm_id)
 154                                 goto again;
 155                 }
 156         }
 157         kvm_svm->avic_vm_id = vm_id;
 158         hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
 159         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
 160
 161         return 0;
 162
 163 free_avic:
 164         avic_vm_destroy(kvm);
 165         return err;
 166 }
 167
 168 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb)
 169 {
 170         struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
 171         phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
 172         phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
 173         phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
 174
 175         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
 176         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
 177         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
 178         vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
 179         vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK;
 180
 181         if (kvm_apicv_activated(svm->vcpu.kvm))
 182                 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
 183         else
 184                 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
 185 }
 186
 187 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
 188                                        unsigned int index)
 189 {
 190         u64 *avic_physical_id_table;
 191         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
 192
 193         if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
 194                 return NULL;
 195
 196         avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
 197
 198         return &avic_physical_id_table[index];
 199 }
 200
 201 /*
 202  * Note:
 203  * AVIC hardware walks the nested page table to check permissions,
 204  * but does not use the SPA address specified in the leaf page
 205  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
 206  * field of the VMCB. Therefore, we set up the
 207  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
 208  */
 209 static int avic_alloc_access_page(struct kvm *kvm)
 210 {
 211         void __user *ret;
 212         int r = 0;
 213
 214         mutex_lock(&kvm->slots_lock);
 215
 216         if (kvm->arch.apic_access_memslot_enabled)
 217                 goto out;
 218
 219         ret = __x86_set_memory_region(kvm,
 220                                       APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
 221                                       APIC_DEFAULT_PHYS_BASE,
 222                                       PAGE_SIZE);
 223         if (IS_ERR(ret)) {
 224                 r = PTR_ERR(ret);
 225                 goto out;
 226         }
 227
 228         kvm->arch.apic_access_memslot_enabled = true;
 229 out:
 230         mutex_unlock(&kvm->slots_lock);
 231         return r;
 232 }
 233
 234 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
 235 {
 236         u64 *entry, new_entry;
 237         int id = vcpu->vcpu_id;
 238         struct vcpu_svm *svm = to_svm(vcpu);
 239
 240         if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
 241                 return -EINVAL;
 242
 243         if (!vcpu->arch.apic->regs)
 244                 return -EINVAL;
 245
 246         if (kvm_apicv_activated(vcpu->kvm)) {
 247                 int ret;
 248
 249                 ret = avic_alloc_access_page(vcpu->kvm);
 250                 if (ret)
 251                         return ret;
 252         }
 253
 254         svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs);
 255
 256         /* Setting AVIC backing page address in the phy APIC ID table */
 257         entry = avic_get_physical_id_entry(vcpu, id);
 258         if (!entry)
 259                 return -EINVAL;
 260
 261         new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
 262                               AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
 263                               AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
 264         WRITE_ONCE(*entry, new_entry);
 265
 266         svm->avic_physical_id_cache = entry;
 267
 268         return 0;
 269 }
 270
 271 void avic_ring_doorbell(struct kvm_vcpu *vcpu)
 272 {
 273         /*
 274          * Note, the vCPU could get migrated to a different pCPU at any point,
 275          * which could result in signalling the wrong/previous pCPU.  But if
 276          * that happens the vCPU is guaranteed to do a VMRUN (after being
 277          * migrated) and thus will process pending interrupts, i.e. a doorbell
 278          * is not needed (and the spurious one is harmless).
 279          */
 280         int cpu = READ_ONCE(vcpu->cpu);
 281
 282         if (cpu != get_cpu())
 283                 wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu));
 284         put_cpu();
 285 }
 286
 287 /*
 288  * A fast-path version of avic_kick_target_vcpus(), which attempts to match
 289  * destination APIC ID to vCPU without looping through all vCPUs.
 290  */
 291 static int avic_kick_target_vcpus_fast(struct kvm *kvm, struct kvm_lapic *source,
 292                                        u32 icrl, u32 icrh, u32 index)
 293 {
 294         u32 l1_physical_id, dest;
 295         struct kvm_vcpu *target_vcpu;
 296         int dest_mode = icrl & APIC_DEST_MASK;
 297         int shorthand = icrl & APIC_SHORT_MASK;
 298         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 299
 300         if (shorthand != APIC_DEST_NOSHORT)
 301                 return -EINVAL;
 302
 303         if (apic_x2apic_mode(source))
 304                 dest = icrh;
 305         else
 306                 dest = GET_APIC_DEST_FIELD(icrh);
 307
 308         if (dest_mode == APIC_DEST_PHYSICAL) {
 309                 /* broadcast destination, use slow path */
 310                 if (apic_x2apic_mode(source) && dest == X2APIC_BROADCAST)
 311                         return -EINVAL;
 312                 if (!apic_x2apic_mode(source) && dest == APIC_BROADCAST)
 313                         return -EINVAL;
 314
 315                 l1_physical_id = dest;
 316
 317                 if (WARN_ON_ONCE(l1_physical_id != index))
 318                         return -EINVAL;
 319
 320         } else {
 321                 u32 bitmap, cluster;
 322                 int logid_index;
 323
 324                 if (apic_x2apic_mode(source)) {
 325                         /* 16 bit dest mask, 16 bit cluster id */
 326                         bitmap = dest & 0xFFFF0000;
 327                         cluster = (dest >> 16) << 4;
 328                 } else if (kvm_lapic_get_reg(source, APIC_DFR) == APIC_DFR_FLAT) {
 329                         /* 8 bit dest mask*/
 330                         bitmap = dest;
 331                         cluster = 0;
 332                 } else {
 333                         /* 4 bit desk mask, 4 bit cluster id */
 334                         bitmap = dest & 0xF;
 335                         cluster = (dest >> 4) << 2;
 336                 }
 337
 338                 if (unlikely(!bitmap))
 339                         /* guest bug: nobody to send the logical interrupt to */
 340                         return 0;
 341
 342                 if (!is_power_of_2(bitmap))
 343                         /* multiple logical destinations, use slow path */
 344                         return -EINVAL;
 345
 346                 logid_index = cluster + __ffs(bitmap);
 347
 348                 if (apic_x2apic_mode(source)) {
 349                         l1_physical_id = logid_index;
 350                 } else {
 351                         u32 *avic_logical_id_table =
 352                                 page_address(kvm_svm->avic_logical_id_table_page);
 353
 354                         u32 logid_entry = avic_logical_id_table[logid_index];
 355
 356                         if (WARN_ON_ONCE(index != logid_index))
 357                                 return -EINVAL;
 358
 359                         /* guest bug: non existing/reserved logical destination */
 360                         if (unlikely(!(logid_entry & AVIC_LOGICAL_ID_ENTRY_VALID_MASK)))
 361                                 return 0;
 362
 363                         l1_physical_id = logid_entry &
 364                                          AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
 365                 }
 366         }
 367
 368         target_vcpu = kvm_get_vcpu_by_id(kvm, l1_physical_id);
 369         if (unlikely(!target_vcpu))
 370                 /* guest bug: non existing vCPU is a target of this IPI*/
 371                 return 0;
 372
 373         target_vcpu->arch.apic->irr_pending = true;
 374         svm_complete_interrupt_delivery(target_vcpu,
 375                                         icrl & APIC_MODE_MASK,
 376                                         icrl & APIC_INT_LEVELTRIG,
 377                                         icrl & APIC_VECTOR_MASK);
 378         return 0;
 379 }
 380
 381 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source,
 382                                    u32 icrl, u32 icrh, u32 index)
 383 {
 384         unsigned long i;
 385         struct kvm_vcpu *vcpu;
 386
 387         if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index))
 388                 return;
 389
 390         trace_kvm_avic_kick_vcpu_slowpath(icrh, icrl, index);
 391
 392         /*
 393          * Wake any target vCPUs that are blocking, i.e. waiting for a wake
 394          * event.  There's no need to signal doorbells, as hardware has handled
 395          * vCPUs that were in guest at the time of the IPI, and vCPUs that have
 396          * since entered the guest will have processed pending IRQs at VMRUN.
 397          */
 398         kvm_for_each_vcpu(i, vcpu, kvm) {
 399                 if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK,
 400                                         GET_APIC_DEST_FIELD(icrh),
 401                                         icrl & APIC_DEST_MASK)) {
 402                         vcpu->arch.apic->irr_pending = true;
 403                         svm_complete_interrupt_delivery(vcpu,
 404                                                         icrl & APIC_MODE_MASK,
 405                                                         icrl & APIC_INT_LEVELTRIG,
 406                                                         icrl & APIC_VECTOR_MASK);
 407                 }
 408         }
 409 }
 410
 411 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu)
 412 {
 413         struct vcpu_svm *svm = to_svm(vcpu);
 414         u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
 415         u32 icrl = svm->vmcb->control.exit_info_1;
 416         u32 id = svm->vmcb->control.exit_info_2 >> 32;
 417         u32 index = svm->vmcb->control.exit_info_2 & 0x1FF;
 418         struct kvm_lapic *apic = vcpu->arch.apic;
 419
 420         trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index);
 421
 422         switch (id) {
 423         case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
 424                 /*
 425                  * Emulate IPIs that are not handled by AVIC hardware, which
 426                  * only virtualizes Fixed, Edge-Triggered INTRs.  The exit is
 427                  * a trap, e.g. ICR holds the correct value and RIP has been
 428                  * advanced, KVM is responsible only for emulating the IPI.
 429                  * Sadly, hardware may sometimes leave the BUSY flag set, in
 430                  * which case KVM needs to emulate the ICR write as well in
 431                  * order to clear the BUSY flag.
 432                  */
 433                 if (icrl & APIC_ICR_BUSY)
 434                         kvm_apic_write_nodecode(vcpu, APIC_ICR);
 435                 else
 436                         kvm_apic_send_ipi(apic, icrl, icrh);
 437                 break;
 438         case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING:
 439                 /*
 440                  * At this point, we expect that the AVIC HW has already
 441                  * set the appropriate IRR bits on the valid target
 442                  * vcpus. So, we just need to kick the appropriate vcpu.
 443                  */
 444                 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index);
 445                 break;
 446         case AVIC_IPI_FAILURE_INVALID_TARGET:
 447                 break;
 448         case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
 449                 WARN_ONCE(1, "Invalid backing page\n");
 450                 break;
 451         default:
 452                 pr_err("Unknown IPI interception\n");
 453         }
 454
 455         return 1;
 456 }
 457
 458 unsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu)
 459 {
 460         if (is_guest_mode(vcpu))
 461                 return APICV_INHIBIT_REASON_NESTED;
 462         return 0;
 463 }
 464
 465 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
 466 {
 467         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
 468         int index;
 469         u32 *logical_apic_id_table;
 470         int dlid = GET_APIC_LOGICAL_ID(ldr);
 471
 472         if (!dlid)
 473                 return NULL;
 474
 475         if (flat) { /* flat */
 476                 index = ffs(dlid) - 1;
 477                 if (index > 7)
 478                         return NULL;
 479         } else { /* cluster */
 480                 int cluster = (dlid & 0xf0) >> 4;
 481                 int apic = ffs(dlid & 0x0f) - 1;
 482
 483                 if ((apic < 0) || (apic > 7) ||
 484                     (cluster >= 0xf))
 485                         return NULL;
 486                 index = (cluster << 2) + apic;
 487         }
 488
 489         logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
 490
 491         return &logical_apic_id_table[index];
 492 }
 493
 494 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr)
 495 {
 496         bool flat;
 497         u32 *entry, new_entry;
 498
 499         flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
 500         entry = avic_get_logical_id_entry(vcpu, ldr, flat);
 501         if (!entry)
 502                 return -EINVAL;
 503
 504         new_entry = READ_ONCE(*entry);
 505         new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
 506         new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
 507         new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
 508         WRITE_ONCE(*entry, new_entry);
 509
 510         return 0;
 511 }
 512
 513 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu)
 514 {
 515         struct vcpu_svm *svm = to_svm(vcpu);
 516         bool flat = svm->dfr_reg == APIC_DFR_FLAT;
 517         u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat);
 518
 519         if (entry)
 520                 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry);
 521 }
 522
 523 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
 524 {
 525         int ret = 0;
 526         struct vcpu_svm *svm = to_svm(vcpu);
 527         u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
 528         u32 id = kvm_xapic_id(vcpu->arch.apic);
 529
 530         if (ldr == svm->ldr_reg)
 531                 return 0;
 532
 533         avic_invalidate_logical_id_entry(vcpu);
 534
 535         if (ldr)
 536                 ret = avic_ldr_write(vcpu, id, ldr);
 537
 538         if (!ret)
 539                 svm->ldr_reg = ldr;
 540
 541         return ret;
 542 }
 543
 544 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu)
 545 {
 546         struct vcpu_svm *svm = to_svm(vcpu);
 547         u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
 548
 549         if (svm->dfr_reg == dfr)
 550                 return;
 551
 552         avic_invalidate_logical_id_entry(vcpu);
 553         svm->dfr_reg = dfr;
 554 }
 555
 556 static int avic_unaccel_trap_write(struct kvm_vcpu *vcpu)
 557 {
 558         u32 offset = to_svm(vcpu)->vmcb->control.exit_info_1 &
 559                                 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
 560
 561         switch (offset) {
 562         case APIC_LDR:
 563                 if (avic_handle_ldr_update(vcpu))
 564                         return 0;
 565                 break;
 566         case APIC_DFR:
 567                 avic_handle_dfr_update(vcpu);
 568                 break;
 569         default:
 570                 break;
 571         }
 572
 573         kvm_apic_write_nodecode(vcpu, offset);
 574         return 1;
 575 }
 576
 577 static bool is_avic_unaccelerated_access_trap(u32 offset)
 578 {
 579         bool ret = false;
 580
 581         switch (offset) {
 582         case APIC_ID:
 583         case APIC_EOI:
 584         case APIC_RRR:
 585         case APIC_LDR:
 586         case APIC_DFR:
 587         case APIC_SPIV:
 588         case APIC_ESR:
 589         case APIC_ICR:
 590         case APIC_LVTT:
 591         case APIC_LVTTHMR:
 592         case APIC_LVTPC:
 593         case APIC_LVT0:
 594         case APIC_LVT1:
 595         case APIC_LVTERR:
 596         case APIC_TMICT:
 597         case APIC_TDCR:
 598                 ret = true;
 599                 break;
 600         default:
 601                 break;
 602         }
 603         return ret;
 604 }
 605
 606 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu)
 607 {
 608         struct vcpu_svm *svm = to_svm(vcpu);
 609         int ret = 0;
 610         u32 offset = svm->vmcb->control.exit_info_1 &
 611                      AVIC_UNACCEL_ACCESS_OFFSET_MASK;
 612         u32 vector = svm->vmcb->control.exit_info_2 &
 613                      AVIC_UNACCEL_ACCESS_VECTOR_MASK;
 614         bool write = (svm->vmcb->control.exit_info_1 >> 32) &
 615                      AVIC_UNACCEL_ACCESS_WRITE_MASK;
 616         bool trap = is_avic_unaccelerated_access_trap(offset);
 617
 618         trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset,
 619                                             trap, write, vector);
 620         if (trap) {
 621                 /* Handling Trap */
 622                 WARN_ONCE(!write, "svm: Handling trap read.\n");
 623                 ret = avic_unaccel_trap_write(vcpu);
 624         } else {
 625                 /* Handling Fault */
 626                 ret = kvm_emulate_instruction(vcpu, 0);
 627         }
 628
 629         return ret;
 630 }
 631
 632 int avic_init_vcpu(struct vcpu_svm *svm)
 633 {
 634         int ret;
 635         struct kvm_vcpu *vcpu = &svm->vcpu;
 636
 637         if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm))
 638                 return 0;
 639
 640         ret = avic_init_backing_page(vcpu);
 641         if (ret)
 642                 return ret;
 643
 644         INIT_LIST_HEAD(&svm->ir_list);
 645         spin_lock_init(&svm->ir_list_lock);
 646         svm->dfr_reg = APIC_DFR_FLAT;
 647
 648         return ret;
 649 }
 650
 651 void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu)
 652 {
 653         avic_handle_dfr_update(vcpu);
 654         avic_handle_ldr_update(vcpu);
 655 }
 656
 657 static int avic_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate)
 658 {
 659         int ret = 0;
 660         unsigned long flags;
 661         struct amd_svm_iommu_ir *ir;
 662         struct vcpu_svm *svm = to_svm(vcpu);
 663
 664         if (!kvm_arch_has_assigned_device(vcpu->kvm))
 665                 return 0;
 666
 667         /*
 668          * Here, we go through the per-vcpu ir_list to update all existing
 669          * interrupt remapping table entry targeting this vcpu.
 670          */
 671         spin_lock_irqsave(&svm->ir_list_lock, flags);
 672
 673         if (list_empty(&svm->ir_list))
 674                 goto out;
 675
 676         list_for_each_entry(ir, &svm->ir_list, node) {
 677                 if (activate)
 678                         ret = amd_iommu_activate_guest_mode(ir->data);
 679                 else
 680                         ret = amd_iommu_deactivate_guest_mode(ir->data);
 681                 if (ret)
 682                         break;
 683         }
 684 out:
 685         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
 686         return ret;
 687 }
 688
 689 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
 690 {
 691         unsigned long flags;
 692         struct amd_svm_iommu_ir *cur;
 693
 694         spin_lock_irqsave(&svm->ir_list_lock, flags);
 695         list_for_each_entry(cur, &svm->ir_list, node) {
 696                 if (cur->data != pi->ir_data)
 697                         continue;
 698                 list_del(&cur->node);
 699                 kfree(cur);
 700                 break;
 701         }
 702         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
 703 }
 704
 705 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
 706 {
 707         int ret = 0;
 708         unsigned long flags;
 709         struct amd_svm_iommu_ir *ir;
 710
 711         /**
 712          * In some cases, the existing irte is updated and re-set,
 713          * so we need to check here if it's already been * added
 714          * to the ir_list.
 715          */
 716         if (pi->ir_data && (pi->prev_ga_tag != 0)) {
 717                 struct kvm *kvm = svm->vcpu.kvm;
 718                 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
 719                 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
 720                 struct vcpu_svm *prev_svm;
 721
 722                 if (!prev_vcpu) {
 723                         ret = -EINVAL;
 724                         goto out;
 725                 }
 726
 727                 prev_svm = to_svm(prev_vcpu);
 728                 svm_ir_list_del(prev_svm, pi);
 729         }
 730
 731         /**
 732          * Allocating new amd_iommu_pi_data, which will get
 733          * add to the per-vcpu ir_list.
 734          */
 735         ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT);
 736         if (!ir) {
 737                 ret = -ENOMEM;
 738                 goto out;
 739         }
 740         ir->data = pi->ir_data;
 741
 742         spin_lock_irqsave(&svm->ir_list_lock, flags);
 743         list_add(&ir->node, &svm->ir_list);
 744         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
 745 out:
 746         return ret;
 747 }
 748
 749 /*
 750  * Note:
 751  * The HW cannot support posting multicast/broadcast
 752  * interrupts to a vCPU. So, we still use legacy interrupt
 753  * remapping for these kind of interrupts.
 754  *
 755  * For lowest-priority interrupts, we only support
 756  * those with single CPU as the destination, e.g. user
 757  * configures the interrupts via /proc/irq or uses
 758  * irqbalance to make the interrupts single-CPU.
 759  */
 760 static int
 761 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
 762                  struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
 763 {
 764         struct kvm_lapic_irq irq;
 765         struct kvm_vcpu *vcpu = NULL;
 766
 767         kvm_set_msi_irq(kvm, e, &irq);
 768
 769         if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
 770             !kvm_irq_is_postable(&irq)) {
 771                 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
 772                          __func__, irq.vector);
 773                 return -1;
 774         }
 775
 776         pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
 777                  irq.vector);
 778         *svm = to_svm(vcpu);
 779         vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
 780         vcpu_info->vector = irq.vector;
 781
 782         return 0;
 783 }
 784
 785 /*
 786  * avic_pi_update_irte - set IRTE for Posted-Interrupts
 787  *
 788  * @kvm: kvm
 789  * @host_irq: host irq of the interrupt
 790  * @guest_irq: gsi of the interrupt
 791  * @set: set or unset PI
 792  * returns 0 on success, < 0 on failure
 793  */
 794 int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq,
 795                         uint32_t guest_irq, bool set)
 796 {
 797         struct kvm_kernel_irq_routing_entry *e;
 798         struct kvm_irq_routing_table *irq_rt;
 799         int idx, ret = 0;
 800
 801         if (!kvm_arch_has_assigned_device(kvm) ||
 802             !irq_remapping_cap(IRQ_POSTING_CAP))
 803                 return 0;
 804
 805         pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
 806                  __func__, host_irq, guest_irq, set);
 807
 808         idx = srcu_read_lock(&kvm->irq_srcu);
 809         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
 810
 811         if (guest_irq >= irq_rt->nr_rt_entries ||
 812                 hlist_empty(&irq_rt->map[guest_irq])) {
 813                 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
 814                              guest_irq, irq_rt->nr_rt_entries);
 815                 goto out;
 816         }
 817
 818         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
 819                 struct vcpu_data vcpu_info;
 820                 struct vcpu_svm *svm = NULL;
 821
 822                 if (e->type != KVM_IRQ_ROUTING_MSI)
 823                         continue;
 824
 825                 /**
 826                  * Here, we setup with legacy mode in the following cases:
 827                  * 1. When cannot target interrupt to a specific vcpu.
 828                  * 2. Unsetting posted interrupt.
 829                  * 3. APIC virtualization is disabled for the vcpu.
 830                  * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
 831                  */
 832                 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
 833                     kvm_vcpu_apicv_active(&svm->vcpu)) {
 834                         struct amd_iommu_pi_data pi;
 835
 836                         /* Try to enable guest_mode in IRTE */
 837                         pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
 838                                             AVIC_HPA_MASK);
 839                         pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
 840                                                      svm->vcpu.vcpu_id);
 841                         pi.is_guest_mode = true;
 842                         pi.vcpu_data = &vcpu_info;
 843                         ret = irq_set_vcpu_affinity(host_irq, &pi);
 844
 845                         /**
 846                          * Here, we successfully setting up vcpu affinity in
 847                          * IOMMU guest mode. Now, we need to store the posted
 848                          * interrupt information in a per-vcpu ir_list so that
 849                          * we can reference to them directly when we update vcpu
 850                          * scheduling information in IOMMU irte.
 851                          */
 852                         if (!ret && pi.is_guest_mode)
 853                                 svm_ir_list_add(svm, &pi);
 854                 } else {
 855                         /* Use legacy mode in IRTE */
 856                         struct amd_iommu_pi_data pi;
 857
 858                         /**
 859                          * Here, pi is used to:
 860                          * - Tell IOMMU to use legacy mode for this interrupt.
 861                          * - Retrieve ga_tag of prior interrupt remapping data.
 862                          */
 863                         pi.prev_ga_tag = 0;
 864                         pi.is_guest_mode = false;
 865                         ret = irq_set_vcpu_affinity(host_irq, &pi);
 866
 867                         /**
 868                          * Check if the posted interrupt was previously
 869                          * setup with the guest_mode by checking if the ga_tag
 870                          * was cached. If so, we need to clean up the per-vcpu
 871                          * ir_list.
 872                          */
 873                         if (!ret && pi.prev_ga_tag) {
 874                                 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
 875                                 struct kvm_vcpu *vcpu;
 876
 877                                 vcpu = kvm_get_vcpu_by_id(kvm, id);
 878                                 if (vcpu)
 879                                         svm_ir_list_del(to_svm(vcpu), &pi);
 880                         }
 881                 }
 882
 883                 if (!ret && svm) {
 884                         trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
 885                                                  e->gsi, vcpu_info.vector,
 886                                                  vcpu_info.pi_desc_addr, set);
 887                 }
 888
 889                 if (ret < 0) {
 890                         pr_err("%s: failed to update PI IRTE\n", __func__);
 891                         goto out;
 892                 }
 893         }
 894
 895         ret = 0;
 896 out:
 897         srcu_read_unlock(&kvm->irq_srcu, idx);
 898         return ret;
 899 }
 900
 901 bool avic_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason)
 902 {
 903         ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) |
 904                           BIT(APICV_INHIBIT_REASON_ABSENT) |
 905                           BIT(APICV_INHIBIT_REASON_HYPERV) |
 906                           BIT(APICV_INHIBIT_REASON_NESTED) |
 907                           BIT(APICV_INHIBIT_REASON_IRQWIN) |
 908                           BIT(APICV_INHIBIT_REASON_PIT_REINJ) |
 909                           BIT(APICV_INHIBIT_REASON_X2APIC) |
 910                           BIT(APICV_INHIBIT_REASON_BLOCKIRQ) |
 911                           BIT(APICV_INHIBIT_REASON_SEV)      |
 912                           BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) |
 913                           BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED);
 914
 915         return supported & BIT(reason);
 916 }
 917
 918
 919 static inline int
 920 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
 921 {
 922         int ret = 0;
 923         unsigned long flags;
 924         struct amd_svm_iommu_ir *ir;
 925         struct vcpu_svm *svm = to_svm(vcpu);
 926
 927         if (!kvm_arch_has_assigned_device(vcpu->kvm))
 928                 return 0;
 929
 930         /*
 931          * Here, we go through the per-vcpu ir_list to update all existing
 932          * interrupt remapping table entry targeting this vcpu.
 933          */
 934         spin_lock_irqsave(&svm->ir_list_lock, flags);
 935
 936         if (list_empty(&svm->ir_list))
 937                 goto out;
 938
 939         list_for_each_entry(ir, &svm->ir_list, node) {
 940                 ret = amd_iommu_update_ga(cpu, r, ir->data);
 941                 if (ret)
 942                         break;
 943         }
 944 out:
 945         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
 946         return ret;
 947 }
 948
 949 void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 950 {
 951         u64 entry;
 952         int h_physical_id = kvm_cpu_get_apicid(cpu);
 953         struct vcpu_svm *svm = to_svm(vcpu);
 954
 955         lockdep_assert_preemption_disabled();
 956
 957         if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
 958                 return;
 959
 960         /*
 961          * No need to update anything if the vCPU is blocking, i.e. if the vCPU
 962          * is being scheduled in after being preempted.  The CPU entries in the
 963          * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'.
 964          * If the vCPU was migrated, its new CPU value will be stuffed when the
 965          * vCPU unblocks.
 966          */
 967         if (kvm_vcpu_is_blocking(vcpu))
 968                 return;
 969
 970         entry = READ_ONCE(*(svm->avic_physical_id_cache));
 971         WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
 972
 973         entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
 974         entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
 975         entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
 976
 977         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
 978         avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true);
 979 }
 980
 981 void avic_vcpu_put(struct kvm_vcpu *vcpu)
 982 {
 983         u64 entry;
 984         struct vcpu_svm *svm = to_svm(vcpu);
 985
 986         lockdep_assert_preemption_disabled();
 987
 988         entry = READ_ONCE(*(svm->avic_physical_id_cache));
 989
 990         /* Nothing to do if IsRunning == '0' due to vCPU blocking. */
 991         if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK))
 992                 return;
 993
 994         avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
 995
 996         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
 997         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
 998 }
 999
1000
1001 void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
1002 {
1003         struct vcpu_svm *svm = to_svm(vcpu);
1004         struct vmcb *vmcb = svm->vmcb01.ptr;
1005         bool activated = kvm_vcpu_apicv_active(vcpu);
1006
1007         if (!enable_apicv)
1008                 return;
1009
1010         if (activated) {
1011                 /**
1012                  * During AVIC temporary deactivation, guest could update
1013                  * APIC ID, DFR and LDR registers, which would not be trapped
1014                  * by avic_unaccelerated_access_interception(). In this case,
1015                  * we need to check and update the AVIC logical APIC ID table
1016                  * accordingly before re-activating.
1017                  */
1018                 avic_apicv_post_state_restore(vcpu);
1019                 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1020         } else {
1021                 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
1022         }
1023         vmcb_mark_dirty(vmcb, VMCB_AVIC);
1024
1025         if (activated)
1026                 avic_vcpu_load(vcpu, vcpu->cpu);
1027         else
1028                 avic_vcpu_put(vcpu);
1029
1030         avic_set_pi_irte_mode(vcpu, activated);
1031 }
1032
1033 void avic_vcpu_blocking(struct kvm_vcpu *vcpu)
1034 {
1035         if (!kvm_vcpu_apicv_active(vcpu))
1036                 return;
1037
1038        /*
1039         * Unload the AVIC when the vCPU is about to block, _before_
1040         * the vCPU actually blocks.
1041         *
1042         * Any IRQs that arrive before IsRunning=0 will not cause an
1043         * incomplete IPI vmexit on the source, therefore vIRR will also
1044         * be checked by kvm_vcpu_check_block() before blocking.  The
1045         * memory barrier implicit in set_current_state orders writing
1046         * IsRunning=0 before reading the vIRR.  The processor needs a
1047         * matching memory barrier on interrupt delivery between writing
1048         * IRR and reading IsRunning; the lack of this barrier might be
1049         * the cause of errata #1235).
1050         */
1051         avic_vcpu_put(vcpu);
1052 }
1053
1054 void avic_vcpu_unblocking(struct kvm_vcpu *vcpu)
1055 {
1056         if (!kvm_vcpu_apicv_active(vcpu))
1057                 return;
1058
1059         avic_vcpu_load(vcpu, vcpu->cpu);
1060 }