arch/x86/kvm/svm.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * AMD SVM support
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  *
   9  * Authors:
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) "SVM: " fmt
  19
  20 #include <linux/kvm_host.h>
  21
  22 #include "irq.h"
  23 #include "mmu.h"
  24 #include "kvm_cache_regs.h"
  25 #include "x86.h"
  26 #include "cpuid.h"
  27 #include "pmu.h"
  28
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/kernel.h>
  32 #include <linux/vmalloc.h>
  33 #include <linux/highmem.h>
  34 #include <linux/sched.h>
  35 #include <linux/trace_events.h>
  36 #include <linux/slab.h>
  37 #include <linux/amd-iommu.h>
  38 #include <linux/hashtable.h>
  39 #include <linux/frame.h>
  40 #include <linux/psp-sev.h>
  41 #include <linux/file.h>
  42 #include <linux/pagemap.h>
  43 #include <linux/swap.h>
  44
  45 #include <asm/apic.h>
  46 #include <asm/perf_event.h>
  47 #include <asm/tlbflush.h>
  48 #include <asm/desc.h>
  49 #include <asm/debugreg.h>
  50 #include <asm/kvm_para.h>
  51 #include <asm/irq_remapping.h>
  52 #include <asm/microcode.h>
  53 #include <asm/nospec-branch.h>
  54
  55 #include <asm/virtext.h>
  56 #include "trace.h"
  57
  58 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  59
  60 MODULE_AUTHOR("Qumranet");
  61 MODULE_LICENSE("GPL");
  62
  63 static const struct x86_cpu_id svm_cpu_id[] = {
  64         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  65         {}
  66 };
  67 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  68
  69 #define IOPM_ALLOC_ORDER 2
  70 #define MSRPM_ALLOC_ORDER 1
  71
  72 #define SEG_TYPE_LDT 2
  73 #define SEG_TYPE_BUSY_TSS16 3
  74
  75 #define SVM_FEATURE_NPT            (1 <<  0)
  76 #define SVM_FEATURE_LBRV           (1 <<  1)
  77 #define SVM_FEATURE_SVML           (1 <<  2)
  78 #define SVM_FEATURE_NRIP           (1 <<  3)
  79 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  80 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  81 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  82 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  83 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  84
  85 #define SVM_AVIC_DOORBELL       0xc001011b
  86
  87 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  88 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  89 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  90
  91 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  92
  93 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  94 #define TSC_RATIO_MIN           0x0000000000000001ULL
  95 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  96
  97 #define AVIC_HPA_MASK   ~((0xFFFULL << 52) | 0xFFF)
  98
  99 /*
 100  * 0xff is broadcast, so the max index allowed for physical APIC ID
 101  * table is 0xfe.  APIC IDs above 0xff are reserved.
 102  */
 103 #define AVIC_MAX_PHYSICAL_ID_COUNT      255
 104
 105 #define AVIC_UNACCEL_ACCESS_WRITE_MASK          1
 106 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK         0xFF0
 107 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK         0xFFFFFFFF
 108
 109 /* AVIC GATAG is encoded using VM and VCPU IDs */
 110 #define AVIC_VCPU_ID_BITS               8
 111 #define AVIC_VCPU_ID_MASK               ((1 << AVIC_VCPU_ID_BITS) - 1)
 112
 113 #define AVIC_VM_ID_BITS                 24
 114 #define AVIC_VM_ID_NR                   (1 << AVIC_VM_ID_BITS)
 115 #define AVIC_VM_ID_MASK                 ((1 << AVIC_VM_ID_BITS) - 1)
 116
 117 #define AVIC_GATAG(x, y)                (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
 118                                                 (y & AVIC_VCPU_ID_MASK))
 119 #define AVIC_GATAG_TO_VMID(x)           ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
 120 #define AVIC_GATAG_TO_VCPUID(x)         (x & AVIC_VCPU_ID_MASK)
 121
 122 static bool erratum_383_found __read_mostly;
 123
 124 static const u32 host_save_user_msrs[] = {
 125 #ifdef CONFIG_X86_64
 126         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
 127         MSR_FS_BASE,
 128 #endif
 129         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 130         MSR_TSC_AUX,
 131 };
 132
 133 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
 134
 135 struct kvm_vcpu;
 136
 137 struct nested_state {
 138         struct vmcb *hsave;
 139         u64 hsave_msr;
 140         u64 vm_cr_msr;
 141         u64 vmcb;
 142
 143         /* These are the merged vectors */
 144         u32 *msrpm;
 145
 146         /* gpa pointers to the real vectors */
 147         u64 vmcb_msrpm;
 148         u64 vmcb_iopm;
 149
 150         /* A VMEXIT is required but not yet emulated */
 151         bool exit_required;
 152
 153         /* cache for intercepts of the guest */
 154         u32 intercept_cr;
 155         u32 intercept_dr;
 156         u32 intercept_exceptions;
 157         u64 intercept;
 158
 159         /* Nested Paging related state */
 160         u64 nested_cr3;
 161 };
 162
 163 #define MSRPM_OFFSETS   16
 164 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 165
 166 /*
 167  * Set osvw_len to higher value when updated Revision Guides
 168  * are published and we know what the new status bits are
 169  */
 170 static uint64_t osvw_len = 4, osvw_status;
 171
 172 struct vcpu_svm {
 173         struct kvm_vcpu vcpu;
 174         struct vmcb *vmcb;
 175         unsigned long vmcb_pa;
 176         struct svm_cpu_data *svm_data;
 177         uint64_t asid_generation;
 178         uint64_t sysenter_esp;
 179         uint64_t sysenter_eip;
 180         uint64_t tsc_aux;
 181
 182         u64 msr_decfg;
 183
 184         u64 next_rip;
 185
 186         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 187         struct {
 188                 u16 fs;
 189                 u16 gs;
 190                 u16 ldt;
 191                 u64 gs_base;
 192         } host;
 193
 194         u64 spec_ctrl;
 195
 196         u32 *msrpm;
 197
 198         ulong nmi_iret_rip;
 199
 200         struct nested_state nested;
 201
 202         bool nmi_singlestep;
 203         u64 nmi_singlestep_guest_rflags;
 204
 205         unsigned int3_injected;
 206         unsigned long int3_rip;
 207
 208         /* cached guest cpuid flags for faster access */
 209         bool nrips_enabled      : 1;
 210
 211         u32 ldr_reg;
 212         struct page *avic_backing_page;
 213         u64 *avic_physical_id_cache;
 214         bool avic_is_running;
 215
 216         /*
 217          * Per-vcpu list of struct amd_svm_iommu_ir:
 218          * This is used mainly to store interrupt remapping information used
 219          * when update the vcpu affinity. This avoids the need to scan for
 220          * IRTE and try to match ga_tag in the IOMMU driver.
 221          */
 222         struct list_head ir_list;
 223         spinlock_t ir_list_lock;
 224
 225         /* which host CPU was used for running this vcpu */
 226         unsigned int last_cpu;
 227 };
 228
 229 /*
 230  * This is a wrapper of struct amd_iommu_ir_data.
 231  */
 232 struct amd_svm_iommu_ir {
 233         struct list_head node;  /* Used by SVM for per-vcpu ir_list */
 234         void *data;             /* Storing pointer to struct amd_ir_data */
 235 };
 236
 237 #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK    (0xFF)
 238 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK                (1 << 31)
 239
 240 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK    (0xFFULL)
 241 #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK        (0xFFFFFFFFFFULL << 12)
 242 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK          (1ULL << 62)
 243 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK               (1ULL << 63)
 244
 245 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 246 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 247
 248 #define MSR_INVALID                     0xffffffffU
 249
 250 static const struct svm_direct_access_msrs {
 251         u32 index;   /* Index of the MSR */
 252         bool always; /* True if intercept is always on */
 253 } direct_access_msrs[] = {
 254         { .index = MSR_STAR,                            .always = true  },
 255         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 256 #ifdef CONFIG_X86_64
 257         { .index = MSR_GS_BASE,                         .always = true  },
 258         { .index = MSR_FS_BASE,                         .always = true  },
 259         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 260         { .index = MSR_LSTAR,                           .always = true  },
 261         { .index = MSR_CSTAR,                           .always = true  },
 262         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 263 #endif
 264         { .index = MSR_IA32_SPEC_CTRL,                  .always = false },
 265         { .index = MSR_IA32_PRED_CMD,                   .always = false },
 266         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 267         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 268         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 269         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 270         { .index = MSR_INVALID,                         .always = false },
 271 };
 272
 273 /* enable NPT for AMD64 and X86 with PAE */
 274 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 275 static bool npt_enabled = true;
 276 #else
 277 static bool npt_enabled;
 278 #endif
 279
 280 /* allow nested paging (virtualized MMU) for all guests */
 281 static int npt = true;
 282 module_param(npt, int, S_IRUGO);
 283
 284 /* allow nested virtualization in KVM/SVM */
 285 static int nested = true;
 286 module_param(nested, int, S_IRUGO);
 287
 288 /* enable / disable AVIC */
 289 static int avic;
 290 #ifdef CONFIG_X86_LOCAL_APIC
 291 module_param(avic, int, S_IRUGO);
 292 #endif
 293
 294 /* enable/disable Virtual VMLOAD VMSAVE */
 295 static int vls = true;
 296 module_param(vls, int, 0444);
 297
 298 /* enable/disable Virtual GIF */
 299 static int vgif = true;
 300 module_param(vgif, int, 0444);
 301
 302 /* enable/disable SEV support */
 303 static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
 304 module_param(sev, int, 0444);
 305
 306 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 307
 308 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 309 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
 310 static void svm_complete_interrupts(struct vcpu_svm *svm);
 311
 312 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 313 static int nested_svm_intercept(struct vcpu_svm *svm);
 314 static int nested_svm_vmexit(struct vcpu_svm *svm);
 315 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 316                                       bool has_error_code, u32 error_code);
 317
 318 enum {
 319         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 320                             pause filter count */
 321         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 322         VMCB_ASID,       /* ASID */
 323         VMCB_INTR,       /* int_ctl, int_vector */
 324         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 325         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 326         VMCB_DR,         /* DR6, DR7 */
 327         VMCB_DT,         /* GDT, IDT */
 328         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 329         VMCB_CR2,        /* CR2 only */
 330         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 331         VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
 332                           * AVIC PHYSICAL_TABLE pointer,
 333                           * AVIC LOGICAL_TABLE pointer
 334                           */
 335         VMCB_DIRTY_MAX,
 336 };
 337
 338 /* TPR and CR2 are always written before VMRUN */
 339 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 340
 341 #define VMCB_AVIC_APIC_BAR_MASK         0xFFFFFFFFFF000ULL
 342
 343 static unsigned int max_sev_asid;
 344 static unsigned int min_sev_asid;
 345 static unsigned long *sev_asid_bitmap;
 346 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
 347
 348 struct enc_region {
 349         struct list_head list;
 350         unsigned long npages;
 351         struct page **pages;
 352         unsigned long uaddr;
 353         unsigned long size;
 354 };
 355
 356 static inline bool svm_sev_enabled(void)
 357 {
 358         return max_sev_asid;
 359 }
 360
 361 static inline bool sev_guest(struct kvm *kvm)
 362 {
 363         struct kvm_sev_info *sev = &kvm->arch.sev_info;
 364
 365         return sev->active;
 366 }
 367
 368 static inline int sev_get_asid(struct kvm *kvm)
 369 {
 370         struct kvm_sev_info *sev = &kvm->arch.sev_info;
 371
 372         return sev->asid;
 373 }
 374
 375 static inline void mark_all_dirty(struct vmcb *vmcb)
 376 {
 377         vmcb->control.clean = 0;
 378 }
 379
 380 static inline void mark_all_clean(struct vmcb *vmcb)
 381 {
 382         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 383                                & ~VMCB_ALWAYS_DIRTY_MASK;
 384 }
 385
 386 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 387 {
 388         vmcb->control.clean &= ~(1 << bit);
 389 }
 390
 391 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 392 {
 393         return container_of(vcpu, struct vcpu_svm, vcpu);
 394 }
 395
 396 static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
 397 {
 398         svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
 399         mark_dirty(svm->vmcb, VMCB_AVIC);
 400 }
 401
 402 static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
 403 {
 404         struct vcpu_svm *svm = to_svm(vcpu);
 405         u64 *entry = svm->avic_physical_id_cache;
 406
 407         if (!entry)
 408                 return false;
 409
 410         return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
 411 }
 412
 413 static void recalc_intercepts(struct vcpu_svm *svm)
 414 {
 415         struct vmcb_control_area *c, *h;
 416         struct nested_state *g;
 417
 418         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 419
 420         if (!is_guest_mode(&svm->vcpu))
 421                 return;
 422
 423         c = &svm->vmcb->control;
 424         h = &svm->nested.hsave->control;
 425         g = &svm->nested;
 426
 427         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 428         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 429         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 430         c->intercept = h->intercept | g->intercept;
 431 }
 432
 433 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 434 {
 435         if (is_guest_mode(&svm->vcpu))
 436                 return svm->nested.hsave;
 437         else
 438                 return svm->vmcb;
 439 }
 440
 441 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 442 {
 443         struct vmcb *vmcb = get_host_vmcb(svm);
 444
 445         vmcb->control.intercept_cr |= (1U << bit);
 446
 447         recalc_intercepts(svm);
 448 }
 449
 450 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 451 {
 452         struct vmcb *vmcb = get_host_vmcb(svm);
 453
 454         vmcb->control.intercept_cr &= ~(1U << bit);
 455
 456         recalc_intercepts(svm);
 457 }
 458
 459 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 460 {
 461         struct vmcb *vmcb = get_host_vmcb(svm);
 462
 463         return vmcb->control.intercept_cr & (1U << bit);
 464 }
 465
 466 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 467 {
 468         struct vmcb *vmcb = get_host_vmcb(svm);
 469
 470         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 471                 | (1 << INTERCEPT_DR1_READ)
 472                 | (1 << INTERCEPT_DR2_READ)
 473                 | (1 << INTERCEPT_DR3_READ)
 474                 | (1 << INTERCEPT_DR4_READ)
 475                 | (1 << INTERCEPT_DR5_READ)
 476                 | (1 << INTERCEPT_DR6_READ)
 477                 | (1 << INTERCEPT_DR7_READ)
 478                 | (1 << INTERCEPT_DR0_WRITE)
 479                 | (1 << INTERCEPT_DR1_WRITE)
 480                 | (1 << INTERCEPT_DR2_WRITE)
 481                 | (1 << INTERCEPT_DR3_WRITE)
 482                 | (1 << INTERCEPT_DR4_WRITE)
 483                 | (1 << INTERCEPT_DR5_WRITE)
 484                 | (1 << INTERCEPT_DR6_WRITE)
 485                 | (1 << INTERCEPT_DR7_WRITE);
 486
 487         recalc_intercepts(svm);
 488 }
 489
 490 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 491 {
 492         struct vmcb *vmcb = get_host_vmcb(svm);
 493
 494         vmcb->control.intercept_dr = 0;
 495
 496         recalc_intercepts(svm);
 497 }
 498
 499 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 500 {
 501         struct vmcb *vmcb = get_host_vmcb(svm);
 502
 503         vmcb->control.intercept_exceptions |= (1U << bit);
 504
 505         recalc_intercepts(svm);
 506 }
 507
 508 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 509 {
 510         struct vmcb *vmcb = get_host_vmcb(svm);
 511
 512         vmcb->control.intercept_exceptions &= ~(1U << bit);
 513
 514         recalc_intercepts(svm);
 515 }
 516
 517 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 518 {
 519         struct vmcb *vmcb = get_host_vmcb(svm);
 520
 521         vmcb->control.intercept |= (1ULL << bit);
 522
 523         recalc_intercepts(svm);
 524 }
 525
 526 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 527 {
 528         struct vmcb *vmcb = get_host_vmcb(svm);
 529
 530         vmcb->control.intercept &= ~(1ULL << bit);
 531
 532         recalc_intercepts(svm);
 533 }
 534
 535 static inline bool vgif_enabled(struct vcpu_svm *svm)
 536 {
 537         return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK);
 538 }
 539
 540 static inline void enable_gif(struct vcpu_svm *svm)
 541 {
 542         if (vgif_enabled(svm))
 543                 svm->vmcb->control.int_ctl |= V_GIF_MASK;
 544         else
 545                 svm->vcpu.arch.hflags |= HF_GIF_MASK;
 546 }
 547
 548 static inline void disable_gif(struct vcpu_svm *svm)
 549 {
 550         if (vgif_enabled(svm))
 551                 svm->vmcb->control.int_ctl &= ~V_GIF_MASK;
 552         else
 553                 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 554 }
 555
 556 static inline bool gif_set(struct vcpu_svm *svm)
 557 {
 558         if (vgif_enabled(svm))
 559                 return !!(svm->vmcb->control.int_ctl & V_GIF_MASK);
 560         else
 561                 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 562 }
 563
 564 static unsigned long iopm_base;
 565
 566 struct kvm_ldttss_desc {
 567         u16 limit0;
 568         u16 base0;
 569         unsigned base1:8, type:5, dpl:2, p:1;
 570         unsigned limit1:4, zero0:3, g:1, base2:8;
 571         u32 base3;
 572         u32 zero1;
 573 } __attribute__((packed));
 574
 575 struct svm_cpu_data {
 576         int cpu;
 577
 578         u64 asid_generation;
 579         u32 max_asid;
 580         u32 next_asid;
 581         u32 min_asid;
 582         struct kvm_ldttss_desc *tss_desc;
 583
 584         struct page *save_area;
 585         struct vmcb *current_vmcb;
 586
 587         /* index = sev_asid, value = vmcb pointer */
 588         struct vmcb **sev_vmcbs;
 589 };
 590
 591 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 592
 593 struct svm_init_data {
 594         int cpu;
 595         int r;
 596 };
 597
 598 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 599
 600 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 601 #define MSRS_RANGE_SIZE 2048
 602 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 603
 604 static u32 svm_msrpm_offset(u32 msr)
 605 {
 606         u32 offset;
 607         int i;
 608
 609         for (i = 0; i < NUM_MSR_MAPS; i++) {
 610                 if (msr < msrpm_ranges[i] ||
 611                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 612                         continue;
 613
 614                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 615                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 616
 617                 /* Now we have the u8 offset - but need the u32 offset */
 618                 return offset / 4;
 619         }
 620
 621         /* MSR not in any range */
 622         return MSR_INVALID;
 623 }
 624
 625 #define MAX_INST_SIZE 15
 626
 627 static inline void clgi(void)
 628 {
 629         asm volatile (__ex(SVM_CLGI));
 630 }
 631
 632 static inline void stgi(void)
 633 {
 634         asm volatile (__ex(SVM_STGI));
 635 }
 636
 637 static inline void invlpga(unsigned long addr, u32 asid)
 638 {
 639         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 640 }
 641
 642 static int get_npt_level(struct kvm_vcpu *vcpu)
 643 {
 644 #ifdef CONFIG_X86_64
 645         return PT64_ROOT_4LEVEL;
 646 #else
 647         return PT32E_ROOT_LEVEL;
 648 #endif
 649 }
 650
 651 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 652 {
 653         vcpu->arch.efer = efer;
 654         if (!npt_enabled && !(efer & EFER_LMA))
 655                 efer &= ~EFER_LME;
 656
 657         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 658         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 659 }
 660
 661 static int is_external_interrupt(u32 info)
 662 {
 663         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 664         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 665 }
 666
 667 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 668 {
 669         struct vcpu_svm *svm = to_svm(vcpu);
 670         u32 ret = 0;
 671
 672         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 673                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 674         return ret;
 675 }
 676
 677 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 678 {
 679         struct vcpu_svm *svm = to_svm(vcpu);
 680
 681         if (mask == 0)
 682                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 683         else
 684                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 685
 686 }
 687
 688 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 689 {
 690         struct vcpu_svm *svm = to_svm(vcpu);
 691
 692         if (svm->vmcb->control.next_rip != 0) {
 693                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 694                 svm->next_rip = svm->vmcb->control.next_rip;
 695         }
 696
 697         if (!svm->next_rip) {
 698                 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
 699                                 EMULATE_DONE)
 700                         printk(KERN_DEBUG "%s: NOP\n", __func__);
 701                 return;
 702         }
 703         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 704                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 705                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 706
 707         kvm_rip_write(vcpu, svm->next_rip);
 708         svm_set_interrupt_shadow(vcpu, 0);
 709 }
 710
 711 static void svm_queue_exception(struct kvm_vcpu *vcpu)
 712 {
 713         struct vcpu_svm *svm = to_svm(vcpu);
 714         unsigned nr = vcpu->arch.exception.nr;
 715         bool has_error_code = vcpu->arch.exception.has_error_code;
 716         bool reinject = vcpu->arch.exception.injected;
 717         u32 error_code = vcpu->arch.exception.error_code;
 718
 719         /*
 720          * If we are within a nested VM we'd better #VMEXIT and let the guest
 721          * handle the exception
 722          */
 723         if (!reinject &&
 724             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 725                 return;
 726
 727         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
 728                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 729
 730                 /*
 731                  * For guest debugging where we have to reinject #BP if some
 732                  * INT3 is guest-owned:
 733                  * Emulate nRIP by moving RIP forward. Will fail if injection
 734                  * raises a fault that is not intercepted. Still better than
 735                  * failing in all cases.
 736                  */
 737                 skip_emulated_instruction(&svm->vcpu);
 738                 rip = kvm_rip_read(&svm->vcpu);
 739                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 740                 svm->int3_injected = rip - old_rip;
 741         }
 742
 743         svm->vmcb->control.event_inj = nr
 744                 | SVM_EVTINJ_VALID
 745                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 746                 | SVM_EVTINJ_TYPE_EXEPT;
 747         svm->vmcb->control.event_inj_err = error_code;
 748 }
 749
 750 static void svm_init_erratum_383(void)
 751 {
 752         u32 low, high;
 753         int err;
 754         u64 val;
 755
 756         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 757                 return;
 758
 759         /* Use _safe variants to not break nested virtualization */
 760         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 761         if (err)
 762                 return;
 763
 764         val |= (1ULL << 47);
 765
 766         low  = lower_32_bits(val);
 767         high = upper_32_bits(val);
 768
 769         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 770
 771         erratum_383_found = true;
 772 }
 773
 774 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 775 {
 776         /*
 777          * Guests should see errata 400 and 415 as fixed (assuming that
 778          * HLT and IO instructions are intercepted).
 779          */
 780         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 781         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 782
 783         /*
 784          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 785          * all osvw.status bits inside that length, including bit 0 (which is
 786          * reserved for erratum 298), are valid. However, if host processor's
 787          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 788          * be conservative here and therefore we tell the guest that erratum 298
 789          * is present (because we really don't know).
 790          */
 791         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 792                 vcpu->arch.osvw.status |= 1;
 793 }
 794
 795 static int has_svm(void)
 796 {
 797         const char *msg;
 798
 799         if (!cpu_has_svm(&msg)) {
 800                 printk(KERN_INFO "has_svm: %s\n", msg);
 801                 return 0;
 802         }
 803
 804         return 1;
 805 }
 806
 807 static void svm_hardware_disable(void)
 808 {
 809         /* Make sure we clean up behind us */
 810         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 811                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 812
 813         cpu_svm_disable();
 814
 815         amd_pmu_disable_virt();
 816 }
 817
 818 static int svm_hardware_enable(void)
 819 {
 820
 821         struct svm_cpu_data *sd;
 822         uint64_t efer;
 823         struct desc_struct *gdt;
 824         int me = raw_smp_processor_id();
 825
 826         rdmsrl(MSR_EFER, efer);
 827         if (efer & EFER_SVME)
 828                 return -EBUSY;
 829
 830         if (!has_svm()) {
 831                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 832                 return -EINVAL;
 833         }
 834         sd = per_cpu(svm_data, me);
 835         if (!sd) {
 836                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 837                 return -EINVAL;
 838         }
 839
 840         sd->asid_generation = 1;
 841         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 842         sd->next_asid = sd->max_asid + 1;
 843         sd->min_asid = max_sev_asid + 1;
 844
 845         gdt = get_current_gdt_rw();
 846         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 847
 848         wrmsrl(MSR_EFER, efer | EFER_SVME);
 849
 850         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 851
 852         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 853                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 854                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 855         }
 856
 857
 858         /*
 859          * Get OSVW bits.
 860          *
 861          * Note that it is possible to have a system with mixed processor
 862          * revisions and therefore different OSVW bits. If bits are not the same
 863          * on different processors then choose the worst case (i.e. if erratum
 864          * is present on one processor and not on another then assume that the
 865          * erratum is present everywhere).
 866          */
 867         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 868                 uint64_t len, status = 0;
 869                 int err;
 870
 871                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 872                 if (!err)
 873                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 874                                                       &err);
 875
 876                 if (err)
 877                         osvw_status = osvw_len = 0;
 878                 else {
 879                         if (len < osvw_len)
 880                                 osvw_len = len;
 881                         osvw_status |= status;
 882                         osvw_status &= (1ULL << osvw_len) - 1;
 883                 }
 884         } else
 885                 osvw_status = osvw_len = 0;
 886
 887         svm_init_erratum_383();
 888
 889         amd_pmu_enable_virt();
 890
 891         return 0;
 892 }
 893
 894 static void svm_cpu_uninit(int cpu)
 895 {
 896         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 897
 898         if (!sd)
 899                 return;
 900
 901         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 902         kfree(sd->sev_vmcbs);
 903         __free_page(sd->save_area);
 904         kfree(sd);
 905 }
 906
 907 static int svm_cpu_init(int cpu)
 908 {
 909         struct svm_cpu_data *sd;
 910         int r;
 911
 912         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 913         if (!sd)
 914                 return -ENOMEM;
 915         sd->cpu = cpu;
 916         r = -ENOMEM;
 917         sd->save_area = alloc_page(GFP_KERNEL);
 918         if (!sd->save_area)
 919                 goto err_1;
 920
 921         if (svm_sev_enabled()) {
 922                 r = -ENOMEM;
 923                 sd->sev_vmcbs = kmalloc((max_sev_asid + 1) * sizeof(void *), GFP_KERNEL);
 924                 if (!sd->sev_vmcbs)
 925                         goto err_1;
 926         }
 927
 928         per_cpu(svm_data, cpu) = sd;
 929
 930         return 0;
 931
 932 err_1:
 933         kfree(sd);
 934         return r;
 935
 936 }
 937
 938 static bool valid_msr_intercept(u32 index)
 939 {
 940         int i;
 941
 942         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 943                 if (direct_access_msrs[i].index == index)
 944                         return true;
 945
 946         return false;
 947 }
 948
 949 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
 950 {
 951         u8 bit_write;
 952         unsigned long tmp;
 953         u32 offset;
 954         u32 *msrpm;
 955
 956         msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
 957                                       to_svm(vcpu)->msrpm;
 958
 959         offset    = svm_msrpm_offset(msr);
 960         bit_write = 2 * (msr & 0x0f) + 1;
 961         tmp       = msrpm[offset];
 962
 963         BUG_ON(offset == MSR_INVALID);
 964
 965         return !!test_bit(bit_write,  &tmp);
 966 }
 967
 968 static void set_msr_interception(u32 *msrpm, unsigned msr,
 969                                  int read, int write)
 970 {
 971         u8 bit_read, bit_write;
 972         unsigned long tmp;
 973         u32 offset;
 974
 975         /*
 976          * If this warning triggers extend the direct_access_msrs list at the
 977          * beginning of the file
 978          */
 979         WARN_ON(!valid_msr_intercept(msr));
 980
 981         offset    = svm_msrpm_offset(msr);
 982         bit_read  = 2 * (msr & 0x0f);
 983         bit_write = 2 * (msr & 0x0f) + 1;
 984         tmp       = msrpm[offset];
 985
 986         BUG_ON(offset == MSR_INVALID);
 987
 988         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 989         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 990
 991         msrpm[offset] = tmp;
 992 }
 993
 994 static void svm_vcpu_init_msrpm(u32 *msrpm)
 995 {
 996         int i;
 997
 998         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
 999
1000         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1001                 if (!direct_access_msrs[i].always)
1002                         continue;
1003
1004                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
1005         }
1006 }
1007
1008 static void add_msr_offset(u32 offset)
1009 {
1010         int i;
1011
1012         for (i = 0; i < MSRPM_OFFSETS; ++i) {
1013
1014                 /* Offset already in list? */
1015                 if (msrpm_offsets[i] == offset)
1016                         return;
1017
1018                 /* Slot used by another offset? */
1019                 if (msrpm_offsets[i] != MSR_INVALID)
1020                         continue;
1021
1022                 /* Add offset to list */
1023                 msrpm_offsets[i] = offset;
1024
1025                 return;
1026         }
1027
1028         /*
1029          * If this BUG triggers the msrpm_offsets table has an overflow. Just
1030          * increase MSRPM_OFFSETS in this case.
1031          */
1032         BUG();
1033 }
1034
1035 static void init_msrpm_offsets(void)
1036 {
1037         int i;
1038
1039         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
1040
1041         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1042                 u32 offset;
1043
1044                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
1045                 BUG_ON(offset == MSR_INVALID);
1046
1047                 add_msr_offset(offset);
1048         }
1049 }
1050
1051 static void svm_enable_lbrv(struct vcpu_svm *svm)
1052 {
1053         u32 *msrpm = svm->msrpm;
1054
1055         svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1056         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1057         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1058         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1059         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1060 }
1061
1062 static void svm_disable_lbrv(struct vcpu_svm *svm)
1063 {
1064         u32 *msrpm = svm->msrpm;
1065
1066         svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1067         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1068         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1069         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1070         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1071 }
1072
1073 static void disable_nmi_singlestep(struct vcpu_svm *svm)
1074 {
1075         svm->nmi_singlestep = false;
1076
1077         if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1078                 /* Clear our flags if they were not set by the guest */
1079                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1080                         svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1081                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1082                         svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1083         }
1084 }
1085
1086 /* Note:
1087  * This hash table is used to map VM_ID to a struct kvm_arch,
1088  * when handling AMD IOMMU GALOG notification to schedule in
1089  * a particular vCPU.
1090  */
1091 #define SVM_VM_DATA_HASH_BITS   8
1092 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
1093 static u32 next_vm_id = 0;
1094 static bool next_vm_id_wrapped = 0;
1095 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
1096
1097 /* Note:
1098  * This function is called from IOMMU driver to notify
1099  * SVM to schedule in a particular vCPU of a particular VM.
1100  */
1101 static int avic_ga_log_notifier(u32 ga_tag)
1102 {
1103         unsigned long flags;
1104         struct kvm_arch *ka = NULL;
1105         struct kvm_vcpu *vcpu = NULL;
1106         u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
1107         u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
1108
1109         pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
1110
1111         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1112         hash_for_each_possible(svm_vm_data_hash, ka, hnode, vm_id) {
1113                 struct kvm *kvm = container_of(ka, struct kvm, arch);
1114                 struct kvm_arch *vm_data = &kvm->arch;
1115
1116                 if (vm_data->avic_vm_id != vm_id)
1117                         continue;
1118                 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1119                 break;
1120         }
1121         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1122
1123         /* Note:
1124          * At this point, the IOMMU should have already set the pending
1125          * bit in the vAPIC backing page. So, we just need to schedule
1126          * in the vcpu.
1127          */
1128         if (vcpu)
1129                 kvm_vcpu_wake_up(vcpu);
1130
1131         return 0;
1132 }
1133
1134 static __init int sev_hardware_setup(void)
1135 {
1136         struct sev_user_data_status *status;
1137         int rc;
1138
1139         /* Maximum number of encrypted guests supported simultaneously */
1140         max_sev_asid = cpuid_ecx(0x8000001F);
1141
1142         if (!max_sev_asid)
1143                 return 1;
1144
1145         /* Minimum ASID value that should be used for SEV guest */
1146         min_sev_asid = cpuid_edx(0x8000001F);
1147
1148         /* Initialize SEV ASID bitmap */
1149         sev_asid_bitmap = kcalloc(BITS_TO_LONGS(max_sev_asid),
1150                                 sizeof(unsigned long), GFP_KERNEL);
1151         if (!sev_asid_bitmap)
1152                 return 1;
1153
1154         status = kmalloc(sizeof(*status), GFP_KERNEL);
1155         if (!status)
1156                 return 1;
1157
1158         /*
1159          * Check SEV platform status.
1160          *
1161          * PLATFORM_STATUS can be called in any state, if we failed to query
1162          * the PLATFORM status then either PSP firmware does not support SEV
1163          * feature or SEV firmware is dead.
1164          */
1165         rc = sev_platform_status(status, NULL);
1166         if (rc)
1167                 goto err;
1168
1169         pr_info("SEV supported\n");
1170
1171 err:
1172         kfree(status);
1173         return rc;
1174 }
1175
1176 static __init int svm_hardware_setup(void)
1177 {
1178         int cpu;
1179         struct page *iopm_pages;
1180         void *iopm_va;
1181         int r;
1182
1183         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
1184
1185         if (!iopm_pages)
1186                 return -ENOMEM;
1187
1188         iopm_va = page_address(iopm_pages);
1189         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
1190         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
1191
1192         init_msrpm_offsets();
1193
1194         if (boot_cpu_has(X86_FEATURE_NX))
1195                 kvm_enable_efer_bits(EFER_NX);
1196
1197         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
1198                 kvm_enable_efer_bits(EFER_FFXSR);
1199
1200         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1201                 kvm_has_tsc_control = true;
1202                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1203                 kvm_tsc_scaling_ratio_frac_bits = 32;
1204         }
1205
1206         if (nested) {
1207                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1208                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1209         }
1210
1211         if (sev) {
1212                 if (boot_cpu_has(X86_FEATURE_SEV) &&
1213                     IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
1214                         r = sev_hardware_setup();
1215                         if (r)
1216                                 sev = false;
1217                 } else {
1218                         sev = false;
1219                 }
1220         }
1221
1222         for_each_possible_cpu(cpu) {
1223                 r = svm_cpu_init(cpu);
1224                 if (r)
1225                         goto err;
1226         }
1227
1228         if (!boot_cpu_has(X86_FEATURE_NPT))
1229                 npt_enabled = false;
1230
1231         if (npt_enabled && !npt) {
1232                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
1233                 npt_enabled = false;
1234         }
1235
1236         if (npt_enabled) {
1237                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
1238                 kvm_enable_tdp();
1239         } else
1240                 kvm_disable_tdp();
1241
1242         if (avic) {
1243                 if (!npt_enabled ||
1244                     !boot_cpu_has(X86_FEATURE_AVIC) ||
1245                     !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
1246                         avic = false;
1247                 } else {
1248                         pr_info("AVIC enabled\n");
1249
1250                         amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1251                 }
1252         }
1253
1254         if (vls) {
1255                 if (!npt_enabled ||
1256                     !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1257                     !IS_ENABLED(CONFIG_X86_64)) {
1258                         vls = false;
1259                 } else {
1260                         pr_info("Virtual VMLOAD VMSAVE supported\n");
1261                 }
1262         }
1263
1264         if (vgif) {
1265                 if (!boot_cpu_has(X86_FEATURE_VGIF))
1266                         vgif = false;
1267                 else
1268                         pr_info("Virtual GIF supported\n");
1269         }
1270
1271         return 0;
1272
1273 err:
1274         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
1275         iopm_base = 0;
1276         return r;
1277 }
1278
1279 static __exit void svm_hardware_unsetup(void)
1280 {
1281         int cpu;
1282
1283         if (svm_sev_enabled())
1284                 kfree(sev_asid_bitmap);
1285
1286         for_each_possible_cpu(cpu)
1287                 svm_cpu_uninit(cpu);
1288
1289         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1290         iopm_base = 0;
1291 }
1292
1293 static void init_seg(struct vmcb_seg *seg)
1294 {
1295         seg->selector = 0;
1296         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1297                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1298         seg->limit = 0xffff;
1299         seg->base = 0;
1300 }
1301
1302 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1303 {
1304         seg->selector = 0;
1305         seg->attrib = SVM_SELECTOR_P_MASK | type;
1306         seg->limit = 0xffff;
1307         seg->base = 0;
1308 }
1309
1310 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1311 {
1312         struct vcpu_svm *svm = to_svm(vcpu);
1313         u64 g_tsc_offset = 0;
1314
1315         if (is_guest_mode(vcpu)) {
1316                 g_tsc_offset = svm->vmcb->control.tsc_offset -
1317                                svm->nested.hsave->control.tsc_offset;
1318                 svm->nested.hsave->control.tsc_offset = offset;
1319         } else
1320                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1321                                            svm->vmcb->control.tsc_offset,
1322                                            offset);
1323
1324         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1325
1326         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1327 }
1328
1329 static void avic_init_vmcb(struct vcpu_svm *svm)
1330 {
1331         struct vmcb *vmcb = svm->vmcb;
1332         struct kvm_arch *vm_data = &svm->vcpu.kvm->arch;
1333         phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
1334         phys_addr_t lpa = __sme_set(page_to_phys(vm_data->avic_logical_id_table_page));
1335         phys_addr_t ppa = __sme_set(page_to_phys(vm_data->avic_physical_id_table_page));
1336
1337         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1338         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1339         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1340         vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1341         vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1342 }
1343
1344 static void init_vmcb(struct vcpu_svm *svm)
1345 {
1346         struct vmcb_control_area *control = &svm->vmcb->control;
1347         struct vmcb_save_area *save = &svm->vmcb->save;
1348
1349         svm->vcpu.arch.hflags = 0;
1350
1351         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1352         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1353         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1354         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1355         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1356         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1357         if (!kvm_vcpu_apicv_active(&svm->vcpu))
1358                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1359
1360         set_dr_intercepts(svm);
1361
1362         set_exception_intercept(svm, PF_VECTOR);
1363         set_exception_intercept(svm, UD_VECTOR);
1364         set_exception_intercept(svm, MC_VECTOR);
1365         set_exception_intercept(svm, AC_VECTOR);
1366         set_exception_intercept(svm, DB_VECTOR);
1367
1368         set_intercept(svm, INTERCEPT_INTR);
1369         set_intercept(svm, INTERCEPT_NMI);
1370         set_intercept(svm, INTERCEPT_SMI);
1371         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1372         set_intercept(svm, INTERCEPT_RDPMC);
1373         set_intercept(svm, INTERCEPT_CPUID);
1374         set_intercept(svm, INTERCEPT_INVD);
1375         set_intercept(svm, INTERCEPT_HLT);
1376         set_intercept(svm, INTERCEPT_INVLPG);
1377         set_intercept(svm, INTERCEPT_INVLPGA);
1378         set_intercept(svm, INTERCEPT_IOIO_PROT);
1379         set_intercept(svm, INTERCEPT_MSR_PROT);
1380         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1381         set_intercept(svm, INTERCEPT_SHUTDOWN);
1382         set_intercept(svm, INTERCEPT_VMRUN);
1383         set_intercept(svm, INTERCEPT_VMMCALL);
1384         set_intercept(svm, INTERCEPT_VMLOAD);
1385         set_intercept(svm, INTERCEPT_VMSAVE);
1386         set_intercept(svm, INTERCEPT_STGI);
1387         set_intercept(svm, INTERCEPT_CLGI);
1388         set_intercept(svm, INTERCEPT_SKINIT);
1389         set_intercept(svm, INTERCEPT_WBINVD);
1390         set_intercept(svm, INTERCEPT_XSETBV);
1391         set_intercept(svm, INTERCEPT_RSM);
1392
1393         if (!kvm_mwait_in_guest()) {
1394                 set_intercept(svm, INTERCEPT_MONITOR);
1395                 set_intercept(svm, INTERCEPT_MWAIT);
1396         }
1397
1398         control->iopm_base_pa = __sme_set(iopm_base);
1399         control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1400         control->int_ctl = V_INTR_MASKING_MASK;
1401
1402         init_seg(&save->es);
1403         init_seg(&save->ss);
1404         init_seg(&save->ds);
1405         init_seg(&save->fs);
1406         init_seg(&save->gs);
1407
1408         save->cs.selector = 0xf000;
1409         save->cs.base = 0xffff0000;
1410         /* Executable/Readable Code Segment */
1411         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1412                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1413         save->cs.limit = 0xffff;
1414
1415         save->gdtr.limit = 0xffff;
1416         save->idtr.limit = 0xffff;
1417
1418         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1419         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1420
1421         svm_set_efer(&svm->vcpu, 0);
1422         save->dr6 = 0xffff0ff0;
1423         kvm_set_rflags(&svm->vcpu, 2);
1424         save->rip = 0x0000fff0;
1425         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1426
1427         /*
1428          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1429          * It also updates the guest-visible cr0 value.
1430          */
1431         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1432         kvm_mmu_reset_context(&svm->vcpu);
1433
1434         save->cr4 = X86_CR4_PAE;
1435         /* rdx = ?? */
1436
1437         if (npt_enabled) {
1438                 /* Setup VMCB for Nested Paging */
1439                 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1440                 clr_intercept(svm, INTERCEPT_INVLPG);
1441                 clr_exception_intercept(svm, PF_VECTOR);
1442                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1443                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1444                 save->g_pat = svm->vcpu.arch.pat;
1445                 save->cr3 = 0;
1446                 save->cr4 = 0;
1447         }
1448         svm->asid_generation = 0;
1449
1450         svm->nested.vmcb = 0;
1451         svm->vcpu.arch.hflags = 0;
1452
1453         if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1454                 control->pause_filter_count = 3000;
1455                 set_intercept(svm, INTERCEPT_PAUSE);
1456         }
1457
1458         if (kvm_vcpu_apicv_active(&svm->vcpu))
1459                 avic_init_vmcb(svm);
1460
1461         /*
1462          * If hardware supports Virtual VMLOAD VMSAVE then enable it
1463          * in VMCB and clear intercepts to avoid #VMEXIT.
1464          */
1465         if (vls) {
1466                 clr_intercept(svm, INTERCEPT_VMLOAD);
1467                 clr_intercept(svm, INTERCEPT_VMSAVE);
1468                 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1469         }
1470
1471         if (vgif) {
1472                 clr_intercept(svm, INTERCEPT_STGI);
1473                 clr_intercept(svm, INTERCEPT_CLGI);
1474                 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1475         }
1476
1477         if (sev_guest(svm->vcpu.kvm)) {
1478                 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1479                 clr_exception_intercept(svm, UD_VECTOR);
1480         }
1481
1482         mark_all_dirty(svm->vmcb);
1483
1484         enable_gif(svm);
1485
1486 }
1487
1488 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
1489                                        unsigned int index)
1490 {
1491         u64 *avic_physical_id_table;
1492         struct kvm_arch *vm_data = &vcpu->kvm->arch;
1493
1494         if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1495                 return NULL;
1496
1497         avic_physical_id_table = page_address(vm_data->avic_physical_id_table_page);
1498
1499         return &avic_physical_id_table[index];
1500 }
1501
1502 /**
1503  * Note:
1504  * AVIC hardware walks the nested page table to check permissions,
1505  * but does not use the SPA address specified in the leaf page
1506  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
1507  * field of the VMCB. Therefore, we set up the
1508  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1509  */
1510 static int avic_init_access_page(struct kvm_vcpu *vcpu)
1511 {
1512         struct kvm *kvm = vcpu->kvm;
1513         int ret;
1514
1515         if (kvm->arch.apic_access_page_done)
1516                 return 0;
1517
1518         ret = x86_set_memory_region(kvm,
1519                                     APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1520                                     APIC_DEFAULT_PHYS_BASE,
1521                                     PAGE_SIZE);
1522         if (ret)
1523                 return ret;
1524
1525         kvm->arch.apic_access_page_done = true;
1526         return 0;
1527 }
1528
1529 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1530 {
1531         int ret;
1532         u64 *entry, new_entry;
1533         int id = vcpu->vcpu_id;
1534         struct vcpu_svm *svm = to_svm(vcpu);
1535
1536         ret = avic_init_access_page(vcpu);
1537         if (ret)
1538                 return ret;
1539
1540         if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1541                 return -EINVAL;
1542
1543         if (!svm->vcpu.arch.apic->regs)
1544                 return -EINVAL;
1545
1546         svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1547
1548         /* Setting AVIC backing page address in the phy APIC ID table */
1549         entry = avic_get_physical_id_entry(vcpu, id);
1550         if (!entry)
1551                 return -EINVAL;
1552
1553         new_entry = READ_ONCE(*entry);
1554         new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
1555                               AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1556                               AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
1557         WRITE_ONCE(*entry, new_entry);
1558
1559         svm->avic_physical_id_cache = entry;
1560
1561         return 0;
1562 }
1563
1564 static void __sev_asid_free(int asid)
1565 {
1566         struct svm_cpu_data *sd;
1567         int cpu, pos;
1568
1569         pos = asid - 1;
1570         clear_bit(pos, sev_asid_bitmap);
1571
1572         for_each_possible_cpu(cpu) {
1573                 sd = per_cpu(svm_data, cpu);
1574                 sd->sev_vmcbs[pos] = NULL;
1575         }
1576 }
1577
1578 static void sev_asid_free(struct kvm *kvm)
1579 {
1580         struct kvm_sev_info *sev = &kvm->arch.sev_info;
1581
1582         __sev_asid_free(sev->asid);
1583 }
1584
1585 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
1586 {
1587         struct sev_data_decommission *decommission;
1588         struct sev_data_deactivate *data;
1589
1590         if (!handle)
1591                 return;
1592
1593         data = kzalloc(sizeof(*data), GFP_KERNEL);
1594         if (!data)
1595                 return;
1596
1597         /* deactivate handle */
1598         data->handle = handle;
1599         sev_guest_deactivate(data, NULL);
1600
1601         wbinvd_on_all_cpus();
1602         sev_guest_df_flush(NULL);
1603         kfree(data);
1604
1605         decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
1606         if (!decommission)
1607                 return;
1608
1609         /* decommission handle */
1610         decommission->handle = handle;
1611         sev_guest_decommission(decommission, NULL);
1612
1613         kfree(decommission);
1614 }
1615
1616 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
1617                                     unsigned long ulen, unsigned long *n,
1618                                     int write)
1619 {
1620         struct kvm_sev_info *sev = &kvm->arch.sev_info;
1621         unsigned long npages, npinned, size;
1622         unsigned long locked, lock_limit;
1623         struct page **pages;
1624         int first, last;
1625
1626         /* Calculate number of pages. */
1627         first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
1628         last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
1629         npages = (last - first + 1);
1630
1631         locked = sev->pages_locked + npages;
1632         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1633         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
1634                 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
1635                 return NULL;
1636         }
1637
1638         /* Avoid using vmalloc for smaller buffers. */
1639         size = npages * sizeof(struct page *);
1640         if (size > PAGE_SIZE)
1641                 pages = vmalloc(size);
1642         else
1643                 pages = kmalloc(size, GFP_KERNEL);
1644
1645         if (!pages)
1646                 return NULL;
1647
1648         /* Pin the user virtual address. */
1649         npinned = get_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
1650         if (npinned != npages) {
1651                 pr_err("SEV: Failure locking %lu pages.\n", npages);
1652                 goto err;
1653         }
1654
1655         *n = npages;
1656         sev->pages_locked = locked;
1657
1658         return pages;
1659
1660 err:
1661         if (npinned > 0)
1662                 release_pages(pages, npinned);
1663
1664         kvfree(pages);
1665         return NULL;
1666 }
1667
1668 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
1669                              unsigned long npages)
1670 {
1671         struct kvm_sev_info *sev = &kvm->arch.sev_info;
1672
1673         release_pages(pages, npages);
1674         kvfree(pages);
1675         sev->pages_locked -= npages;
1676 }
1677
1678 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
1679 {
1680         uint8_t *page_virtual;
1681         unsigned long i;
1682
1683         if (npages == 0 || pages == NULL)
1684                 return;
1685
1686         for (i = 0; i < npages; i++) {
1687                 page_virtual = kmap_atomic(pages[i]);
1688                 clflush_cache_range(page_virtual, PAGE_SIZE);
1689                 kunmap_atomic(page_virtual);
1690         }
1691 }
1692
1693 static void __unregister_enc_region_locked(struct kvm *kvm,
1694                                            struct enc_region *region)
1695 {
1696         /*
1697          * The guest may change the memory encryption attribute from C=0 -> C=1
1698          * or vice versa for this memory range. Lets make sure caches are
1699          * flushed to ensure that guest data gets written into memory with
1700          * correct C-bit.
1701          */
1702         sev_clflush_pages(region->pages, region->npages);
1703
1704         sev_unpin_memory(kvm, region->pages, region->npages);
1705         list_del(&region->list);
1706         kfree(region);
1707 }
1708
1709 static void sev_vm_destroy(struct kvm *kvm)
1710 {
1711         struct kvm_sev_info *sev = &kvm->arch.sev_info;
1712         struct list_head *head = &sev->regions_list;
1713         struct list_head *pos, *q;
1714
1715         if (!sev_guest(kvm))
1716                 return;
1717
1718         mutex_lock(&kvm->lock);
1719
1720         /*
1721          * if userspace was terminated before unregistering the memory regions
1722          * then lets unpin all the registered memory.
1723          */
1724         if (!list_empty(head)) {
1725                 list_for_each_safe(pos, q, head) {
1726                         __unregister_enc_region_locked(kvm,
1727                                 list_entry(pos, struct enc_region, list));
1728                 }
1729         }
1730
1731         mutex_unlock(&kvm->lock);
1732
1733         sev_unbind_asid(kvm, sev->handle);
1734         sev_asid_free(kvm);
1735 }
1736
1737 static void avic_vm_destroy(struct kvm *kvm)
1738 {
1739         unsigned long flags;
1740         struct kvm_arch *vm_data = &kvm->arch;
1741
1742         if (!avic)
1743                 return;
1744
1745         if (vm_data->avic_logical_id_table_page)
1746                 __free_page(vm_data->avic_logical_id_table_page);
1747         if (vm_data->avic_physical_id_table_page)
1748                 __free_page(vm_data->avic_physical_id_table_page);
1749
1750         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1751         hash_del(&vm_data->hnode);
1752         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1753 }
1754
1755 static void svm_vm_destroy(struct kvm *kvm)
1756 {
1757         avic_vm_destroy(kvm);
1758         sev_vm_destroy(kvm);
1759 }
1760
1761 static int avic_vm_init(struct kvm *kvm)
1762 {
1763         unsigned long flags;
1764         int err = -ENOMEM;
1765         struct kvm_arch *vm_data = &kvm->arch;
1766         struct page *p_page;
1767         struct page *l_page;
1768         struct kvm_arch *ka;
1769         u32 vm_id;
1770
1771         if (!avic)
1772                 return 0;
1773
1774         /* Allocating physical APIC ID table (4KB) */
1775         p_page = alloc_page(GFP_KERNEL);
1776         if (!p_page)
1777                 goto free_avic;
1778
1779         vm_data->avic_physical_id_table_page = p_page;
1780         clear_page(page_address(p_page));
1781
1782         /* Allocating logical APIC ID table (4KB) */
1783         l_page = alloc_page(GFP_KERNEL);
1784         if (!l_page)
1785                 goto free_avic;
1786
1787         vm_data->avic_logical_id_table_page = l_page;
1788         clear_page(page_address(l_page));
1789
1790         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1791  again:
1792         vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
1793         if (vm_id == 0) { /* id is 1-based, zero is not okay */
1794                 next_vm_id_wrapped = 1;
1795                 goto again;
1796         }
1797         /* Is it still in use? Only possible if wrapped at least once */
1798         if (next_vm_id_wrapped) {
1799                 hash_for_each_possible(svm_vm_data_hash, ka, hnode, vm_id) {
1800                         struct kvm *k2 = container_of(ka, struct kvm, arch);
1801                         struct kvm_arch *vd2 = &k2->arch;
1802                         if (vd2->avic_vm_id == vm_id)
1803                                 goto again;
1804                 }
1805         }
1806         vm_data->avic_vm_id = vm_id;
1807         hash_add(svm_vm_data_hash, &vm_data->hnode, vm_data->avic_vm_id);
1808         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1809
1810         return 0;
1811
1812 free_avic:
1813         avic_vm_destroy(kvm);
1814         return err;
1815 }
1816
1817 static inline int
1818 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
1819 {
1820         int ret = 0;
1821         unsigned long flags;
1822         struct amd_svm_iommu_ir *ir;
1823         struct vcpu_svm *svm = to_svm(vcpu);
1824
1825         if (!kvm_arch_has_assigned_device(vcpu->kvm))
1826                 return 0;
1827
1828         /*
1829          * Here, we go through the per-vcpu ir_list to update all existing
1830          * interrupt remapping table entry targeting this vcpu.
1831          */
1832         spin_lock_irqsave(&svm->ir_list_lock, flags);
1833
1834         if (list_empty(&svm->ir_list))
1835                 goto out;
1836
1837         list_for_each_entry(ir, &svm->ir_list, node) {
1838                 ret = amd_iommu_update_ga(cpu, r, ir->data);
1839                 if (ret)
1840                         break;
1841         }
1842 out:
1843         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
1844         return ret;
1845 }
1846
1847 static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1848 {
1849         u64 entry;
1850         /* ID = 0xff (broadcast), ID > 0xff (reserved) */
1851         int h_physical_id = kvm_cpu_get_apicid(cpu);
1852         struct vcpu_svm *svm = to_svm(vcpu);
1853
1854         if (!kvm_vcpu_apicv_active(vcpu))
1855                 return;
1856
1857         if (WARN_ON(h_physical_id >= AVIC_MAX_PHYSICAL_ID_COUNT))
1858                 return;
1859
1860         entry = READ_ONCE(*(svm->avic_physical_id_cache));
1861         WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
1862
1863         entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
1864         entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
1865
1866         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
1867         if (svm->avic_is_running)
1868                 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
1869
1870         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
1871         avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
1872                                         svm->avic_is_running);
1873 }
1874
1875 static void avic_vcpu_put(struct kvm_vcpu *vcpu)
1876 {
1877         u64 entry;
1878         struct vcpu_svm *svm = to_svm(vcpu);
1879
1880         if (!kvm_vcpu_apicv_active(vcpu))
1881                 return;
1882
1883         entry = READ_ONCE(*(svm->avic_physical_id_cache));
1884         if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
1885                 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
1886
1887         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
1888         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
1889 }
1890
1891 /**
1892  * This function is called during VCPU halt/unhalt.
1893  */
1894 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
1895 {
1896         struct vcpu_svm *svm = to_svm(vcpu);
1897
1898         svm->avic_is_running = is_run;
1899         if (is_run)
1900                 avic_vcpu_load(vcpu, vcpu->cpu);
1901         else
1902                 avic_vcpu_put(vcpu);
1903 }
1904
1905 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1906 {
1907         struct vcpu_svm *svm = to_svm(vcpu);
1908         u32 dummy;
1909         u32 eax = 1;
1910
1911         vcpu->arch.microcode_version = 0x01000065;
1912         svm->spec_ctrl = 0;
1913
1914         if (!init_event) {
1915                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1916                                            MSR_IA32_APICBASE_ENABLE;
1917                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1918                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1919         }
1920         init_vmcb(svm);
1921
1922         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);
1923         kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
1924
1925         if (kvm_vcpu_apicv_active(vcpu) && !init_event)
1926                 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
1927 }
1928
1929 static int avic_init_vcpu(struct vcpu_svm *svm)
1930 {
1931         int ret;
1932
1933         if (!kvm_vcpu_apicv_active(&svm->vcpu))
1934                 return 0;
1935
1936         ret = avic_init_backing_page(&svm->vcpu);
1937         if (ret)
1938                 return ret;
1939
1940         INIT_LIST_HEAD(&svm->ir_list);
1941         spin_lock_init(&svm->ir_list_lock);
1942
1943         return ret;
1944 }
1945
1946 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1947 {
1948         struct vcpu_svm *svm;
1949         struct page *page;
1950         struct page *msrpm_pages;
1951         struct page *hsave_page;
1952         struct page *nested_msrpm_pages;
1953         int err;
1954
1955         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1956         if (!svm) {
1957                 err = -ENOMEM;
1958                 goto out;
1959         }
1960
1961         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
1962         if (err)
1963                 goto free_svm;
1964
1965         err = -ENOMEM;
1966         page = alloc_page(GFP_KERNEL);
1967         if (!page)
1968                 goto uninit;
1969
1970         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1971         if (!msrpm_pages)
1972                 goto free_page1;
1973
1974         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1975         if (!nested_msrpm_pages)
1976                 goto free_page2;
1977
1978         hsave_page = alloc_page(GFP_KERNEL);
1979         if (!hsave_page)
1980                 goto free_page3;
1981
1982         err = avic_init_vcpu(svm);
1983         if (err)
1984                 goto free_page4;
1985
1986         /* We initialize this flag to true to make sure that the is_running
1987          * bit would be set the first time the vcpu is loaded.
1988          */
1989         svm->avic_is_running = true;
1990
1991         svm->nested.hsave = page_address(hsave_page);
1992
1993         svm->msrpm = page_address(msrpm_pages);
1994         svm_vcpu_init_msrpm(svm->msrpm);
1995
1996         svm->nested.msrpm = page_address(nested_msrpm_pages);
1997         svm_vcpu_init_msrpm(svm->nested.msrpm);
1998
1999         svm->vmcb = page_address(page);
2000         clear_page(svm->vmcb);
2001         svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
2002         svm->asid_generation = 0;
2003         init_vmcb(svm);
2004
2005         svm_init_osvw(&svm->vcpu);
2006
2007         return &svm->vcpu;
2008
2009 free_page4:
2010         __free_page(hsave_page);
2011 free_page3:
2012         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
2013 free_page2:
2014         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
2015 free_page1:
2016         __free_page(page);
2017 uninit:
2018         kvm_vcpu_uninit(&svm->vcpu);
2019 free_svm:
2020         kmem_cache_free(kvm_vcpu_cache, svm);
2021 out:
2022         return ERR_PTR(err);
2023 }
2024
2025 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
2026 {
2027         struct vcpu_svm *svm = to_svm(vcpu);
2028
2029         __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
2030         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
2031         __free_page(virt_to_page(svm->nested.hsave));
2032         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
2033         kvm_vcpu_uninit(vcpu);
2034         kmem_cache_free(kvm_vcpu_cache, svm);
2035         /*
2036          * The vmcb page can be recycled, causing a false negative in
2037          * svm_vcpu_load(). So do a full IBPB now.
2038          */
2039         indirect_branch_prediction_barrier();
2040 }
2041
2042 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2043 {
2044         struct vcpu_svm *svm = to_svm(vcpu);
2045         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2046         int i;
2047
2048         if (unlikely(cpu != vcpu->cpu)) {
2049                 svm->asid_generation = 0;
2050                 mark_all_dirty(svm->vmcb);
2051         }
2052
2053 #ifdef CONFIG_X86_64
2054         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
2055 #endif
2056         savesegment(fs, svm->host.fs);
2057         savesegment(gs, svm->host.gs);
2058         svm->host.ldt = kvm_read_ldt();
2059
2060         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2061                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2062
2063         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
2064                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
2065                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
2066                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
2067                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
2068                 }
2069         }
2070         /* This assumes that the kernel never uses MSR_TSC_AUX */
2071         if (static_cpu_has(X86_FEATURE_RDTSCP))
2072                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2073
2074         if (sd->current_vmcb != svm->vmcb) {
2075                 sd->current_vmcb = svm->vmcb;
2076                 indirect_branch_prediction_barrier();
2077         }
2078         avic_vcpu_load(vcpu, cpu);
2079 }
2080
2081 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
2082 {
2083         struct vcpu_svm *svm = to_svm(vcpu);
2084         int i;
2085
2086         avic_vcpu_put(vcpu);
2087
2088         ++vcpu->stat.host_state_reload;
2089         kvm_load_ldt(svm->host.ldt);
2090 #ifdef CONFIG_X86_64
2091         loadsegment(fs, svm->host.fs);
2092         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
2093         load_gs_index(svm->host.gs);
2094 #else
2095 #ifdef CONFIG_X86_32_LAZY_GS
2096         loadsegment(gs, svm->host.gs);
2097 #endif
2098 #endif
2099         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2100                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2101 }
2102
2103 static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
2104 {
2105         avic_set_running(vcpu, false);
2106 }
2107
2108 static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
2109 {
2110         avic_set_running(vcpu, true);
2111 }
2112
2113 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
2114 {
2115         struct vcpu_svm *svm = to_svm(vcpu);
2116         unsigned long rflags = svm->vmcb->save.rflags;
2117
2118         if (svm->nmi_singlestep) {
2119                 /* Hide our flags if they were not set by the guest */
2120                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
2121                         rflags &= ~X86_EFLAGS_TF;
2122                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
2123                         rflags &= ~X86_EFLAGS_RF;
2124         }
2125         return rflags;
2126 }
2127
2128 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2129 {
2130         if (to_svm(vcpu)->nmi_singlestep)
2131                 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
2132
2133        /*
2134         * Any change of EFLAGS.VM is accompanied by a reload of SS
2135         * (caused by either a task switch or an inter-privilege IRET),
2136         * so we do not need to update the CPL here.
2137         */
2138         to_svm(vcpu)->vmcb->save.rflags = rflags;
2139 }
2140
2141 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2142 {
2143         switch (reg) {
2144         case VCPU_EXREG_PDPTR:
2145                 BUG_ON(!npt_enabled);
2146                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
2147                 break;
2148         default:
2149                 BUG();
2150         }
2151 }
2152
2153 static void svm_set_vintr(struct vcpu_svm *svm)
2154 {
2155         set_intercept(svm, INTERCEPT_VINTR);
2156 }
2157
2158 static void svm_clear_vintr(struct vcpu_svm *svm)
2159 {
2160         clr_intercept(svm, INTERCEPT_VINTR);
2161 }
2162
2163 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
2164 {
2165         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2166
2167         switch (seg) {
2168         case VCPU_SREG_CS: return &save->cs;
2169         case VCPU_SREG_DS: return &save->ds;
2170         case VCPU_SREG_ES: return &save->es;
2171         case VCPU_SREG_FS: return &save->fs;
2172         case VCPU_SREG_GS: return &save->gs;
2173         case VCPU_SREG_SS: return &save->ss;
2174         case VCPU_SREG_TR: return &save->tr;
2175         case VCPU_SREG_LDTR: return &save->ldtr;
2176         }
2177         BUG();
2178         return NULL;
2179 }
2180
2181 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
2182 {
2183         struct vmcb_seg *s = svm_seg(vcpu, seg);
2184
2185         return s->base;
2186 }
2187
2188 static void svm_get_segment(struct kvm_vcpu *vcpu,
2189                             struct kvm_segment *var, int seg)
2190 {
2191         struct vmcb_seg *s = svm_seg(vcpu, seg);
2192
2193         var->base = s->base;
2194         var->limit = s->limit;
2195         var->selector = s->selector;
2196         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
2197         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
2198         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
2199         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
2200         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
2201         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
2202         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
2203
2204         /*
2205          * AMD CPUs circa 2014 track the G bit for all segments except CS.
2206          * However, the SVM spec states that the G bit is not observed by the
2207          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
2208          * So let's synthesize a legal G bit for all segments, this helps
2209          * running KVM nested. It also helps cross-vendor migration, because
2210          * Intel's vmentry has a check on the 'G' bit.
2211          */
2212         var->g = s->limit > 0xfffff;
2213
2214         /*
2215          * AMD's VMCB does not have an explicit unusable field, so emulate it
2216          * for cross vendor migration purposes by "not present"
2217          */
2218         var->unusable = !var->present;
2219
2220         switch (seg) {
2221         case VCPU_SREG_TR:
2222                 /*
2223                  * Work around a bug where the busy flag in the tr selector
2224                  * isn't exposed
2225                  */
2226                 var->type |= 0x2;
2227                 break;
2228         case VCPU_SREG_DS:
2229         case VCPU_SREG_ES:
2230         case VCPU_SREG_FS:
2231         case VCPU_SREG_GS:
2232                 /*
2233                  * The accessed bit must always be set in the segment
2234                  * descriptor cache, although it can be cleared in the
2235                  * descriptor, the cached bit always remains at 1. Since
2236                  * Intel has a check on this, set it here to support
2237                  * cross-vendor migration.
2238                  */
2239                 if (!var->unusable)
2240                         var->type |= 0x1;
2241                 break;
2242         case VCPU_SREG_SS:
2243                 /*
2244                  * On AMD CPUs sometimes the DB bit in the segment
2245                  * descriptor is left as 1, although the whole segment has
2246                  * been made unusable. Clear it here to pass an Intel VMX
2247                  * entry check when cross vendor migrating.
2248                  */
2249                 if (var->unusable)
2250                         var->db = 0;
2251                 /* This is symmetric with svm_set_segment() */
2252                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
2253                 break;
2254         }
2255 }
2256
2257 static int svm_get_cpl(struct kvm_vcpu *vcpu)
2258 {
2259         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2260
2261         return save->cpl;
2262 }
2263
2264 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2265 {
2266         struct vcpu_svm *svm = to_svm(vcpu);
2267
2268         dt->size = svm->vmcb->save.idtr.limit;
2269         dt->address = svm->vmcb->save.idtr.base;
2270 }
2271
2272 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2273 {
2274         struct vcpu_svm *svm = to_svm(vcpu);
2275
2276         svm->vmcb->save.idtr.limit = dt->size;
2277         svm->vmcb->save.idtr.base = dt->address ;
2278         mark_dirty(svm->vmcb, VMCB_DT);
2279 }
2280
2281 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2282 {
2283         struct vcpu_svm *svm = to_svm(vcpu);
2284
2285         dt->size = svm->vmcb->save.gdtr.limit;
2286         dt->address = svm->vmcb->save.gdtr.base;
2287 }
2288
2289 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2290 {
2291         struct vcpu_svm *svm = to_svm(vcpu);
2292
2293         svm->vmcb->save.gdtr.limit = dt->size;
2294         svm->vmcb->save.gdtr.base = dt->address ;
2295         mark_dirty(svm->vmcb, VMCB_DT);
2296 }
2297
2298 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2299 {
2300 }
2301
2302 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
2303 {
2304 }
2305
2306 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
2307 {
2308 }
2309
2310 static void update_cr0_intercept(struct vcpu_svm *svm)
2311 {
2312         ulong gcr0 = svm->vcpu.arch.cr0;
2313         u64 *hcr0 = &svm->vmcb->save.cr0;
2314
2315         *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
2316                 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
2317
2318         mark_dirty(svm->vmcb, VMCB_CR);
2319
2320         if (gcr0 == *hcr0) {
2321                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
2322                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2323         } else {
2324                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
2325                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2326         }
2327 }
2328
2329 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
2330 {
2331         struct vcpu_svm *svm = to_svm(vcpu);
2332
2333 #ifdef CONFIG_X86_64
2334         if (vcpu->arch.efer & EFER_LME) {
2335                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
2336                         vcpu->arch.efer |= EFER_LMA;
2337                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
2338                 }
2339
2340                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
2341                         vcpu->arch.efer &= ~EFER_LMA;
2342                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
2343                 }
2344         }
2345 #endif
2346         vcpu->arch.cr0 = cr0;
2347
2348         if (!npt_enabled)
2349                 cr0 |= X86_CR0_PG | X86_CR0_WP;
2350
2351         /*
2352          * re-enable caching here because the QEMU bios
2353          * does not do it - this results in some delay at
2354          * reboot
2355          */
2356         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
2357                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
2358         svm->vmcb->save.cr0 = cr0;
2359         mark_dirty(svm->vmcb, VMCB_CR);
2360         update_cr0_intercept(svm);
2361 }
2362
2363 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
2364 {
2365         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
2366         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
2367
2368         if (cr4 & X86_CR4_VMXE)
2369                 return 1;
2370
2371         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
2372                 svm_flush_tlb(vcpu, true);
2373
2374         vcpu->arch.cr4 = cr4;
2375         if (!npt_enabled)
2376                 cr4 |= X86_CR4_PAE;
2377         cr4 |= host_cr4_mce;
2378         to_svm(vcpu)->vmcb->save.cr4 = cr4;
2379         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2380         return 0;
2381 }
2382
2383 static void svm_set_segment(struct kvm_vcpu *vcpu,
2384                             struct kvm_segment *var, int seg)
2385 {
2386         struct vcpu_svm *svm = to_svm(vcpu);
2387         struct vmcb_seg *s = svm_seg(vcpu, seg);
2388
2389         s->base = var->base;
2390         s->limit = var->limit;
2391         s->selector = var->selector;
2392         s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2393         s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2394         s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2395         s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2396         s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2397         s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2398         s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2399         s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
2400
2401         /*
2402          * This is always accurate, except if SYSRET returned to a segment
2403          * with SS.DPL != 3.  Intel does not have this quirk, and always
2404          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2405          * would entail passing the CPL to userspace and back.
2406          */
2407         if (seg == VCPU_SREG_SS)
2408                 /* This is symmetric with svm_get_segment() */
2409                 svm->vmcb->save.cpl = (var->dpl & 3);
2410
2411         mark_dirty(svm->vmcb, VMCB_SEG);
2412 }
2413
2414 static void update_bp_intercept(struct kvm_vcpu *vcpu)
2415 {
2416         struct vcpu_svm *svm = to_svm(vcpu);
2417
2418         clr_exception_intercept(svm, BP_VECTOR);
2419
2420         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
2421                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2422                         set_exception_intercept(svm, BP_VECTOR);
2423         } else
2424                 vcpu->guest_debug = 0;
2425 }
2426
2427 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2428 {
2429         if (sd->next_asid > sd->max_asid) {
2430                 ++sd->asid_generation;
2431                 sd->next_asid = sd->min_asid;
2432                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2433         }
2434
2435         svm->asid_generation = sd->asid_generation;
2436         svm->vmcb->control.asid = sd->next_asid++;
2437
2438         mark_dirty(svm->vmcb, VMCB_ASID);
2439 }
2440
2441 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
2442 {
2443         return to_svm(vcpu)->vmcb->save.dr6;
2444 }
2445
2446 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2447 {
2448         struct vcpu_svm *svm = to_svm(vcpu);
2449
2450         svm->vmcb->save.dr6 = value;
2451         mark_dirty(svm->vmcb, VMCB_DR);
2452 }
2453
2454 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2455 {
2456         struct vcpu_svm *svm = to_svm(vcpu);
2457
2458         get_debugreg(vcpu->arch.db[0], 0);
2459         get_debugreg(vcpu->arch.db[1], 1);
2460         get_debugreg(vcpu->arch.db[2], 2);
2461         get_debugreg(vcpu->arch.db[3], 3);
2462         vcpu->arch.dr6 = svm_get_dr6(vcpu);
2463         vcpu->arch.dr7 = svm->vmcb->save.dr7;
2464
2465         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2466         set_dr_intercepts(svm);
2467 }
2468
2469 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2470 {
2471         struct vcpu_svm *svm = to_svm(vcpu);
2472
2473         svm->vmcb->save.dr7 = value;
2474         mark_dirty(svm->vmcb, VMCB_DR);
2475 }
2476
2477 static int pf_interception(struct vcpu_svm *svm)
2478 {
2479         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2480         u64 error_code = svm->vmcb->control.exit_info_1;
2481
2482         return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
2483                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2484                         svm->vmcb->control.insn_bytes : NULL,
2485                         svm->vmcb->control.insn_len);
2486 }
2487
2488 static int npf_interception(struct vcpu_svm *svm)
2489 {
2490         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2491         u64 error_code = svm->vmcb->control.exit_info_1;
2492
2493         trace_kvm_page_fault(fault_address, error_code);
2494         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
2495                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2496                         svm->vmcb->control.insn_bytes : NULL,
2497                         svm->vmcb->control.insn_len);
2498 }
2499
2500 static int db_interception(struct vcpu_svm *svm)
2501 {
2502         struct kvm_run *kvm_run = svm->vcpu.run;
2503
2504         if (!(svm->vcpu.guest_debug &
2505               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2506                 !svm->nmi_singlestep) {
2507                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
2508                 return 1;
2509         }
2510
2511         if (svm->nmi_singlestep) {
2512                 disable_nmi_singlestep(svm);
2513         }
2514
2515         if (svm->vcpu.guest_debug &
2516             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2517                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2518                 kvm_run->debug.arch.pc =
2519                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2520                 kvm_run->debug.arch.exception = DB_VECTOR;
2521                 return 0;
2522         }
2523
2524         return 1;
2525 }
2526
2527 static int bp_interception(struct vcpu_svm *svm)
2528 {
2529         struct kvm_run *kvm_run = svm->vcpu.run;
2530
2531         kvm_run->exit_reason = KVM_EXIT_DEBUG;
2532         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2533         kvm_run->debug.arch.exception = BP_VECTOR;
2534         return 0;
2535 }
2536
2537 static int ud_interception(struct vcpu_svm *svm)
2538 {
2539         int er;
2540
2541         er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
2542         if (er == EMULATE_USER_EXIT)
2543                 return 0;
2544         if (er != EMULATE_DONE)
2545                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2546         return 1;
2547 }
2548
2549 static int ac_interception(struct vcpu_svm *svm)
2550 {
2551         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
2552         return 1;
2553 }
2554
2555 static bool is_erratum_383(void)
2556 {
2557         int err, i;
2558         u64 value;
2559
2560         if (!erratum_383_found)
2561                 return false;
2562
2563         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2564         if (err)
2565                 return false;
2566
2567         /* Bit 62 may or may not be set for this mce */
2568         value &= ~(1ULL << 62);
2569
2570         if (value != 0xb600000000010015ULL)
2571                 return false;
2572
2573         /* Clear MCi_STATUS registers */
2574         for (i = 0; i < 6; ++i)
2575                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2576
2577         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2578         if (!err) {
2579                 u32 low, high;
2580
2581                 value &= ~(1ULL << 2);
2582                 low    = lower_32_bits(value);
2583                 high   = upper_32_bits(value);
2584
2585                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2586         }
2587
2588         /* Flush tlb to evict multi-match entries */
2589         __flush_tlb_all();
2590
2591         return true;
2592 }
2593
2594 static void svm_handle_mce(struct vcpu_svm *svm)
2595 {
2596         if (is_erratum_383()) {
2597                 /*
2598                  * Erratum 383 triggered. Guest state is corrupt so kill the
2599                  * guest.
2600                  */
2601                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2602
2603                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
2604
2605                 return;
2606         }
2607
2608         /*
2609          * On an #MC intercept the MCE handler is not called automatically in
2610          * the host. So do it by hand here.
2611          */
2612         asm volatile (
2613                 "int $0x12\n");
2614         /* not sure if we ever come back to this point */
2615
2616         return;
2617 }
2618
2619 static int mc_interception(struct vcpu_svm *svm)
2620 {
2621         return 1;
2622 }
2623
2624 static int shutdown_interception(struct vcpu_svm *svm)
2625 {
2626         struct kvm_run *kvm_run = svm->vcpu.run;
2627
2628         /*
2629          * VMCB is undefined after a SHUTDOWN intercept
2630          * so reinitialize it.
2631          */
2632         clear_page(svm->vmcb);
2633         init_vmcb(svm);
2634
2635         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2636         return 0;
2637 }
2638
2639 static int io_interception(struct vcpu_svm *svm)
2640 {
2641         struct kvm_vcpu *vcpu = &svm->vcpu;
2642         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2643         int size, in, string, ret;
2644         unsigned port;
2645
2646         ++svm->vcpu.stat.io_exits;
2647         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2648         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2649         if (string)
2650                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
2651
2652         port = io_info >> 16;
2653         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2654         svm->next_rip = svm->vmcb->control.exit_info_2;
2655         ret = kvm_skip_emulated_instruction(&svm->vcpu);
2656
2657         /*
2658          * TODO: we might be squashing a KVM_GUESTDBG_SINGLESTEP-triggered
2659          * KVM_EXIT_DEBUG here.
2660          */
2661         if (in)
2662                 return kvm_fast_pio_in(vcpu, size, port) && ret;
2663         else
2664                 return kvm_fast_pio_out(vcpu, size, port) && ret;
2665 }
2666
2667 static int nmi_interception(struct vcpu_svm *svm)
2668 {
2669         return 1;
2670 }
2671
2672 static int intr_interception(struct vcpu_svm *svm)
2673 {
2674         ++svm->vcpu.stat.irq_exits;
2675         return 1;
2676 }
2677
2678 static int nop_on_interception(struct vcpu_svm *svm)
2679 {
2680         return 1;
2681 }
2682
2683 static int halt_interception(struct vcpu_svm *svm)
2684 {
2685         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
2686         return kvm_emulate_halt(&svm->vcpu);
2687 }
2688
2689 static int vmmcall_interception(struct vcpu_svm *svm)
2690 {
2691         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2692         return kvm_emulate_hypercall(&svm->vcpu);
2693 }
2694
2695 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2696 {
2697         struct vcpu_svm *svm = to_svm(vcpu);
2698
2699         return svm->nested.nested_cr3;
2700 }
2701
2702 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2703 {
2704         struct vcpu_svm *svm = to_svm(vcpu);
2705         u64 cr3 = svm->nested.nested_cr3;
2706         u64 pdpte;
2707         int ret;
2708
2709         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
2710                                        offset_in_page(cr3) + index * 8, 8);
2711         if (ret)
2712                 return 0;
2713         return pdpte;
2714 }
2715
2716 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2717                                    unsigned long root)
2718 {
2719         struct vcpu_svm *svm = to_svm(vcpu);
2720
2721         svm->vmcb->control.nested_cr3 = __sme_set(root);
2722         mark_dirty(svm->vmcb, VMCB_NPT);
2723         svm_flush_tlb(vcpu, true);
2724 }
2725
2726 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2727                                        struct x86_exception *fault)
2728 {
2729         struct vcpu_svm *svm = to_svm(vcpu);
2730
2731         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2732                 /*
2733                  * TODO: track the cause of the nested page fault, and
2734                  * correctly fill in the high bits of exit_info_1.
2735                  */
2736                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2737                 svm->vmcb->control.exit_code_hi = 0;
2738                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2739                 svm->vmcb->control.exit_info_2 = fault->address;
2740         }
2741
2742         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2743         svm->vmcb->control.exit_info_1 |= fault->error_code;
2744
2745         /*
2746          * The present bit is always zero for page structure faults on real
2747          * hardware.
2748          */
2749         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2750                 svm->vmcb->control.exit_info_1 &= ~1;
2751
2752         nested_svm_vmexit(svm);
2753 }
2754
2755 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2756 {
2757         WARN_ON(mmu_is_nested(vcpu));
2758         kvm_init_shadow_mmu(vcpu);
2759         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
2760         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
2761         vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
2762         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
2763         vcpu->arch.mmu.shadow_root_level = get_npt_level(vcpu);
2764         reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
2765         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
2766 }
2767
2768 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
2769 {
2770         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
2771 }
2772
2773 static int nested_svm_check_permissions(struct vcpu_svm *svm)
2774 {
2775         if (!(svm->vcpu.arch.efer & EFER_SVME) ||
2776             !is_paging(&svm->vcpu)) {
2777                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2778                 return 1;
2779         }
2780
2781         if (svm->vmcb->save.cpl) {
2782                 kvm_inject_gp(&svm->vcpu, 0);
2783                 return 1;
2784         }
2785
2786         return 0;
2787 }
2788
2789 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
2790                                       bool has_error_code, u32 error_code)
2791 {
2792         int vmexit;
2793
2794         if (!is_guest_mode(&svm->vcpu))
2795                 return 0;
2796
2797         vmexit = nested_svm_intercept(svm);
2798         if (vmexit != NESTED_EXIT_DONE)
2799                 return 0;
2800
2801         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
2802         svm->vmcb->control.exit_code_hi = 0;
2803         svm->vmcb->control.exit_info_1 = error_code;
2804
2805         /*
2806          * FIXME: we should not write CR2 when L1 intercepts an L2 #PF exception.
2807          * The fix is to add the ancillary datum (CR2 or DR6) to structs
2808          * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6 can be
2809          * written only when inject_pending_event runs (DR6 would written here
2810          * too).  This should be conditional on a new capability---if the
2811          * capability is disabled, kvm_multiple_exception would write the
2812          * ancillary information to CR2 or DR6, for backwards ABI-compatibility.
2813          */
2814         if (svm->vcpu.arch.exception.nested_apf)
2815                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
2816         else
2817                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
2818
2819         svm->nested.exit_required = true;
2820         return vmexit;
2821 }
2822
2823 /* This function returns true if it is save to enable the irq window */
2824 static inline bool nested_svm_intr(struct vcpu_svm *svm)
2825 {
2826         if (!is_guest_mode(&svm->vcpu))
2827                 return true;
2828
2829         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2830                 return true;
2831
2832         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
2833                 return false;
2834
2835         /*
2836          * if vmexit was already requested (by intercepted exception
2837          * for instance) do not overwrite it with "external interrupt"
2838          * vmexit.
2839          */
2840         if (svm->nested.exit_required)
2841                 return false;
2842
2843         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
2844         svm->vmcb->control.exit_info_1 = 0;
2845         svm->vmcb->control.exit_info_2 = 0;
2846
2847         if (svm->nested.intercept & 1ULL) {
2848                 /*
2849                  * The #vmexit can't be emulated here directly because this
2850                  * code path runs with irqs and preemption disabled. A
2851                  * #vmexit emulation might sleep. Only signal request for
2852                  * the #vmexit here.
2853                  */
2854                 svm->nested.exit_required = true;
2855                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2856                 return false;
2857         }
2858
2859         return true;
2860 }
2861
2862 /* This function returns true if it is save to enable the nmi window */
2863 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
2864 {
2865         if (!is_guest_mode(&svm->vcpu))
2866                 return true;
2867
2868         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
2869                 return true;
2870
2871         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
2872         svm->nested.exit_required = true;
2873
2874         return false;
2875 }
2876
2877 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2878 {
2879         struct page *page;
2880
2881         might_sleep();
2882
2883         page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
2884         if (is_error_page(page))
2885                 goto error;
2886
2887         *_page = page;
2888
2889         return kmap(page);
2890
2891 error:
2892         kvm_inject_gp(&svm->vcpu, 0);
2893
2894         return NULL;
2895 }
2896
2897 static void nested_svm_unmap(struct page *page)
2898 {
2899         kunmap(page);
2900         kvm_release_page_dirty(page);
2901 }
2902
2903 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
2904 {
2905         unsigned port, size, iopm_len;
2906         u16 val, mask;
2907         u8 start_bit;
2908         u64 gpa;
2909
2910         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
2911                 return NESTED_EXIT_HOST;
2912
2913         port = svm->vmcb->control.exit_info_1 >> 16;
2914         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
2915                 SVM_IOIO_SIZE_SHIFT;
2916         gpa  = svm->nested.vmcb_iopm + (port / 8);
2917         start_bit = port % 8;
2918         iopm_len = (start_bit + size > 8) ? 2 : 1;
2919         mask = (0xf >> (4 - size)) << start_bit;
2920         val = 0;
2921
2922         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
2923                 return NESTED_EXIT_DONE;
2924
2925         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2926 }
2927
2928 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
2929 {
2930         u32 offset, msr, value;
2931         int write, mask;
2932
2933         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2934                 return NESTED_EXIT_HOST;
2935
2936         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2937         offset = svm_msrpm_offset(msr);
2938         write  = svm->vmcb->control.exit_info_1 & 1;
2939         mask   = 1 << ((2 * (msr & 0xf)) + write);
2940
2941         if (offset == MSR_INVALID)
2942                 return NESTED_EXIT_DONE;
2943
2944         /* Offset is in 32 bit units but need in 8 bit units */
2945         offset *= 4;
2946
2947         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
2948                 return NESTED_EXIT_DONE;
2949
2950         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2951 }
2952
2953 /* DB exceptions for our internal use must not cause vmexit */
2954 static int nested_svm_intercept_db(struct vcpu_svm *svm)
2955 {
2956         unsigned long dr6;
2957
2958         /* if we're not singlestepping, it's not ours */
2959         if (!svm->nmi_singlestep)
2960                 return NESTED_EXIT_DONE;
2961
2962         /* if it's not a singlestep exception, it's not ours */
2963         if (kvm_get_dr(&svm->vcpu, 6, &dr6))
2964                 return NESTED_EXIT_DONE;
2965         if (!(dr6 & DR6_BS))
2966                 return NESTED_EXIT_DONE;
2967
2968         /* if the guest is singlestepping, it should get the vmexit */
2969         if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
2970                 disable_nmi_singlestep(svm);
2971                 return NESTED_EXIT_DONE;
2972         }
2973
2974         /* it's ours, the nested hypervisor must not see this one */
2975         return NESTED_EXIT_HOST;
2976 }
2977
2978 static int nested_svm_exit_special(struct vcpu_svm *svm)
2979 {
2980         u32 exit_code = svm->vmcb->control.exit_code;
2981
2982         switch (exit_code) {
2983         case SVM_EXIT_INTR:
2984         case SVM_EXIT_NMI:
2985         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
2986                 return NESTED_EXIT_HOST;
2987         case SVM_EXIT_NPF:
2988                 /* For now we are always handling NPFs when using them */
2989                 if (npt_enabled)
2990                         return NESTED_EXIT_HOST;
2991                 break;
2992         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
2993                 /* When we're shadowing, trap PFs, but not async PF */
2994                 if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
2995                         return NESTED_EXIT_HOST;
2996                 break;
2997         default:
2998                 break;
2999         }
3000
3001         return NESTED_EXIT_CONTINUE;
3002 }
3003
3004 /*
3005  * If this function returns true, this #vmexit was already handled
3006  */
3007 static int nested_svm_intercept(struct vcpu_svm *svm)
3008 {
3009         u32 exit_code = svm->vmcb->control.exit_code;
3010         int vmexit = NESTED_EXIT_HOST;
3011
3012         switch (exit_code) {
3013         case SVM_EXIT_MSR:
3014                 vmexit = nested_svm_exit_handled_msr(svm);
3015                 break;
3016         case SVM_EXIT_IOIO:
3017                 vmexit = nested_svm_intercept_ioio(svm);
3018                 break;
3019         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
3020                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
3021                 if (svm->nested.intercept_cr & bit)
3022                         vmexit = NESTED_EXIT_DONE;
3023                 break;
3024         }
3025         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
3026                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
3027                 if (svm->nested.intercept_dr & bit)
3028                         vmexit = NESTED_EXIT_DONE;
3029                 break;
3030         }
3031         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
3032                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
3033                 if (svm->nested.intercept_exceptions & excp_bits) {
3034                         if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
3035                                 vmexit = nested_svm_intercept_db(svm);
3036                         else
3037                                 vmexit = NESTED_EXIT_DONE;
3038                 }
3039                 /* async page fault always cause vmexit */
3040                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
3041                          svm->vcpu.arch.exception.nested_apf != 0)
3042                         vmexit = NESTED_EXIT_DONE;
3043                 break;
3044         }
3045         case SVM_EXIT_ERR: {
3046                 vmexit = NESTED_EXIT_DONE;
3047                 break;
3048         }
3049         default: {
3050                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
3051                 if (svm->nested.intercept & exit_bits)
3052                         vmexit = NESTED_EXIT_DONE;
3053         }
3054         }
3055
3056         return vmexit;
3057 }
3058
3059 static int nested_svm_exit_handled(struct vcpu_svm *svm)
3060 {
3061         int vmexit;
3062
3063         vmexit = nested_svm_intercept(svm);
3064
3065         if (vmexit == NESTED_EXIT_DONE)
3066                 nested_svm_vmexit(svm);
3067
3068         return vmexit;
3069 }
3070
3071 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
3072 {
3073         struct vmcb_control_area *dst  = &dst_vmcb->control;
3074         struct vmcb_control_area *from = &from_vmcb->control;
3075
3076         dst->intercept_cr         = from->intercept_cr;
3077         dst->intercept_dr         = from->intercept_dr;
3078         dst->intercept_exceptions = from->intercept_exceptions;
3079         dst->intercept            = from->intercept;
3080         dst->iopm_base_pa         = from->iopm_base_pa;
3081         dst->msrpm_base_pa        = from->msrpm_base_pa;
3082         dst->tsc_offset           = from->tsc_offset;
3083         dst->asid                 = from->asid;
3084         dst->tlb_ctl              = from->tlb_ctl;
3085         dst->int_ctl              = from->int_ctl;
3086         dst->int_vector           = from->int_vector;
3087         dst->int_state            = from->int_state;
3088         dst->exit_code            = from->exit_code;
3089         dst->exit_code_hi         = from->exit_code_hi;
3090         dst->exit_info_1          = from->exit_info_1;
3091         dst->exit_info_2          = from->exit_info_2;
3092         dst->exit_int_info        = from->exit_int_info;
3093         dst->exit_int_info_err    = from->exit_int_info_err;
3094         dst->nested_ctl           = from->nested_ctl;
3095         dst->event_inj            = from->event_inj;
3096         dst->event_inj_err        = from->event_inj_err;
3097         dst->nested_cr3           = from->nested_cr3;
3098         dst->virt_ext              = from->virt_ext;
3099 }
3100
3101 static int nested_svm_vmexit(struct vcpu_svm *svm)
3102 {
3103         struct vmcb *nested_vmcb;
3104         struct vmcb *hsave = svm->nested.hsave;
3105         struct vmcb *vmcb = svm->vmcb;
3106         struct page *page;
3107
3108         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
3109                                        vmcb->control.exit_info_1,
3110                                        vmcb->control.exit_info_2,
3111                                        vmcb->control.exit_int_info,
3112                                        vmcb->control.exit_int_info_err,
3113                                        KVM_ISA_SVM);
3114
3115         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
3116         if (!nested_vmcb)
3117                 return 1;
3118
3119         /* Exit Guest-Mode */
3120         leave_guest_mode(&svm->vcpu);
3121         svm->nested.vmcb = 0;
3122
3123         /* Give the current vmcb to the guest */
3124         disable_gif(svm);
3125
3126         nested_vmcb->save.es     = vmcb->save.es;
3127         nested_vmcb->save.cs     = vmcb->save.cs;
3128         nested_vmcb->save.ss     = vmcb->save.ss;
3129         nested_vmcb->save.ds     = vmcb->save.ds;
3130         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
3131         nested_vmcb->save.idtr   = vmcb->save.idtr;
3132         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
3133         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
3134         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
3135         nested_vmcb->save.cr2    = vmcb->save.cr2;
3136         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
3137         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
3138         nested_vmcb->save.rip    = vmcb->save.rip;
3139         nested_vmcb->save.rsp    = vmcb->save.rsp;
3140         nested_vmcb->save.rax    = vmcb->save.rax;
3141         nested_vmcb->save.dr7    = vmcb->save.dr7;
3142         nested_vmcb->save.dr6    = vmcb->save.dr6;
3143         nested_vmcb->save.cpl    = vmcb->save.cpl;
3144
3145         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
3146         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
3147         nested_vmcb->control.int_state         = vmcb->control.int_state;
3148         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
3149         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
3150         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
3151         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
3152         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
3153         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
3154
3155         if (svm->nrips_enabled)
3156                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
3157
3158         /*
3159          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
3160          * to make sure that we do not lose injected events. So check event_inj
3161          * here and copy it to exit_int_info if it is valid.
3162          * Exit_int_info and event_inj can't be both valid because the case
3163          * below only happens on a VMRUN instruction intercept which has
3164          * no valid exit_int_info set.
3165          */
3166         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
3167                 struct vmcb_control_area *nc = &nested_vmcb->control;
3168
3169                 nc->exit_int_info     = vmcb->control.event_inj;
3170                 nc->exit_int_info_err = vmcb->control.event_inj_err;
3171         }
3172
3173         nested_vmcb->control.tlb_ctl           = 0;
3174         nested_vmcb->control.event_inj         = 0;
3175         nested_vmcb->control.event_inj_err     = 0;
3176
3177         /* We always set V_INTR_MASKING and remember the old value in hflags */
3178         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3179                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3180
3181         /* Restore the original control entries */
3182         copy_vmcb_control_area(vmcb, hsave);
3183
3184         kvm_clear_exception_queue(&svm->vcpu);
3185         kvm_clear_interrupt_queue(&svm->vcpu);
3186
3187         svm->nested.nested_cr3 = 0;
3188
3189         /* Restore selected save entries */
3190         svm->vmcb->save.es = hsave->save.es;
3191         svm->vmcb->save.cs = hsave->save.cs;
3192         svm->vmcb->save.ss = hsave->save.ss;
3193         svm->vmcb->save.ds = hsave->save.ds;
3194         svm->vmcb->save.gdtr = hsave->save.gdtr;
3195         svm->vmcb->save.idtr = hsave->save.idtr;
3196         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
3197         svm_set_efer(&svm->vcpu, hsave->save.efer);
3198         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
3199         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
3200         if (npt_enabled) {
3201                 svm->vmcb->save.cr3 = hsave->save.cr3;
3202                 svm->vcpu.arch.cr3 = hsave->save.cr3;
3203         } else {
3204                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
3205         }
3206         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
3207         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
3208         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
3209         svm->vmcb->save.dr7 = 0;
3210         svm->vmcb->save.cpl = 0;
3211         svm->vmcb->control.exit_int_info = 0;
3212
3213         mark_all_dirty(svm->vmcb);
3214
3215         nested_svm_unmap(page);
3216
3217         nested_svm_uninit_mmu_context(&svm->vcpu);
3218         kvm_mmu_reset_context(&svm->vcpu);
3219         kvm_mmu_load(&svm->vcpu);
3220
3221         return 0;
3222 }
3223
3224 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
3225 {
3226         /*
3227          * This function merges the msr permission bitmaps of kvm and the
3228          * nested vmcb. It is optimized in that it only merges the parts where
3229          * the kvm msr permission bitmap may contain zero bits
3230          */
3231         int i;
3232
3233         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3234                 return true;
3235
3236         for (i = 0; i < MSRPM_OFFSETS; i++) {
3237                 u32 value, p;
3238                 u64 offset;
3239
3240                 if (msrpm_offsets[i] == 0xffffffff)
3241                         break;
3242
3243                 p      = msrpm_offsets[i];
3244                 offset = svm->nested.vmcb_msrpm + (p * 4);
3245
3246                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
3247                         return false;
3248
3249                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
3250         }
3251
3252         svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
3253
3254         return true;
3255 }
3256
3257 static bool nested_vmcb_checks(struct vmcb *vmcb)
3258 {
3259         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
3260                 return false;
3261
3262         if (vmcb->control.asid == 0)
3263                 return false;
3264
3265         if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
3266             !npt_enabled)
3267                 return false;
3268
3269         return true;
3270 }
3271
3272 static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
3273                                  struct vmcb *nested_vmcb, struct page *page)
3274 {
3275         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
3276                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
3277         else
3278                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
3279
3280         if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
3281                 kvm_mmu_unload(&svm->vcpu);
3282                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
3283                 nested_svm_init_mmu_context(&svm->vcpu);
3284         }
3285
3286         /* Load the nested guest state */
3287         svm->vmcb->save.es = nested_vmcb->save.es;
3288         svm->vmcb->save.cs = nested_vmcb->save.cs;
3289         svm->vmcb->save.ss = nested_vmcb->save.ss;
3290         svm->vmcb->save.ds = nested_vmcb->save.ds;
3291         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
3292         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
3293         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
3294         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
3295         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
3296         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
3297         if (npt_enabled) {
3298                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
3299                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
3300         } else
3301                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
3302
3303         /* Guest paging mode is active - reset mmu */
3304         kvm_mmu_reset_context(&svm->vcpu);
3305
3306         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
3307         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
3308         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
3309         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
3310
3311         /* In case we don't even reach vcpu_run, the fields are not updated */
3312         svm->vmcb->save.rax = nested_vmcb->save.rax;
3313         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
3314         svm->vmcb->save.rip = nested_vmcb->save.rip;
3315         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
3316         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
3317         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
3318
3319         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
3320         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
3321
3322         /* cache intercepts */
3323         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
3324         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
3325         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
3326         svm->nested.intercept            = nested_vmcb->control.intercept;
3327
3328         svm_flush_tlb(&svm->vcpu, true);
3329         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
3330         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
3331                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
3332         else
3333                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
3334
3335         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
3336                 /* We only want the cr8 intercept bits of the guest */
3337                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
3338                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3339         }
3340
3341         /* We don't want to see VMMCALLs from a nested guest */
3342         clr_intercept(svm, INTERCEPT_VMMCALL);
3343
3344         svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
3345         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
3346         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
3347         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
3348         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
3349         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
3350
3351         nested_svm_unmap(page);
3352
3353         /* Enter Guest-Mode */
3354         enter_guest_mode(&svm->vcpu);
3355
3356         /*
3357          * Merge guest and host intercepts - must be called  with vcpu in
3358          * guest-mode to take affect here
3359          */
3360         recalc_intercepts(svm);
3361
3362         svm->nested.vmcb = vmcb_gpa;
3363
3364         enable_gif(svm);
3365
3366         mark_all_dirty(svm->vmcb);
3367 }
3368
3369 static bool nested_svm_vmrun(struct vcpu_svm *svm)
3370 {
3371         struct vmcb *nested_vmcb;
3372         struct vmcb *hsave = svm->nested.hsave;
3373         struct vmcb *vmcb = svm->vmcb;
3374         struct page *page;
3375         u64 vmcb_gpa;
3376
3377         vmcb_gpa = svm->vmcb->save.rax;
3378
3379         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3380         if (!nested_vmcb)
3381                 return false;
3382
3383         if (!nested_vmcb_checks(nested_vmcb)) {
3384                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
3385                 nested_vmcb->control.exit_code_hi = 0;
3386                 nested_vmcb->control.exit_info_1  = 0;
3387                 nested_vmcb->control.exit_info_2  = 0;
3388
3389                 nested_svm_unmap(page);
3390
3391                 return false;
3392         }
3393
3394         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
3395                                nested_vmcb->save.rip,
3396                                nested_vmcb->control.int_ctl,
3397                                nested_vmcb->control.event_inj,
3398                                nested_vmcb->control.nested_ctl);
3399
3400         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
3401                                     nested_vmcb->control.intercept_cr >> 16,
3402                                     nested_vmcb->control.intercept_exceptions,
3403                                     nested_vmcb->control.intercept);
3404
3405         /* Clear internal status */
3406         kvm_clear_exception_queue(&svm->vcpu);
3407         kvm_clear_interrupt_queue(&svm->vcpu);
3408
3409         /*
3410          * Save the old vmcb, so we don't need to pick what we save, but can
3411          * restore everything when a VMEXIT occurs
3412          */
3413         hsave->save.es     = vmcb->save.es;
3414         hsave->save.cs     = vmcb->save.cs;
3415         hsave->save.ss     = vmcb->save.ss;
3416         hsave->save.ds     = vmcb->save.ds;
3417         hsave->save.gdtr   = vmcb->save.gdtr;
3418         hsave->save.idtr   = vmcb->save.idtr;
3419         hsave->save.efer   = svm->vcpu.arch.efer;
3420         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
3421         hsave->save.cr4    = svm->vcpu.arch.cr4;
3422         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
3423         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
3424         hsave->save.rsp    = vmcb->save.rsp;
3425         hsave->save.rax    = vmcb->save.rax;
3426         if (npt_enabled)
3427                 hsave->save.cr3    = vmcb->save.cr3;
3428         else
3429                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
3430
3431         copy_vmcb_control_area(hsave, vmcb);
3432
3433         enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, page);
3434
3435         return true;
3436 }
3437
3438 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
3439 {
3440         to_vmcb->save.fs = from_vmcb->save.fs;
3441         to_vmcb->save.gs = from_vmcb->save.gs;
3442         to_vmcb->save.tr = from_vmcb->save.tr;
3443         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
3444         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
3445         to_vmcb->save.star = from_vmcb->save.star;
3446         to_vmcb->save.lstar = from_vmcb->save.lstar;
3447         to_vmcb->save.cstar = from_vmcb->save.cstar;
3448         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
3449         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
3450         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
3451         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
3452 }
3453
3454 static int vmload_interception(struct vcpu_svm *svm)
3455 {
3456         struct vmcb *nested_vmcb;
3457         struct page *page;
3458         int ret;
3459
3460         if (nested_svm_check_permissions(svm))
3461                 return 1;
3462
3463         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3464         if (!nested_vmcb)
3465                 return 1;
3466
3467         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3468         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3469
3470         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
3471         nested_svm_unmap(page);
3472
3473         return ret;
3474 }
3475
3476 static int vmsave_interception(struct vcpu_svm *svm)
3477 {
3478         struct vmcb *nested_vmcb;
3479         struct page *page;
3480         int ret;
3481
3482         if (nested_svm_check_permissions(svm))
3483                 return 1;
3484
3485         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3486         if (!nested_vmcb)
3487                 return 1;
3488
3489         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3490         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3491
3492         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
3493         nested_svm_unmap(page);
3494
3495         return ret;
3496 }
3497
3498 static int vmrun_interception(struct vcpu_svm *svm)
3499 {
3500         if (nested_svm_check_permissions(svm))
3501                 return 1;
3502
3503         /* Save rip after vmrun instruction */
3504         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
3505
3506         if (!nested_svm_vmrun(svm))
3507                 return 1;
3508
3509         if (!nested_svm_vmrun_msrpm(svm))
3510                 goto failed;
3511
3512         return 1;
3513
3514 failed:
3515
3516         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
3517         svm->vmcb->control.exit_code_hi = 0;
3518         svm->vmcb->control.exit_info_1  = 0;
3519         svm->vmcb->control.exit_info_2  = 0;
3520
3521         nested_svm_vmexit(svm);
3522
3523         return 1;
3524 }
3525
3526 static int stgi_interception(struct vcpu_svm *svm)
3527 {
3528         int ret;
3529
3530         if (nested_svm_check_permissions(svm))
3531                 return 1;
3532
3533         /*
3534          * If VGIF is enabled, the STGI intercept is only added to
3535          * detect the opening of the SMI/NMI window; remove it now.
3536          */
3537         if (vgif_enabled(svm))
3538                 clr_intercept(svm, INTERCEPT_STGI);
3539
3540         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3541         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3542         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3543
3544         enable_gif(svm);
3545
3546         return ret;
3547 }
3548
3549 static int clgi_interception(struct vcpu_svm *svm)
3550 {
3551         int ret;
3552
3553         if (nested_svm_check_permissions(svm))
3554                 return 1;
3555
3556         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3557         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3558
3559         disable_gif(svm);
3560
3561         /* After a CLGI no interrupts should come */
3562         if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
3563                 svm_clear_vintr(svm);
3564                 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3565                 mark_dirty(svm->vmcb, VMCB_INTR);
3566         }
3567
3568         return ret;
3569 }
3570
3571 static int invlpga_interception(struct vcpu_svm *svm)
3572 {
3573         struct kvm_vcpu *vcpu = &svm->vcpu;
3574
3575         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
3576                           kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3577
3578         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3579         kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3580
3581         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3582         return kvm_skip_emulated_instruction(&svm->vcpu);
3583 }
3584
3585 static int skinit_interception(struct vcpu_svm *svm)
3586 {
3587         trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
3588
3589         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3590         return 1;
3591 }
3592
3593 static int wbinvd_interception(struct vcpu_svm *svm)
3594 {
3595         return kvm_emulate_wbinvd(&svm->vcpu);
3596 }
3597
3598 static int xsetbv_interception(struct vcpu_svm *svm)
3599 {
3600         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
3601         u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3602
3603         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
3604                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3605                 return kvm_skip_emulated_instruction(&svm->vcpu);
3606         }
3607
3608         return 1;
3609 }
3610
3611 static int task_switch_interception(struct vcpu_svm *svm)
3612 {
3613         u16 tss_selector;
3614         int reason;
3615         int int_type = svm->vmcb->control.exit_int_info &
3616                 SVM_EXITINTINFO_TYPE_MASK;
3617         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
3618         uint32_t type =
3619                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
3620         uint32_t idt_v =
3621                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
3622         bool has_error_code = false;
3623         u32 error_code = 0;
3624
3625         tss_selector = (u16)svm->vmcb->control.exit_info_1;
3626
3627         if (svm->vmcb->control.exit_info_2 &
3628             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
3629                 reason = TASK_SWITCH_IRET;
3630         else if (svm->vmcb->control.exit_info_2 &
3631                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
3632                 reason = TASK_SWITCH_JMP;
3633         else if (idt_v)
3634                 reason = TASK_SWITCH_GATE;
3635         else
3636                 reason = TASK_SWITCH_CALL;
3637
3638         if (reason == TASK_SWITCH_GATE) {
3639                 switch (type) {
3640                 case SVM_EXITINTINFO_TYPE_NMI:
3641                         svm->vcpu.arch.nmi_injected = false;
3642                         break;
3643                 case SVM_EXITINTINFO_TYPE_EXEPT:
3644                         if (svm->vmcb->control.exit_info_2 &
3645                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
3646                                 has_error_code = true;
3647                                 error_code =
3648                                         (u32)svm->vmcb->control.exit_info_2;
3649                         }
3650                         kvm_clear_exception_queue(&svm->vcpu);
3651                         break;
3652                 case SVM_EXITINTINFO_TYPE_INTR:
3653                         kvm_clear_interrupt_queue(&svm->vcpu);
3654                         break;
3655                 default:
3656                         break;
3657                 }
3658         }
3659
3660         if (reason != TASK_SWITCH_GATE ||
3661             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
3662             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3663              (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
3664                 skip_emulated_instruction(&svm->vcpu);
3665
3666         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3667                 int_vec = -1;
3668
3669         if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3670                                 has_error_code, error_code) == EMULATE_FAIL) {
3671                 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3672                 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
3673                 svm->vcpu.run->internal.ndata = 0;
3674                 return 0;
3675         }
3676         return 1;
3677 }
3678
3679 static int cpuid_interception(struct vcpu_svm *svm)
3680 {
3681         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3682         return kvm_emulate_cpuid(&svm->vcpu);
3683 }
3684
3685 static int iret_interception(struct vcpu_svm *svm)
3686 {
3687         ++svm->vcpu.stat.nmi_window_exits;
3688         clr_intercept(svm, INTERCEPT_IRET);
3689         svm->vcpu.arch.hflags |= HF_IRET_MASK;
3690         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
3691         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3692         return 1;
3693 }
3694
3695 static int invlpg_interception(struct vcpu_svm *svm)
3696 {
3697         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3698                 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3699
3700         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3701         return kvm_skip_emulated_instruction(&svm->vcpu);
3702 }
3703
3704 static int emulate_on_interception(struct vcpu_svm *svm)
3705 {
3706         return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3707 }
3708
3709 static int rsm_interception(struct vcpu_svm *svm)
3710 {
3711         return x86_emulate_instruction(&svm->vcpu, 0, 0,
3712                                        rsm_ins_bytes, 2) == EMULATE_DONE;
3713 }
3714
3715 static int rdpmc_interception(struct vcpu_svm *svm)
3716 {
3717         int err;
3718
3719         if (!static_cpu_has(X86_FEATURE_NRIPS))
3720                 return emulate_on_interception(svm);
3721
3722         err = kvm_rdpmc(&svm->vcpu);
3723         return kvm_complete_insn_gp(&svm->vcpu, err);
3724 }
3725
3726 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
3727                                             unsigned long val)
3728 {
3729         unsigned long cr0 = svm->vcpu.arch.cr0;
3730         bool ret = false;
3731         u64 intercept;
3732
3733         intercept = svm->nested.intercept;
3734
3735         if (!is_guest_mode(&svm->vcpu) ||
3736             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
3737                 return false;
3738
3739         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
3740         val &= ~SVM_CR0_SELECTIVE_MASK;
3741
3742         if (cr0 ^ val) {
3743                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3744                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
3745         }
3746
3747         return ret;
3748 }
3749
3750 #define CR_VALID (1ULL << 63)
3751
3752 static int cr_interception(struct vcpu_svm *svm)
3753 {
3754         int reg, cr;
3755         unsigned long val;
3756         int err;
3757
3758         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3759                 return emulate_on_interception(svm);
3760
3761         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
3762                 return emulate_on_interception(svm);
3763
3764         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3765         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
3766                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
3767         else
3768                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
3769
3770         err = 0;
3771         if (cr >= 16) { /* mov to cr */
3772                 cr -= 16;
3773                 val = kvm_register_read(&svm->vcpu, reg);
3774                 switch (cr) {
3775                 case 0:
3776                         if (!check_selective_cr0_intercepted(svm, val))
3777                                 err = kvm_set_cr0(&svm->vcpu, val);
3778                         else
3779                                 return 1;
3780
3781                         break;
3782                 case 3:
3783                         err = kvm_set_cr3(&svm->vcpu, val);
3784                         break;
3785                 case 4:
3786                         err = kvm_set_cr4(&svm->vcpu, val);
3787                         break;
3788                 case 8:
3789                         err = kvm_set_cr8(&svm->vcpu, val);
3790                         break;
3791                 default:
3792                         WARN(1, "unhandled write to CR%d", cr);
3793                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3794                         return 1;
3795                 }
3796         } else { /* mov from cr */
3797                 switch (cr) {
3798                 case 0:
3799                         val = kvm_read_cr0(&svm->vcpu);
3800                         break;
3801                 case 2:
3802                         val = svm->vcpu.arch.cr2;
3803                         break;
3804                 case 3:
3805                         val = kvm_read_cr3(&svm->vcpu);
3806                         break;
3807                 case 4:
3808                         val = kvm_read_cr4(&svm->vcpu);
3809                         break;
3810                 case 8:
3811                         val = kvm_get_cr8(&svm->vcpu);
3812                         break;
3813                 default:
3814                         WARN(1, "unhandled read from CR%d", cr);
3815                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3816                         return 1;
3817                 }
3818                 kvm_register_write(&svm->vcpu, reg, val);
3819         }
3820         return kvm_complete_insn_gp(&svm->vcpu, err);
3821 }
3822
3823 static int dr_interception(struct vcpu_svm *svm)
3824 {
3825         int reg, dr;
3826         unsigned long val;
3827
3828         if (svm->vcpu.guest_debug == 0) {
3829                 /*
3830                  * No more DR vmexits; force a reload of the debug registers
3831                  * and reenter on this instruction.  The next vmexit will
3832                  * retrieve the full state of the debug registers.
3833                  */
3834                 clr_dr_intercepts(svm);
3835                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
3836                 return 1;
3837         }
3838
3839         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
3840                 return emulate_on_interception(svm);
3841
3842         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3843         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
3844
3845         if (dr >= 16) { /* mov to DRn */
3846                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
3847                         return 1;
3848                 val = kvm_register_read(&svm->vcpu, reg);
3849                 kvm_set_dr(&svm->vcpu, dr - 16, val);
3850         } else {
3851                 if (!kvm_require_dr(&svm->vcpu, dr))
3852                         return 1;
3853                 kvm_get_dr(&svm->vcpu, dr, &val);
3854                 kvm_register_write(&svm->vcpu, reg, val);
3855         }
3856
3857         return kvm_skip_emulated_instruction(&svm->vcpu);
3858 }
3859
3860 static int cr8_write_interception(struct vcpu_svm *svm)
3861 {
3862         struct kvm_run *kvm_run = svm->vcpu.run;
3863         int r;
3864
3865         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
3866         /* instruction emulation calls kvm_set_cr8() */
3867         r = cr_interception(svm);
3868         if (lapic_in_kernel(&svm->vcpu))
3869                 return r;
3870         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
3871                 return r;
3872         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
3873         return 0;
3874 }
3875
3876 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
3877 {
3878         msr->data = 0;
3879
3880         switch (msr->index) {
3881         case MSR_F10H_DECFG:
3882                 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
3883                         msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
3884                 break;
3885         default:
3886                 return 1;
3887         }
3888
3889         return 0;
3890 }
3891
3892 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3893 {
3894         struct vcpu_svm *svm = to_svm(vcpu);
3895
3896         switch (msr_info->index) {
3897         case MSR_IA32_TSC: {
3898                 msr_info->data = svm->vmcb->control.tsc_offset +
3899                         kvm_scale_tsc(vcpu, rdtsc());
3900
3901                 break;
3902         }
3903         case MSR_STAR:
3904                 msr_info->data = svm->vmcb->save.star;
3905                 break;
3906 #ifdef CONFIG_X86_64
3907         case MSR_LSTAR:
3908                 msr_info->data = svm->vmcb->save.lstar;
3909                 break;
3910         case MSR_CSTAR:
3911                 msr_info->data = svm->vmcb->save.cstar;
3912                 break;
3913         case MSR_KERNEL_GS_BASE:
3914                 msr_info->data = svm->vmcb->save.kernel_gs_base;
3915                 break;
3916         case MSR_SYSCALL_MASK:
3917                 msr_info->data = svm->vmcb->save.sfmask;
3918                 break;
3919 #endif
3920         case MSR_IA32_SYSENTER_CS:
3921                 msr_info->data = svm->vmcb->save.sysenter_cs;
3922                 break;
3923         case MSR_IA32_SYSENTER_EIP:
3924                 msr_info->data = svm->sysenter_eip;
3925                 break;
3926         case MSR_IA32_SYSENTER_ESP:
3927                 msr_info->data = svm->sysenter_esp;
3928                 break;
3929         case MSR_TSC_AUX:
3930                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3931                         return 1;
3932                 msr_info->data = svm->tsc_aux;
3933                 break;
3934         /*
3935          * Nobody will change the following 5 values in the VMCB so we can
3936          * safely return them on rdmsr. They will always be 0 until LBRV is
3937          * implemented.
3938          */
3939         case MSR_IA32_DEBUGCTLMSR:
3940                 msr_info->data = svm->vmcb->save.dbgctl;
3941                 break;
3942         case MSR_IA32_LASTBRANCHFROMIP:
3943                 msr_info->data = svm->vmcb->save.br_from;
3944                 break;
3945         case MSR_IA32_LASTBRANCHTOIP:
3946                 msr_info->data = svm->vmcb->save.br_to;
3947                 break;
3948         case MSR_IA32_LASTINTFROMIP:
3949                 msr_info->data = svm->vmcb->save.last_excp_from;
3950                 break;
3951         case MSR_IA32_LASTINTTOIP:
3952                 msr_info->data = svm->vmcb->save.last_excp_to;
3953                 break;
3954         case MSR_VM_HSAVE_PA:
3955                 msr_info->data = svm->nested.hsave_msr;
3956                 break;
3957         case MSR_VM_CR:
3958                 msr_info->data = svm->nested.vm_cr_msr;
3959                 break;
3960         case MSR_IA32_SPEC_CTRL:
3961                 if (!msr_info->host_initiated &&
3962                     !guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
3963                         return 1;
3964
3965                 msr_info->data = svm->spec_ctrl;
3966                 break;
3967         case MSR_F15H_IC_CFG: {
3968
3969                 int family, model;
3970
3971                 family = guest_cpuid_family(vcpu);
3972                 model  = guest_cpuid_model(vcpu);
3973
3974                 if (family < 0 || model < 0)
3975                         return kvm_get_msr_common(vcpu, msr_info);
3976
3977                 msr_info->data = 0;
3978
3979                 if (family == 0x15 &&
3980                     (model >= 0x2 && model < 0x20))
3981                         msr_info->data = 0x1E;
3982                 }
3983                 break;
3984         case MSR_F10H_DECFG:
3985                 msr_info->data = svm->msr_decfg;
3986                 break;
3987         default:
3988                 return kvm_get_msr_common(vcpu, msr_info);
3989         }
3990         return 0;
3991 }
3992
3993 static int rdmsr_interception(struct vcpu_svm *svm)
3994 {
3995         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3996         struct msr_data msr_info;
3997
3998         msr_info.index = ecx;
3999         msr_info.host_initiated = false;
4000         if (svm_get_msr(&svm->vcpu, &msr_info)) {
4001                 trace_kvm_msr_read_ex(ecx);
4002                 kvm_inject_gp(&svm->vcpu, 0);
4003                 return 1;
4004         } else {
4005                 trace_kvm_msr_read(ecx, msr_info.data);
4006
4007                 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
4008                                    msr_info.data & 0xffffffff);
4009                 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
4010                                    msr_info.data >> 32);
4011                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4012                 return kvm_skip_emulated_instruction(&svm->vcpu);
4013         }
4014 }
4015
4016 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
4017 {
4018         struct vcpu_svm *svm = to_svm(vcpu);
4019         int svm_dis, chg_mask;
4020
4021         if (data & ~SVM_VM_CR_VALID_MASK)
4022                 return 1;
4023
4024         chg_mask = SVM_VM_CR_VALID_MASK;
4025
4026         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
4027                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
4028
4029         svm->nested.vm_cr_msr &= ~chg_mask;
4030         svm->nested.vm_cr_msr |= (data & chg_mask);
4031
4032         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
4033
4034         /* check for svm_disable while efer.svme is set */
4035         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
4036                 return 1;
4037
4038         return 0;
4039 }
4040
4041 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
4042 {
4043         struct vcpu_svm *svm = to_svm(vcpu);
4044
4045         u32 ecx = msr->index;
4046         u64 data = msr->data;
4047         switch (ecx) {
4048         case MSR_IA32_CR_PAT:
4049                 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
4050                         return 1;
4051                 vcpu->arch.pat = data;
4052                 svm->vmcb->save.g_pat = data;
4053                 mark_dirty(svm->vmcb, VMCB_NPT);
4054                 break;
4055         case MSR_IA32_TSC:
4056                 kvm_write_tsc(vcpu, msr);
4057                 break;
4058         case MSR_IA32_SPEC_CTRL:
4059                 if (!msr->host_initiated &&
4060                     !guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
4061                         return 1;
4062
4063                 /* The STIBP bit doesn't fault even if it's not advertised */
4064                 if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP))
4065                         return 1;
4066
4067                 svm->spec_ctrl = data;
4068
4069                 if (!data)
4070                         break;
4071
4072                 /*
4073                  * For non-nested:
4074                  * When it's written (to non-zero) for the first time, pass
4075                  * it through.
4076                  *
4077                  * For nested:
4078                  * The handling of the MSR bitmap for L2 guests is done in
4079                  * nested_svm_vmrun_msrpm.
4080                  * We update the L1 MSR bit as well since it will end up
4081                  * touching the MSR anyway now.
4082                  */
4083                 set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
4084                 break;
4085         case MSR_IA32_PRED_CMD:
4086                 if (!msr->host_initiated &&
4087                     !guest_cpuid_has(vcpu, X86_FEATURE_IBPB))
4088                         return 1;
4089
4090                 if (data & ~PRED_CMD_IBPB)
4091                         return 1;
4092
4093                 if (!data)
4094                         break;
4095
4096                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
4097                 if (is_guest_mode(vcpu))
4098                         break;
4099                 set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
4100                 break;
4101         case MSR_STAR:
4102                 svm->vmcb->save.star = data;
4103                 break;
4104 #ifdef CONFIG_X86_64
4105         case MSR_LSTAR:
4106                 svm->vmcb->save.lstar = data;
4107                 break;
4108         case MSR_CSTAR:
4109                 svm->vmcb->save.cstar = data;
4110                 break;
4111         case MSR_KERNEL_GS_BASE:
4112                 svm->vmcb->save.kernel_gs_base = data;
4113                 break;
4114         case MSR_SYSCALL_MASK:
4115                 svm->vmcb->save.sfmask = data;
4116                 break;
4117 #endif
4118         case MSR_IA32_SYSENTER_CS:
4119                 svm->vmcb->save.sysenter_cs = data;
4120                 break;
4121         case MSR_IA32_SYSENTER_EIP:
4122                 svm->sysenter_eip = data;
4123                 svm->vmcb->save.sysenter_eip = data;
4124                 break;
4125         case MSR_IA32_SYSENTER_ESP:
4126                 svm->sysenter_esp = data;
4127                 svm->vmcb->save.sysenter_esp = data;
4128                 break;
4129         case MSR_TSC_AUX:
4130                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4131                         return 1;
4132
4133                 /*
4134                  * This is rare, so we update the MSR here instead of using
4135                  * direct_access_msrs.  Doing that would require a rdmsr in
4136                  * svm_vcpu_put.
4137                  */
4138                 svm->tsc_aux = data;
4139                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
4140                 break;
4141         case MSR_IA32_DEBUGCTLMSR:
4142                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
4143                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
4144                                     __func__, data);
4145                         break;
4146                 }
4147                 if (data & DEBUGCTL_RESERVED_BITS)
4148                         return 1;
4149
4150                 svm->vmcb->save.dbgctl = data;
4151                 mark_dirty(svm->vmcb, VMCB_LBR);
4152                 if (data & (1ULL<<0))
4153                         svm_enable_lbrv(svm);
4154                 else
4155                         svm_disable_lbrv(svm);
4156                 break;
4157         case MSR_VM_HSAVE_PA:
4158                 svm->nested.hsave_msr = data;
4159                 break;
4160         case MSR_VM_CR:
4161                 return svm_set_vm_cr(vcpu, data);
4162         case MSR_VM_IGNNE:
4163                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
4164                 break;
4165         case MSR_F10H_DECFG: {
4166                 struct kvm_msr_entry msr_entry;
4167
4168                 msr_entry.index = msr->index;
4169                 if (svm_get_msr_feature(&msr_entry))
4170                         return 1;
4171
4172                 /* Check the supported bits */
4173                 if (data & ~msr_entry.data)
4174                         return 1;
4175
4176                 /* Don't allow the guest to change a bit, #GP */
4177                 if (!msr->host_initiated && (data ^ msr_entry.data))
4178                         return 1;
4179
4180                 svm->msr_decfg = data;
4181                 break;
4182         }
4183         case MSR_IA32_APICBASE:
4184                 if (kvm_vcpu_apicv_active(vcpu))
4185                         avic_update_vapic_bar(to_svm(vcpu), data);
4186                 /* Follow through */
4187         default:
4188                 return kvm_set_msr_common(vcpu, msr);
4189         }
4190         return 0;
4191 }
4192
4193 static int wrmsr_interception(struct vcpu_svm *svm)
4194 {
4195         struct msr_data msr;
4196         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
4197         u64 data = kvm_read_edx_eax(&svm->vcpu);
4198
4199         msr.data = data;
4200         msr.index = ecx;
4201         msr.host_initiated = false;
4202
4203         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4204         if (kvm_set_msr(&svm->vcpu, &msr)) {
4205                 trace_kvm_msr_write_ex(ecx, data);
4206                 kvm_inject_gp(&svm->vcpu, 0);
4207                 return 1;
4208         } else {
4209                 trace_kvm_msr_write(ecx, data);
4210                 return kvm_skip_emulated_instruction(&svm->vcpu);
4211         }
4212 }
4213
4214 static int msr_interception(struct vcpu_svm *svm)
4215 {
4216         if (svm->vmcb->control.exit_info_1)
4217                 return wrmsr_interception(svm);
4218         else
4219                 return rdmsr_interception(svm);
4220 }
4221
4222 static int interrupt_window_interception(struct vcpu_svm *svm)
4223 {
4224         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4225         svm_clear_vintr(svm);
4226         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
4227         mark_dirty(svm->vmcb, VMCB_INTR);
4228         ++svm->vcpu.stat.irq_window_exits;
4229         return 1;
4230 }
4231
4232 static int pause_interception(struct vcpu_svm *svm)
4233 {
4234         struct kvm_vcpu *vcpu = &svm->vcpu;
4235         bool in_kernel = (svm_get_cpl(vcpu) == 0);
4236
4237         kvm_vcpu_on_spin(vcpu, in_kernel);
4238         return 1;
4239 }
4240
4241 static int nop_interception(struct vcpu_svm *svm)
4242 {
4243         return kvm_skip_emulated_instruction(&(svm->vcpu));
4244 }
4245
4246 static int monitor_interception(struct vcpu_svm *svm)
4247 {
4248         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
4249         return nop_interception(svm);
4250 }
4251
4252 static int mwait_interception(struct vcpu_svm *svm)
4253 {
4254         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
4255         return nop_interception(svm);
4256 }
4257
4258 enum avic_ipi_failure_cause {
4259         AVIC_IPI_FAILURE_INVALID_INT_TYPE,
4260         AVIC_IPI_FAILURE_TARGET_NOT_RUNNING,
4261         AVIC_IPI_FAILURE_INVALID_TARGET,
4262         AVIC_IPI_FAILURE_INVALID_BACKING_PAGE,
4263 };
4264
4265 static int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
4266 {
4267         u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
4268         u32 icrl = svm->vmcb->control.exit_info_1;
4269         u32 id = svm->vmcb->control.exit_info_2 >> 32;
4270         u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
4271         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4272
4273         trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index);
4274
4275         switch (id) {
4276         case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
4277                 /*
4278                  * AVIC hardware handles the generation of
4279                  * IPIs when the specified Message Type is Fixed
4280                  * (also known as fixed delivery mode) and
4281                  * the Trigger Mode is edge-triggered. The hardware
4282                  * also supports self and broadcast delivery modes
4283                  * specified via the Destination Shorthand(DSH)
4284                  * field of the ICRL. Logical and physical APIC ID
4285                  * formats are supported. All other IPI types cause
4286                  * a #VMEXIT, which needs to emulated.
4287                  */
4288                 kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
4289                 kvm_lapic_reg_write(apic, APIC_ICR, icrl);
4290                 break;
4291         case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
4292                 int i;
4293                 struct kvm_vcpu *vcpu;
4294                 struct kvm *kvm = svm->vcpu.kvm;
4295                 struct kvm_lapic *apic = svm->vcpu.arch.apic;
4296
4297                 /*
4298                  * At this point, we expect that the AVIC HW has already
4299                  * set the appropriate IRR bits on the valid target
4300                  * vcpus. So, we just need to kick the appropriate vcpu.
4301                  */
4302                 kvm_for_each_vcpu(i, vcpu, kvm) {
4303                         bool m = kvm_apic_match_dest(vcpu, apic,
4304                                                      icrl & KVM_APIC_SHORT_MASK,
4305                                                      GET_APIC_DEST_FIELD(icrh),
4306                                                      icrl & KVM_APIC_DEST_MASK);
4307
4308                         if (m && !avic_vcpu_is_running(vcpu))
4309                                 kvm_vcpu_wake_up(vcpu);
4310                 }
4311                 break;
4312         }
4313         case AVIC_IPI_FAILURE_INVALID_TARGET:
4314                 break;
4315         case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
4316                 WARN_ONCE(1, "Invalid backing page\n");
4317                 break;
4318         default:
4319                 pr_err("Unknown IPI interception\n");
4320         }
4321
4322         return 1;
4323 }
4324
4325 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
4326 {
4327         struct kvm_arch *vm_data = &vcpu->kvm->arch;
4328         int index;
4329         u32 *logical_apic_id_table;
4330         int dlid = GET_APIC_LOGICAL_ID(ldr);
4331
4332         if (!dlid)
4333                 return NULL;
4334
4335         if (flat) { /* flat */
4336                 index = ffs(dlid) - 1;
4337                 if (index > 7)
4338                         return NULL;
4339         } else { /* cluster */
4340                 int cluster = (dlid & 0xf0) >> 4;
4341                 int apic = ffs(dlid & 0x0f) - 1;
4342
4343                 if ((apic < 0) || (apic > 7) ||
4344                     (cluster >= 0xf))
4345                         return NULL;
4346                 index = (cluster << 2) + apic;
4347         }
4348
4349         logical_apic_id_table = (u32 *) page_address(vm_data->avic_logical_id_table_page);
4350
4351         return &logical_apic_id_table[index];
4352 }
4353
4354 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr,
4355                           bool valid)
4356 {
4357         bool flat;
4358         u32 *entry, new_entry;
4359
4360         flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
4361         entry = avic_get_logical_id_entry(vcpu, ldr, flat);
4362         if (!entry)
4363                 return -EINVAL;
4364
4365         new_entry = READ_ONCE(*entry);
4366         new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
4367         new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
4368         if (valid)
4369                 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
4370         else
4371                 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
4372         WRITE_ONCE(*entry, new_entry);
4373
4374         return 0;
4375 }
4376
4377 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
4378 {
4379         int ret;
4380         struct vcpu_svm *svm = to_svm(vcpu);
4381         u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
4382
4383         if (!ldr)
4384                 return 1;
4385
4386         ret = avic_ldr_write(vcpu, vcpu->vcpu_id, ldr, true);
4387         if (ret && svm->ldr_reg) {
4388                 avic_ldr_write(vcpu, 0, svm->ldr_reg, false);
4389                 svm->ldr_reg = 0;
4390         } else {
4391                 svm->ldr_reg = ldr;
4392         }
4393         return ret;
4394 }
4395
4396 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
4397 {
4398         u64 *old, *new;
4399         struct vcpu_svm *svm = to_svm(vcpu);
4400         u32 apic_id_reg = kvm_lapic_get_reg(vcpu->arch.apic, APIC_ID);
4401         u32 id = (apic_id_reg >> 24) & 0xff;
4402
4403         if (vcpu->vcpu_id == id)
4404                 return 0;
4405
4406         old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
4407         new = avic_get_physical_id_entry(vcpu, id);
4408         if (!new || !old)
4409                 return 1;
4410
4411         /* We need to move physical_id_entry to new offset */
4412         *new = *old;
4413         *old = 0ULL;
4414         to_svm(vcpu)->avic_physical_id_cache = new;
4415
4416         /*
4417          * Also update the guest physical APIC ID in the logical
4418          * APIC ID table entry if already setup the LDR.
4419          */
4420         if (svm->ldr_reg)
4421                 avic_handle_ldr_update(vcpu);
4422
4423         return 0;
4424 }
4425
4426 static int avic_handle_dfr_update(struct kvm_vcpu *vcpu)
4427 {
4428         struct vcpu_svm *svm = to_svm(vcpu);
4429         struct kvm_arch *vm_data = &vcpu->kvm->arch;
4430         u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
4431         u32 mod = (dfr >> 28) & 0xf;
4432
4433         /*
4434          * We assume that all local APICs are using the same type.
4435          * If this changes, we need to flush the AVIC logical
4436          * APID id table.
4437          */
4438         if (vm_data->ldr_mode == mod)
4439                 return 0;
4440
4441         clear_page(page_address(vm_data->avic_logical_id_table_page));
4442         vm_data->ldr_mode = mod;
4443
4444         if (svm->ldr_reg)
4445                 avic_handle_ldr_update(vcpu);
4446         return 0;
4447 }
4448
4449 static int avic_unaccel_trap_write(struct vcpu_svm *svm)
4450 {
4451         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4452         u32 offset = svm->vmcb->control.exit_info_1 &
4453                                 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4454
4455         switch (offset) {
4456         case APIC_ID:
4457                 if (avic_handle_apic_id_update(&svm->vcpu))
4458                         return 0;
4459                 break;
4460         case APIC_LDR:
4461                 if (avic_handle_ldr_update(&svm->vcpu))
4462                         return 0;
4463                 break;
4464         case APIC_DFR:
4465                 avic_handle_dfr_update(&svm->vcpu);
4466                 break;
4467         default:
4468                 break;
4469         }
4470
4471         kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));
4472
4473         return 1;
4474 }
4475
4476 static bool is_avic_unaccelerated_access_trap(u32 offset)
4477 {
4478         bool ret = false;
4479
4480         switch (offset) {
4481         case APIC_ID:
4482         case APIC_EOI:
4483         case APIC_RRR:
4484         case APIC_LDR:
4485         case APIC_DFR:
4486         case APIC_SPIV:
4487         case APIC_ESR:
4488         case APIC_ICR:
4489         case APIC_LVTT:
4490         case APIC_LVTTHMR:
4491         case APIC_LVTPC:
4492         case APIC_LVT0:
4493         case APIC_LVT1:
4494         case APIC_LVTERR:
4495         case APIC_TMICT:
4496         case APIC_TDCR:
4497                 ret = true;
4498                 break;
4499         default:
4500                 break;
4501         }
4502         return ret;
4503 }
4504
4505 static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
4506 {
4507         int ret = 0;
4508         u32 offset = svm->vmcb->control.exit_info_1 &
4509                      AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4510         u32 vector = svm->vmcb->control.exit_info_2 &
4511                      AVIC_UNACCEL_ACCESS_VECTOR_MASK;
4512         bool write = (svm->vmcb->control.exit_info_1 >> 32) &
4513                      AVIC_UNACCEL_ACCESS_WRITE_MASK;
4514         bool trap = is_avic_unaccelerated_access_trap(offset);
4515
4516         trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset,
4517                                             trap, write, vector);
4518         if (trap) {
4519                 /* Handling Trap */
4520                 WARN_ONCE(!write, "svm: Handling trap read.\n");
4521                 ret = avic_unaccel_trap_write(svm);
4522         } else {
4523                 /* Handling Fault */
4524                 ret = (emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE);
4525         }
4526
4527         return ret;
4528 }
4529
4530 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
4531         [SVM_EXIT_READ_CR0]                     = cr_interception,
4532         [SVM_EXIT_READ_CR3]                     = cr_interception,
4533         [SVM_EXIT_READ_CR4]                     = cr_interception,
4534         [SVM_EXIT_READ_CR8]                     = cr_interception,
4535         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
4536         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
4537         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
4538         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
4539         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
4540         [SVM_EXIT_READ_DR0]                     = dr_interception,
4541         [SVM_EXIT_READ_DR1]                     = dr_interception,
4542         [SVM_EXIT_READ_DR2]                     = dr_interception,
4543         [SVM_EXIT_READ_DR3]                     = dr_interception,
4544         [SVM_EXIT_READ_DR4]                     = dr_interception,
4545         [SVM_EXIT_READ_DR5]                     = dr_interception,
4546         [SVM_EXIT_READ_DR6]                     = dr_interception,
4547         [SVM_EXIT_READ_DR7]                     = dr_interception,
4548         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
4549         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
4550         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
4551         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
4552         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
4553         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
4554         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
4555         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
4556         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
4557         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
4558         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
4559         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
4560         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
4561         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
4562         [SVM_EXIT_INTR]                         = intr_interception,
4563         [SVM_EXIT_NMI]                          = nmi_interception,
4564         [SVM_EXIT_SMI]                          = nop_on_interception,
4565         [SVM_EXIT_INIT]                         = nop_on_interception,
4566         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
4567         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
4568         [SVM_EXIT_CPUID]                        = cpuid_interception,
4569         [SVM_EXIT_IRET]                         = iret_interception,
4570         [SVM_EXIT_INVD]                         = emulate_on_interception,
4571         [SVM_EXIT_PAUSE]                        = pause_interception,
4572         [SVM_EXIT_HLT]                          = halt_interception,
4573         [SVM_EXIT_INVLPG]                       = invlpg_interception,
4574         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
4575         [SVM_EXIT_IOIO]                         = io_interception,
4576         [SVM_EXIT_MSR]                          = msr_interception,
4577         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
4578         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
4579         [SVM_EXIT_VMRUN]                        = vmrun_interception,
4580         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
4581         [SVM_EXIT_VMLOAD]                       = vmload_interception,
4582         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
4583         [SVM_EXIT_STGI]                         = stgi_interception,
4584         [SVM_EXIT_CLGI]                         = clgi_interception,
4585         [SVM_EXIT_SKINIT]                       = skinit_interception,
4586         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
4587         [SVM_EXIT_MONITOR]                      = monitor_interception,
4588         [SVM_EXIT_MWAIT]                        = mwait_interception,
4589         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
4590         [SVM_EXIT_NPF]                          = npf_interception,
4591         [SVM_EXIT_RSM]                          = rsm_interception,
4592         [SVM_EXIT_AVIC_INCOMPLETE_IPI]          = avic_incomplete_ipi_interception,
4593         [SVM_EXIT_AVIC_UNACCELERATED_ACCESS]    = avic_unaccelerated_access_interception,
4594 };
4595
4596 static void dump_vmcb(struct kvm_vcpu *vcpu)
4597 {
4598         struct vcpu_svm *svm = to_svm(vcpu);
4599         struct vmcb_control_area *control = &svm->vmcb->control;
4600         struct vmcb_save_area *save = &svm->vmcb->save;
4601
4602         pr_err("VMCB Control Area:\n");
4603         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
4604         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
4605         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
4606         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
4607         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
4608         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
4609         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
4610         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
4611         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
4612         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
4613         pr_err("%-20s%d\n", "asid:", control->asid);
4614         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
4615         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
4616         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
4617         pr_err("%-20s%08x\n", "int_state:", control->int_state);
4618         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
4619         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
4620         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
4621         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
4622         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
4623         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
4624         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
4625         pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
4626         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
4627         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
4628         pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
4629         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
4630         pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
4631         pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
4632         pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
4633         pr_err("VMCB State Save Area:\n");
4634         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4635                "es:",
4636                save->es.selector, save->es.attrib,
4637                save->es.limit, save->es.base);
4638         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4639                "cs:",
4640                save->cs.selector, save->cs.attrib,
4641                save->cs.limit, save->cs.base);
4642         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4643                "ss:",
4644                save->ss.selector, save->ss.attrib,
4645                save->ss.limit, save->ss.base);
4646         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4647                "ds:",
4648                save->ds.selector, save->ds.attrib,
4649                save->ds.limit, save->ds.base);
4650         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4651                "fs:",
4652                save->fs.selector, save->fs.attrib,
4653                save->fs.limit, save->fs.base);
4654         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4655                "gs:",
4656                save->gs.selector, save->gs.attrib,
4657                save->gs.limit, save->gs.base);
4658         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4659                "gdtr:",
4660                save->gdtr.selector, save->gdtr.attrib,
4661                save->gdtr.limit, save->gdtr.base);
4662         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4663                "ldtr:",
4664                save->ldtr.selector, save->ldtr.attrib,
4665                save->ldtr.limit, save->ldtr.base);
4666         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4667                "idtr:",
4668                save->idtr.selector, save->idtr.attrib,
4669                save->idtr.limit, save->idtr.base);
4670         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4671                "tr:",
4672                save->tr.selector, save->tr.attrib,
4673                save->tr.limit, save->tr.base);
4674         pr_err("cpl:            %d                efer:         %016llx\n",
4675                 save->cpl, save->efer);
4676         pr_err("%-15s %016llx %-13s %016llx\n",
4677                "cr0:", save->cr0, "cr2:", save->cr2);
4678         pr_err("%-15s %016llx %-13s %016llx\n",
4679                "cr3:", save->cr3, "cr4:", save->cr4);
4680         pr_err("%-15s %016llx %-13s %016llx\n",
4681                "dr6:", save->dr6, "dr7:", save->dr7);
4682         pr_err("%-15s %016llx %-13s %016llx\n",
4683                "rip:", save->rip, "rflags:", save->rflags);
4684         pr_err("%-15s %016llx %-13s %016llx\n",
4685                "rsp:", save->rsp, "rax:", save->rax);
4686         pr_err("%-15s %016llx %-13s %016llx\n",
4687                "star:", save->star, "lstar:", save->lstar);
4688         pr_err("%-15s %016llx %-13s %016llx\n",
4689                "cstar:", save->cstar, "sfmask:", save->sfmask);
4690         pr_err("%-15s %016llx %-13s %016llx\n",
4691                "kernel_gs_base:", save->kernel_gs_base,
4692                "sysenter_cs:", save->sysenter_cs);
4693         pr_err("%-15s %016llx %-13s %016llx\n",
4694                "sysenter_esp:", save->sysenter_esp,
4695                "sysenter_eip:", save->sysenter_eip);
4696         pr_err("%-15s %016llx %-13s %016llx\n",
4697                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
4698         pr_err("%-15s %016llx %-13s %016llx\n",
4699                "br_from:", save->br_from, "br_to:", save->br_to);
4700         pr_err("%-15s %016llx %-13s %016llx\n",
4701                "excp_from:", save->last_excp_from,
4702                "excp_to:", save->last_excp_to);
4703 }
4704
4705 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
4706 {
4707         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
4708
4709         *info1 = control->exit_info_1;
4710         *info2 = control->exit_info_2;
4711 }
4712
4713 static int handle_exit(struct kvm_vcpu *vcpu)
4714 {
4715         struct vcpu_svm *svm = to_svm(vcpu);
4716         struct kvm_run *kvm_run = vcpu->run;
4717         u32 exit_code = svm->vmcb->control.exit_code;
4718
4719         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
4720
4721         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
4722                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
4723         if (npt_enabled)
4724                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
4725
4726         if (unlikely(svm->nested.exit_required)) {
4727                 nested_svm_vmexit(svm);
4728                 svm->nested.exit_required = false;
4729
4730                 return 1;
4731         }
4732
4733         if (is_guest_mode(vcpu)) {
4734                 int vmexit;
4735
4736                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
4737                                         svm->vmcb->control.exit_info_1,
4738                                         svm->vmcb->control.exit_info_2,
4739                                         svm->vmcb->control.exit_int_info,
4740                                         svm->vmcb->control.exit_int_info_err,
4741                                         KVM_ISA_SVM);
4742
4743                 vmexit = nested_svm_exit_special(svm);
4744
4745                 if (vmexit == NESTED_EXIT_CONTINUE)
4746                         vmexit = nested_svm_exit_handled(svm);
4747
4748                 if (vmexit == NESTED_EXIT_DONE)
4749                         return 1;
4750         }
4751
4752         svm_complete_interrupts(svm);
4753
4754         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
4755                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4756                 kvm_run->fail_entry.hardware_entry_failure_reason
4757                         = svm->vmcb->control.exit_code;
4758                 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
4759                 dump_vmcb(vcpu);
4760                 return 0;
4761         }
4762
4763         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
4764             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
4765             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
4766             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
4767                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
4768                        "exit_code 0x%x\n",
4769                        __func__, svm->vmcb->control.exit_int_info,
4770                        exit_code);
4771
4772         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
4773             || !svm_exit_handlers[exit_code]) {
4774                 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
4775                 kvm_queue_exception(vcpu, UD_VECTOR);
4776                 return 1;
4777         }
4778
4779         return svm_exit_handlers[exit_code](svm);
4780 }
4781
4782 static void reload_tss(struct kvm_vcpu *vcpu)
4783 {
4784         int cpu = raw_smp_processor_id();
4785
4786         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
4787         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
4788         load_TR_desc();
4789 }
4790
4791 static void pre_sev_run(struct vcpu_svm *svm, int cpu)
4792 {
4793         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
4794         int asid = sev_get_asid(svm->vcpu.kvm);
4795
4796         /* Assign the asid allocated with this SEV guest */
4797         svm->vmcb->control.asid = asid;
4798
4799         /*
4800          * Flush guest TLB:
4801          *
4802          * 1) when different VMCB for the same ASID is to be run on the same host CPU.
4803          * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
4804          */
4805         if (sd->sev_vmcbs[asid] == svm->vmcb &&
4806             svm->last_cpu == cpu)
4807                 return;
4808
4809         svm->last_cpu = cpu;
4810         sd->sev_vmcbs[asid] = svm->vmcb;
4811         svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
4812         mark_dirty(svm->vmcb, VMCB_ASID);
4813 }
4814
4815 static void pre_svm_run(struct vcpu_svm *svm)
4816 {
4817         int cpu = raw_smp_processor_id();
4818
4819         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
4820
4821         if (sev_guest(svm->vcpu.kvm))
4822                 return pre_sev_run(svm, cpu);
4823
4824         /* FIXME: handle wraparound of asid_generation */
4825         if (svm->asid_generation != sd->asid_generation)
4826                 new_asid(svm, sd);
4827 }
4828
4829 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
4830 {
4831         struct vcpu_svm *svm = to_svm(vcpu);
4832
4833         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
4834         vcpu->arch.hflags |= HF_NMI_MASK;
4835         set_intercept(svm, INTERCEPT_IRET);
4836         ++vcpu->stat.nmi_injections;
4837 }
4838
4839 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
4840 {
4841         struct vmcb_control_area *control;
4842
4843         /* The following fields are ignored when AVIC is enabled */
4844         control = &svm->vmcb->control;
4845         control->int_vector = irq;
4846         control->int_ctl &= ~V_INTR_PRIO_MASK;
4847         control->int_ctl |= V_IRQ_MASK |
4848                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
4849         mark_dirty(svm->vmcb, VMCB_INTR);
4850 }
4851
4852 static void svm_set_irq(struct kvm_vcpu *vcpu)
4853 {
4854         struct vcpu_svm *svm = to_svm(vcpu);
4855
4856         BUG_ON(!(gif_set(svm)));
4857
4858         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
4859         ++vcpu->stat.irq_injections;
4860
4861         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
4862                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
4863 }
4864
4865 static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu)
4866 {
4867         return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK);
4868 }
4869
4870 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
4871 {
4872         struct vcpu_svm *svm = to_svm(vcpu);
4873
4874         if (svm_nested_virtualize_tpr(vcpu) ||
4875             kvm_vcpu_apicv_active(vcpu))
4876                 return;
4877
4878         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
4879
4880         if (irr == -1)
4881                 return;
4882
4883         if (tpr >= irr)
4884                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
4885 }
4886
4887 static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
4888 {
4889         return;
4890 }
4891
4892 static bool svm_get_enable_apicv(struct kvm_vcpu *vcpu)
4893 {
4894         return avic && irqchip_split(vcpu->kvm);
4895 }
4896
4897 static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
4898 {
4899 }
4900
4901 static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
4902 {
4903 }
4904
4905 /* Note: Currently only used by Hyper-V. */
4906 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
4907 {
4908         struct vcpu_svm *svm = to_svm(vcpu);
4909         struct vmcb *vmcb = svm->vmcb;
4910
4911         if (!kvm_vcpu_apicv_active(&svm->vcpu))
4912                 return;
4913
4914         vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
4915         mark_dirty(vmcb, VMCB_INTR);
4916 }
4917
4918 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
4919 {
4920         return;
4921 }
4922
4923 static void svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
4924 {
4925         kvm_lapic_set_irr(vec, vcpu->arch.apic);
4926         smp_mb__after_atomic();
4927
4928         if (avic_vcpu_is_running(vcpu))
4929                 wrmsrl(SVM_AVIC_DOORBELL,
4930                        kvm_cpu_get_apicid(vcpu->cpu));
4931         else
4932                 kvm_vcpu_wake_up(vcpu);
4933 }
4934
4935 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
4936 {
4937         unsigned long flags;
4938         struct amd_svm_iommu_ir *cur;
4939
4940         spin_lock_irqsave(&svm->ir_list_lock, flags);
4941         list_for_each_entry(cur, &svm->ir_list, node) {
4942                 if (cur->data != pi->ir_data)
4943                         continue;
4944                 list_del(&cur->node);
4945                 kfree(cur);
4946                 break;
4947         }
4948         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
4949 }
4950
4951 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
4952 {
4953         int ret = 0;
4954         unsigned long flags;
4955         struct amd_svm_iommu_ir *ir;
4956
4957         /**
4958          * In some cases, the existing irte is updaed and re-set,
4959          * so we need to check here if it's already been * added
4960          * to the ir_list.
4961          */
4962         if (pi->ir_data && (pi->prev_ga_tag != 0)) {
4963                 struct kvm *kvm = svm->vcpu.kvm;
4964                 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
4965                 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
4966                 struct vcpu_svm *prev_svm;
4967
4968                 if (!prev_vcpu) {
4969                         ret = -EINVAL;
4970                         goto out;
4971                 }
4972
4973                 prev_svm = to_svm(prev_vcpu);
4974                 svm_ir_list_del(prev_svm, pi);
4975         }
4976
4977         /**
4978          * Allocating new amd_iommu_pi_data, which will get
4979          * add to the per-vcpu ir_list.
4980          */
4981         ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL);
4982         if (!ir) {
4983                 ret = -ENOMEM;
4984                 goto out;
4985         }
4986         ir->data = pi->ir_data;
4987
4988         spin_lock_irqsave(&svm->ir_list_lock, flags);
4989         list_add(&ir->node, &svm->ir_list);
4990         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
4991 out:
4992         return ret;
4993 }
4994
4995 /**
4996  * Note:
4997  * The HW cannot support posting multicast/broadcast
4998  * interrupts to a vCPU. So, we still use legacy interrupt
4999  * remapping for these kind of interrupts.
5000  *
5001  * For lowest-priority interrupts, we only support
5002  * those with single CPU as the destination, e.g. user
5003  * configures the interrupts via /proc/irq or uses
5004  * irqbalance to make the interrupts single-CPU.
5005  */
5006 static int
5007 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
5008                  struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
5009 {
5010         struct kvm_lapic_irq irq;
5011         struct kvm_vcpu *vcpu = NULL;
5012
5013         kvm_set_msi_irq(kvm, e, &irq);
5014
5015         if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
5016                 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
5017                          __func__, irq.vector);
5018                 return -1;
5019         }
5020
5021         pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
5022                  irq.vector);
5023         *svm = to_svm(vcpu);
5024         vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
5025         vcpu_info->vector = irq.vector;
5026
5027         return 0;
5028 }
5029
5030 /*
5031  * svm_update_pi_irte - set IRTE for Posted-Interrupts
5032  *
5033  * @kvm: kvm
5034  * @host_irq: host irq of the interrupt
5035  * @guest_irq: gsi of the interrupt
5036  * @set: set or unset PI
5037  * returns 0 on success, < 0 on failure
5038  */
5039 static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
5040                               uint32_t guest_irq, bool set)
5041 {
5042         struct kvm_kernel_irq_routing_entry *e;
5043         struct kvm_irq_routing_table *irq_rt;
5044         int idx, ret = -EINVAL;
5045
5046         if (!kvm_arch_has_assigned_device(kvm) ||
5047             !irq_remapping_cap(IRQ_POSTING_CAP))
5048                 return 0;
5049
5050         pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
5051                  __func__, host_irq, guest_irq, set);
5052
5053         idx = srcu_read_lock(&kvm->irq_srcu);
5054         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
5055         WARN_ON(guest_irq >= irq_rt->nr_rt_entries);
5056
5057         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
5058                 struct vcpu_data vcpu_info;
5059                 struct vcpu_svm *svm = NULL;
5060
5061                 if (e->type != KVM_IRQ_ROUTING_MSI)
5062                         continue;
5063
5064                 /**
5065                  * Here, we setup with legacy mode in the following cases:
5066                  * 1. When cannot target interrupt to a specific vcpu.
5067                  * 2. Unsetting posted interrupt.
5068                  * 3. APIC virtialization is disabled for the vcpu.
5069                  */
5070                 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
5071                     kvm_vcpu_apicv_active(&svm->vcpu)) {
5072                         struct amd_iommu_pi_data pi;
5073
5074                         /* Try to enable guest_mode in IRTE */
5075                         pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
5076                                             AVIC_HPA_MASK);
5077                         pi.ga_tag = AVIC_GATAG(kvm->arch.avic_vm_id,
5078                                                      svm->vcpu.vcpu_id);
5079                         pi.is_guest_mode = true;
5080                         pi.vcpu_data = &vcpu_info;
5081                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5082
5083                         /**
5084                          * Here, we successfully setting up vcpu affinity in
5085                          * IOMMU guest mode. Now, we need to store the posted
5086                          * interrupt information in a per-vcpu ir_list so that
5087                          * we can reference to them directly when we update vcpu
5088                          * scheduling information in IOMMU irte.
5089                          */
5090                         if (!ret && pi.is_guest_mode)
5091                                 svm_ir_list_add(svm, &pi);
5092                 } else {
5093                         /* Use legacy mode in IRTE */
5094                         struct amd_iommu_pi_data pi;
5095
5096                         /**
5097                          * Here, pi is used to:
5098                          * - Tell IOMMU to use legacy mode for this interrupt.
5099                          * - Retrieve ga_tag of prior interrupt remapping data.
5100                          */
5101                         pi.is_guest_mode = false;
5102                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5103
5104                         /**
5105                          * Check if the posted interrupt was previously
5106                          * setup with the guest_mode by checking if the ga_tag
5107                          * was cached. If so, we need to clean up the per-vcpu
5108                          * ir_list.
5109                          */
5110                         if (!ret && pi.prev_ga_tag) {
5111                                 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
5112                                 struct kvm_vcpu *vcpu;
5113
5114                                 vcpu = kvm_get_vcpu_by_id(kvm, id);
5115                                 if (vcpu)
5116                                         svm_ir_list_del(to_svm(vcpu), &pi);
5117                         }
5118                 }
5119
5120                 if (!ret && svm) {
5121                         trace_kvm_pi_irte_update(svm->vcpu.vcpu_id,
5122                                                  host_irq, e->gsi,
5123                                                  vcpu_info.vector,
5124                                                  vcpu_info.pi_desc_addr, set);
5125                 }
5126
5127                 if (ret < 0) {
5128                         pr_err("%s: failed to update PI IRTE\n", __func__);
5129                         goto out;
5130                 }
5131         }
5132
5133         ret = 0;
5134 out:
5135         srcu_read_unlock(&kvm->irq_srcu, idx);
5136         return ret;
5137 }
5138
5139 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
5140 {
5141         struct vcpu_svm *svm = to_svm(vcpu);
5142         struct vmcb *vmcb = svm->vmcb;
5143         int ret;
5144         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
5145               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
5146         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
5147
5148         return ret;
5149 }
5150
5151 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
5152 {
5153         struct vcpu_svm *svm = to_svm(vcpu);
5154
5155         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
5156 }
5157
5158 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
5159 {
5160         struct vcpu_svm *svm = to_svm(vcpu);
5161
5162         if (masked) {
5163                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
5164                 set_intercept(svm, INTERCEPT_IRET);
5165         } else {
5166                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
5167                 clr_intercept(svm, INTERCEPT_IRET);
5168         }
5169 }
5170
5171 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
5172 {
5173         struct vcpu_svm *svm = to_svm(vcpu);
5174         struct vmcb *vmcb = svm->vmcb;
5175         int ret;
5176
5177         if (!gif_set(svm) ||
5178              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
5179                 return 0;
5180
5181         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
5182
5183         if (is_guest_mode(vcpu))
5184                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
5185
5186         return ret;
5187 }
5188
5189 static void enable_irq_window(struct kvm_vcpu *vcpu)
5190 {
5191         struct vcpu_svm *svm = to_svm(vcpu);
5192
5193         if (kvm_vcpu_apicv_active(vcpu))
5194                 return;
5195
5196         /*
5197          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
5198          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
5199          * get that intercept, this function will be called again though and
5200          * we'll get the vintr intercept. However, if the vGIF feature is
5201          * enabled, the STGI interception will not occur. Enable the irq
5202          * window under the assumption that the hardware will set the GIF.
5203          */
5204         if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) {
5205                 svm_set_vintr(svm);
5206                 svm_inject_irq(svm, 0x0);
5207         }
5208 }
5209
5210 static void enable_nmi_window(struct kvm_vcpu *vcpu)
5211 {
5212         struct vcpu_svm *svm = to_svm(vcpu);
5213
5214         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
5215             == HF_NMI_MASK)
5216                 return; /* IRET will cause a vm exit */
5217
5218         if (!gif_set(svm)) {
5219                 if (vgif_enabled(svm))
5220                         set_intercept(svm, INTERCEPT_STGI);
5221                 return; /* STGI will cause a vm exit */
5222         }
5223
5224         if (svm->nested.exit_required)
5225                 return; /* we're not going to run the guest yet */
5226
5227         /*
5228          * Something prevents NMI from been injected. Single step over possible
5229          * problem (IRET or exception injection or interrupt shadow)
5230          */
5231         svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
5232         svm->nmi_singlestep = true;
5233         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
5234 }
5235
5236 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
5237 {
5238         return 0;
5239 }
5240
5241 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
5242 {
5243         struct vcpu_svm *svm = to_svm(vcpu);
5244
5245         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
5246                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5247         else
5248                 svm->asid_generation--;
5249 }
5250
5251 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
5252 {
5253 }
5254
5255 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
5256 {
5257         struct vcpu_svm *svm = to_svm(vcpu);
5258
5259         if (svm_nested_virtualize_tpr(vcpu))
5260                 return;
5261
5262         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
5263                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
5264                 kvm_set_cr8(vcpu, cr8);
5265         }
5266 }
5267
5268 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
5269 {
5270         struct vcpu_svm *svm = to_svm(vcpu);
5271         u64 cr8;
5272
5273         if (svm_nested_virtualize_tpr(vcpu) ||
5274             kvm_vcpu_apicv_active(vcpu))
5275                 return;
5276
5277         cr8 = kvm_get_cr8(vcpu);
5278         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
5279         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
5280 }
5281
5282 static void svm_complete_interrupts(struct vcpu_svm *svm)
5283 {
5284         u8 vector;
5285         int type;
5286         u32 exitintinfo = svm->vmcb->control.exit_int_info;
5287         unsigned int3_injected = svm->int3_injected;
5288
5289         svm->int3_injected = 0;
5290
5291         /*
5292          * If we've made progress since setting HF_IRET_MASK, we've
5293          * executed an IRET and can allow NMI injection.
5294          */
5295         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
5296             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
5297                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
5298                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5299         }
5300
5301         svm->vcpu.arch.nmi_injected = false;
5302         kvm_clear_exception_queue(&svm->vcpu);
5303         kvm_clear_interrupt_queue(&svm->vcpu);
5304
5305         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
5306                 return;
5307
5308         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5309
5310         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
5311         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
5312
5313         switch (type) {
5314         case SVM_EXITINTINFO_TYPE_NMI:
5315                 svm->vcpu.arch.nmi_injected = true;
5316                 break;
5317         case SVM_EXITINTINFO_TYPE_EXEPT:
5318                 /*
5319                  * In case of software exceptions, do not reinject the vector,
5320                  * but re-execute the instruction instead. Rewind RIP first
5321                  * if we emulated INT3 before.
5322                  */
5323                 if (kvm_exception_is_soft(vector)) {
5324                         if (vector == BP_VECTOR && int3_injected &&
5325                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
5326                                 kvm_rip_write(&svm->vcpu,
5327                                               kvm_rip_read(&svm->vcpu) -
5328                                               int3_injected);
5329                         break;
5330                 }
5331                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
5332                         u32 err = svm->vmcb->control.exit_int_info_err;
5333                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
5334
5335                 } else
5336                         kvm_requeue_exception(&svm->vcpu, vector);
5337                 break;
5338         case SVM_EXITINTINFO_TYPE_INTR:
5339                 kvm_queue_interrupt(&svm->vcpu, vector, false);
5340                 break;
5341         default:
5342                 break;
5343         }
5344 }
5345
5346 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
5347 {
5348         struct vcpu_svm *svm = to_svm(vcpu);
5349         struct vmcb_control_area *control = &svm->vmcb->control;
5350
5351         control->exit_int_info = control->event_inj;
5352         control->exit_int_info_err = control->event_inj_err;
5353         control->event_inj = 0;
5354         svm_complete_interrupts(svm);
5355 }
5356
5357 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
5358 {
5359         struct vcpu_svm *svm = to_svm(vcpu);
5360
5361         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
5362         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
5363         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
5364
5365         /*
5366          * A vmexit emulation is required before the vcpu can be executed
5367          * again.
5368          */
5369         if (unlikely(svm->nested.exit_required))
5370                 return;
5371
5372         /*
5373          * Disable singlestep if we're injecting an interrupt/exception.
5374          * We don't want our modified rflags to be pushed on the stack where
5375          * we might not be able to easily reset them if we disabled NMI
5376          * singlestep later.
5377          */
5378         if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
5379                 /*
5380                  * Event injection happens before external interrupts cause a
5381                  * vmexit and interrupts are disabled here, so smp_send_reschedule
5382                  * is enough to force an immediate vmexit.
5383                  */
5384                 disable_nmi_singlestep(svm);
5385                 smp_send_reschedule(vcpu->cpu);
5386         }
5387
5388         pre_svm_run(svm);
5389
5390         sync_lapic_to_cr8(vcpu);
5391
5392         svm->vmcb->save.cr2 = vcpu->arch.cr2;
5393
5394         clgi();
5395
5396         local_irq_enable();
5397
5398         /*
5399          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
5400          * it's non-zero. Since vmentry is serialising on affected CPUs, there
5401          * is no need to worry about the conditional branch over the wrmsr
5402          * being speculatively taken.
5403          */
5404         if (svm->spec_ctrl)
5405                 native_wrmsrl(MSR_IA32_SPEC_CTRL, svm->spec_ctrl);
5406
5407         asm volatile (
5408                 "push %%" _ASM_BP "; \n\t"
5409                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
5410                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
5411                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
5412                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
5413                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
5414                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
5415 #ifdef CONFIG_X86_64
5416                 "mov %c[r8](%[svm]),  %%r8  \n\t"
5417                 "mov %c[r9](%[svm]),  %%r9  \n\t"
5418                 "mov %c[r10](%[svm]), %%r10 \n\t"
5419                 "mov %c[r11](%[svm]), %%r11 \n\t"
5420                 "mov %c[r12](%[svm]), %%r12 \n\t"
5421                 "mov %c[r13](%[svm]), %%r13 \n\t"
5422                 "mov %c[r14](%[svm]), %%r14 \n\t"
5423                 "mov %c[r15](%[svm]), %%r15 \n\t"
5424 #endif
5425
5426                 /* Enter guest mode */
5427                 "push %%" _ASM_AX " \n\t"
5428                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
5429                 __ex(SVM_VMLOAD) "\n\t"
5430                 __ex(SVM_VMRUN) "\n\t"
5431                 __ex(SVM_VMSAVE) "\n\t"
5432                 "pop %%" _ASM_AX " \n\t"
5433
5434                 /* Save guest registers, load host registers */
5435                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
5436                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
5437                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
5438                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
5439                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
5440                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
5441 #ifdef CONFIG_X86_64
5442                 "mov %%r8,  %c[r8](%[svm]) \n\t"
5443                 "mov %%r9,  %c[r9](%[svm]) \n\t"
5444                 "mov %%r10, %c[r10](%[svm]) \n\t"
5445                 "mov %%r11, %c[r11](%[svm]) \n\t"
5446                 "mov %%r12, %c[r12](%[svm]) \n\t"
5447                 "mov %%r13, %c[r13](%[svm]) \n\t"
5448                 "mov %%r14, %c[r14](%[svm]) \n\t"
5449                 "mov %%r15, %c[r15](%[svm]) \n\t"
5450 #endif
5451                 /*
5452                 * Clear host registers marked as clobbered to prevent
5453                 * speculative use.
5454                 */
5455                 "xor %%" _ASM_BX ", %%" _ASM_BX " \n\t"
5456                 "xor %%" _ASM_CX ", %%" _ASM_CX " \n\t"
5457                 "xor %%" _ASM_DX ", %%" _ASM_DX " \n\t"
5458                 "xor %%" _ASM_SI ", %%" _ASM_SI " \n\t"
5459                 "xor %%" _ASM_DI ", %%" _ASM_DI " \n\t"
5460 #ifdef CONFIG_X86_64
5461                 "xor %%r8, %%r8 \n\t"
5462                 "xor %%r9, %%r9 \n\t"
5463                 "xor %%r10, %%r10 \n\t"
5464                 "xor %%r11, %%r11 \n\t"
5465                 "xor %%r12, %%r12 \n\t"
5466                 "xor %%r13, %%r13 \n\t"
5467                 "xor %%r14, %%r14 \n\t"
5468                 "xor %%r15, %%r15 \n\t"
5469 #endif
5470                 "pop %%" _ASM_BP
5471                 :
5472                 : [svm]"a"(svm),
5473                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
5474                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
5475                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
5476                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
5477                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
5478                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
5479                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
5480 #ifdef CONFIG_X86_64
5481                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
5482                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
5483                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
5484                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
5485                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
5486                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
5487                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
5488                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
5489 #endif
5490                 : "cc", "memory"
5491 #ifdef CONFIG_X86_64
5492                 , "rbx", "rcx", "rdx", "rsi", "rdi"
5493                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
5494 #else
5495                 , "ebx", "ecx", "edx", "esi", "edi"
5496 #endif
5497                 );
5498
5499         /*
5500          * We do not use IBRS in the kernel. If this vCPU has used the
5501          * SPEC_CTRL MSR it may have left it on; save the value and
5502          * turn it off. This is much more efficient than blindly adding
5503          * it to the atomic save/restore list. Especially as the former
5504          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
5505          *
5506          * For non-nested case:
5507          * If the L01 MSR bitmap does not intercept the MSR, then we need to
5508          * save it.
5509          *
5510          * For nested case:
5511          * If the L02 MSR bitmap does not intercept the MSR, then we need to
5512          * save it.
5513          */
5514         if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
5515                 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
5516
5517         if (svm->spec_ctrl)
5518                 native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
5519
5520         /* Eliminate branch target predictions from guest mode */
5521         vmexit_fill_RSB();
5522
5523 #ifdef CONFIG_X86_64
5524         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
5525 #else
5526         loadsegment(fs, svm->host.fs);
5527 #ifndef CONFIG_X86_32_LAZY_GS
5528         loadsegment(gs, svm->host.gs);
5529 #endif
5530 #endif
5531
5532         reload_tss(vcpu);
5533
5534         local_irq_disable();
5535
5536         vcpu->arch.cr2 = svm->vmcb->save.cr2;
5537         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
5538         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
5539         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
5540
5541         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5542                 kvm_before_handle_nmi(&svm->vcpu);
5543
5544         stgi();
5545
5546         /* Any pending NMI will happen here */
5547
5548         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5549                 kvm_after_handle_nmi(&svm->vcpu);
5550
5551         sync_cr8_to_lapic(vcpu);
5552
5553         svm->next_rip = 0;
5554
5555         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
5556
5557         /* if exit due to PF check for async PF */
5558         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
5559                 svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
5560
5561         if (npt_enabled) {
5562                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
5563                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
5564         }
5565
5566         /*
5567          * We need to handle MC intercepts here before the vcpu has a chance to
5568          * change the physical cpu
5569          */
5570         if (unlikely(svm->vmcb->control.exit_code ==
5571                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
5572                 svm_handle_mce(svm);
5573
5574         mark_all_clean(svm->vmcb);
5575 }
5576 STACK_FRAME_NON_STANDARD(svm_vcpu_run);
5577
5578 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5579 {
5580         struct vcpu_svm *svm = to_svm(vcpu);
5581
5582         svm->vmcb->save.cr3 = __sme_set(root);
5583         mark_dirty(svm->vmcb, VMCB_CR);
5584         svm_flush_tlb(vcpu, true);
5585 }
5586
5587 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5588 {
5589         struct vcpu_svm *svm = to_svm(vcpu);
5590
5591         svm->vmcb->control.nested_cr3 = __sme_set(root);
5592         mark_dirty(svm->vmcb, VMCB_NPT);
5593
5594         /* Also sync guest cr3 here in case we live migrate */
5595         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
5596         mark_dirty(svm->vmcb, VMCB_CR);
5597
5598         svm_flush_tlb(vcpu, true);
5599 }
5600
5601 static int is_disabled(void)
5602 {
5603         u64 vm_cr;
5604
5605         rdmsrl(MSR_VM_CR, vm_cr);
5606         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
5607                 return 1;
5608
5609         return 0;
5610 }
5611
5612 static void
5613 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
5614 {
5615         /*
5616          * Patch in the VMMCALL instruction:
5617          */
5618         hypercall[0] = 0x0f;
5619         hypercall[1] = 0x01;
5620         hypercall[2] = 0xd9;
5621 }
5622
5623 static void svm_check_processor_compat(void *rtn)
5624 {
5625         *(int *)rtn = 0;
5626 }
5627
5628 static bool svm_cpu_has_accelerated_tpr(void)
5629 {
5630         return false;
5631 }
5632
5633 static bool svm_has_high_real_mode_segbase(void)
5634 {
5635         return true;
5636 }
5637
5638 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
5639 {
5640         return 0;
5641 }
5642
5643 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
5644 {
5645         struct vcpu_svm *svm = to_svm(vcpu);
5646
5647         /* Update nrips enabled cache */
5648         svm->nrips_enabled = !!guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
5649
5650         if (!kvm_vcpu_apicv_active(vcpu))
5651                 return;
5652
5653         guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
5654 }
5655
5656 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
5657 {
5658         switch (func) {
5659         case 0x1:
5660                 if (avic)
5661                         entry->ecx &= ~bit(X86_FEATURE_X2APIC);
5662                 break;
5663         case 0x80000001:
5664                 if (nested)
5665                         entry->ecx |= (1 << 2); /* Set SVM bit */
5666                 break;
5667         case 0x8000000A:
5668                 entry->eax = 1; /* SVM revision 1 */
5669                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
5670                                    ASID emulation to nested SVM */
5671                 entry->ecx = 0; /* Reserved */
5672                 entry->edx = 0; /* Per default do not support any
5673                                    additional features */
5674
5675                 /* Support next_rip if host supports it */
5676                 if (boot_cpu_has(X86_FEATURE_NRIPS))
5677                         entry->edx |= SVM_FEATURE_NRIP;
5678
5679                 /* Support NPT for the guest if enabled */
5680                 if (npt_enabled)
5681                         entry->edx |= SVM_FEATURE_NPT;
5682
5683                 break;
5684         case 0x8000001F:
5685                 /* Support memory encryption cpuid if host supports it */
5686                 if (boot_cpu_has(X86_FEATURE_SEV))
5687                         cpuid(0x8000001f, &entry->eax, &entry->ebx,
5688                                 &entry->ecx, &entry->edx);
5689
5690         }
5691 }
5692
5693 static int svm_get_lpage_level(void)
5694 {
5695         return PT_PDPE_LEVEL;
5696 }
5697
5698 static bool svm_rdtscp_supported(void)
5699 {
5700         return boot_cpu_has(X86_FEATURE_RDTSCP);
5701 }
5702
5703 static bool svm_invpcid_supported(void)
5704 {
5705         return false;
5706 }
5707
5708 static bool svm_mpx_supported(void)
5709 {
5710         return false;
5711 }
5712
5713 static bool svm_xsaves_supported(void)
5714 {
5715         return false;
5716 }
5717
5718 static bool svm_umip_emulated(void)
5719 {
5720         return false;
5721 }
5722
5723 static bool svm_has_wbinvd_exit(void)
5724 {
5725         return true;
5726 }
5727
5728 #define PRE_EX(exit)  { .exit_code = (exit), \
5729                         .stage = X86_ICPT_PRE_EXCEPT, }
5730 #define POST_EX(exit) { .exit_code = (exit), \
5731                         .stage = X86_ICPT_POST_EXCEPT, }
5732 #define POST_MEM(exit) { .exit_code = (exit), \
5733                         .stage = X86_ICPT_POST_MEMACCESS, }
5734
5735 static const struct __x86_intercept {
5736         u32 exit_code;
5737         enum x86_intercept_stage stage;
5738 } x86_intercept_map[] = {
5739         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
5740         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
5741         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
5742         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
5743         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
5744         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
5745         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
5746         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
5747         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
5748         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
5749         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
5750         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
5751         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
5752         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
5753         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
5754         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
5755         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
5756         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
5757         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
5758         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
5759         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
5760         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
5761         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
5762         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
5763         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
5764         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
5765         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
5766         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
5767         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
5768         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
5769         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
5770         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
5771         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
5772         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
5773         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
5774         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
5775         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
5776         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
5777         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
5778         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
5779         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
5780         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
5781         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
5782         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
5783         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
5784         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
5785 };
5786
5787 #undef PRE_EX
5788 #undef POST_EX
5789 #undef POST_MEM
5790
5791 static int svm_check_intercept(struct kvm_vcpu *vcpu,
5792                                struct x86_instruction_info *info,
5793                                enum x86_intercept_stage stage)
5794 {
5795         struct vcpu_svm *svm = to_svm(vcpu);
5796         int vmexit, ret = X86EMUL_CONTINUE;
5797         struct __x86_intercept icpt_info;
5798         struct vmcb *vmcb = svm->vmcb;
5799
5800         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
5801                 goto out;
5802
5803         icpt_info = x86_intercept_map[info->intercept];
5804
5805         if (stage != icpt_info.stage)
5806                 goto out;
5807
5808         switch (icpt_info.exit_code) {
5809         case SVM_EXIT_READ_CR0:
5810                 if (info->intercept == x86_intercept_cr_read)
5811                         icpt_info.exit_code += info->modrm_reg;
5812                 break;
5813         case SVM_EXIT_WRITE_CR0: {
5814                 unsigned long cr0, val;
5815                 u64 intercept;
5816
5817                 if (info->intercept == x86_intercept_cr_write)
5818                         icpt_info.exit_code += info->modrm_reg;
5819
5820                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
5821                     info->intercept == x86_intercept_clts)
5822                         break;
5823
5824                 intercept = svm->nested.intercept;
5825
5826                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
5827                         break;
5828
5829                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
5830                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
5831
5832                 if (info->intercept == x86_intercept_lmsw) {
5833                         cr0 &= 0xfUL;
5834                         val &= 0xfUL;
5835                         /* lmsw can't clear PE - catch this here */
5836                         if (cr0 & X86_CR0_PE)
5837                                 val |= X86_CR0_PE;
5838                 }
5839
5840                 if (cr0 ^ val)
5841                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
5842
5843                 break;
5844         }
5845         case SVM_EXIT_READ_DR0:
5846         case SVM_EXIT_WRITE_DR0:
5847                 icpt_info.exit_code += info->modrm_reg;
5848                 break;
5849         case SVM_EXIT_MSR:
5850                 if (info->intercept == x86_intercept_wrmsr)
5851                         vmcb->control.exit_info_1 = 1;
5852                 else
5853                         vmcb->control.exit_info_1 = 0;
5854                 break;
5855         case SVM_EXIT_PAUSE:
5856                 /*
5857                  * We get this for NOP only, but pause
5858                  * is rep not, check this here
5859                  */
5860                 if (info->rep_prefix != REPE_PREFIX)
5861                         goto out;
5862                 break;
5863         case SVM_EXIT_IOIO: {
5864                 u64 exit_info;
5865                 u32 bytes;
5866
5867                 if (info->intercept == x86_intercept_in ||
5868                     info->intercept == x86_intercept_ins) {
5869                         exit_info = ((info->src_val & 0xffff) << 16) |
5870                                 SVM_IOIO_TYPE_MASK;
5871                         bytes = info->dst_bytes;
5872                 } else {
5873                         exit_info = (info->dst_val & 0xffff) << 16;
5874                         bytes = info->src_bytes;
5875                 }
5876
5877                 if (info->intercept == x86_intercept_outs ||
5878                     info->intercept == x86_intercept_ins)
5879                         exit_info |= SVM_IOIO_STR_MASK;
5880
5881                 if (info->rep_prefix)
5882                         exit_info |= SVM_IOIO_REP_MASK;
5883
5884                 bytes = min(bytes, 4u);
5885
5886                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
5887
5888                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
5889
5890                 vmcb->control.exit_info_1 = exit_info;
5891                 vmcb->control.exit_info_2 = info->next_rip;
5892
5893                 break;
5894         }
5895         default:
5896                 break;
5897         }
5898
5899         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
5900         if (static_cpu_has(X86_FEATURE_NRIPS))
5901                 vmcb->control.next_rip  = info->next_rip;
5902         vmcb->control.exit_code = icpt_info.exit_code;
5903         vmexit = nested_svm_exit_handled(svm);
5904
5905         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
5906                                            : X86EMUL_CONTINUE;
5907
5908 out:
5909         return ret;
5910 }
5911
5912 static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
5913 {
5914         local_irq_enable();
5915         /*
5916          * We must have an instruction with interrupts enabled, so
5917          * the timer interrupt isn't delayed by the interrupt shadow.
5918          */
5919         asm("nop");
5920         local_irq_disable();
5921 }
5922
5923 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
5924 {
5925 }
5926
5927 static inline void avic_post_state_restore(struct kvm_vcpu *vcpu)
5928 {
5929         if (avic_handle_apic_id_update(vcpu) != 0)
5930                 return;
5931         if (avic_handle_dfr_update(vcpu) != 0)
5932                 return;
5933         avic_handle_ldr_update(vcpu);
5934 }
5935
5936 static void svm_setup_mce(struct kvm_vcpu *vcpu)
5937 {
5938         /* [63:9] are reserved. */
5939         vcpu->arch.mcg_cap &= 0x1ff;
5940 }
5941
5942 static int svm_smi_allowed(struct kvm_vcpu *vcpu)
5943 {
5944         struct vcpu_svm *svm = to_svm(vcpu);
5945
5946         /* Per APM Vol.2 15.22.2 "Response to SMI" */
5947         if (!gif_set(svm))
5948                 return 0;
5949
5950         if (is_guest_mode(&svm->vcpu) &&
5951             svm->nested.intercept & (1ULL << INTERCEPT_SMI)) {
5952                 /* TODO: Might need to set exit_info_1 and exit_info_2 here */
5953                 svm->vmcb->control.exit_code = SVM_EXIT_SMI;
5954                 svm->nested.exit_required = true;
5955                 return 0;
5956         }
5957
5958         return 1;
5959 }
5960
5961 static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
5962 {
5963         struct vcpu_svm *svm = to_svm(vcpu);
5964         int ret;
5965
5966         if (is_guest_mode(vcpu)) {
5967                 /* FED8h - SVM Guest */
5968                 put_smstate(u64, smstate, 0x7ed8, 1);
5969                 /* FEE0h - SVM Guest VMCB Physical Address */
5970                 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb);
5971
5972                 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
5973                 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
5974                 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
5975
5976                 ret = nested_svm_vmexit(svm);
5977                 if (ret)
5978                         return ret;
5979         }
5980         return 0;
5981 }
5982
5983 static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, u64 smbase)
5984 {
5985         struct vcpu_svm *svm = to_svm(vcpu);
5986         struct vmcb *nested_vmcb;
5987         struct page *page;
5988         struct {
5989                 u64 guest;
5990                 u64 vmcb;
5991         } svm_state_save;
5992         int ret;
5993
5994         ret = kvm_vcpu_read_guest(vcpu, smbase + 0xfed8, &svm_state_save,
5995                                   sizeof(svm_state_save));
5996         if (ret)
5997                 return ret;
5998
5999         if (svm_state_save.guest) {
6000                 vcpu->arch.hflags &= ~HF_SMM_MASK;
6001                 nested_vmcb = nested_svm_map(svm, svm_state_save.vmcb, &page);
6002                 if (nested_vmcb)
6003                         enter_svm_guest_mode(svm, svm_state_save.vmcb, nested_vmcb, page);
6004                 else
6005                         ret = 1;
6006                 vcpu->arch.hflags |= HF_SMM_MASK;
6007         }
6008         return ret;
6009 }
6010
6011 static int enable_smi_window(struct kvm_vcpu *vcpu)
6012 {
6013         struct vcpu_svm *svm = to_svm(vcpu);
6014
6015         if (!gif_set(svm)) {
6016                 if (vgif_enabled(svm))
6017                         set_intercept(svm, INTERCEPT_STGI);
6018                 /* STGI will cause a vm exit */
6019                 return 1;
6020         }
6021         return 0;
6022 }
6023
6024 static int sev_asid_new(void)
6025 {
6026         int pos;
6027
6028         /*
6029          * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
6030          */
6031         pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
6032         if (pos >= max_sev_asid)
6033                 return -EBUSY;
6034
6035         set_bit(pos, sev_asid_bitmap);
6036         return pos + 1;
6037 }
6038
6039 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
6040 {
6041         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6042         int asid, ret;
6043
6044         ret = -EBUSY;
6045         asid = sev_asid_new();
6046         if (asid < 0)
6047                 return ret;
6048
6049         ret = sev_platform_init(&argp->error);
6050         if (ret)
6051                 goto e_free;
6052
6053         sev->active = true;
6054         sev->asid = asid;
6055         INIT_LIST_HEAD(&sev->regions_list);
6056
6057         return 0;
6058
6059 e_free:
6060         __sev_asid_free(asid);
6061         return ret;
6062 }
6063
6064 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
6065 {
6066         struct sev_data_activate *data;
6067         int asid = sev_get_asid(kvm);
6068         int ret;
6069
6070         wbinvd_on_all_cpus();
6071
6072         ret = sev_guest_df_flush(error);
6073         if (ret)
6074                 return ret;
6075
6076         data = kzalloc(sizeof(*data), GFP_KERNEL);
6077         if (!data)
6078                 return -ENOMEM;
6079
6080         /* activate ASID on the given handle */
6081         data->handle = handle;
6082         data->asid   = asid;
6083         ret = sev_guest_activate(data, error);
6084         kfree(data);
6085
6086         return ret;
6087 }
6088
6089 static int __sev_issue_cmd(int fd, int id, void *data, int *error)
6090 {
6091         struct fd f;
6092         int ret;
6093
6094         f = fdget(fd);
6095         if (!f.file)
6096                 return -EBADF;
6097
6098         ret = sev_issue_cmd_external_user(f.file, id, data, error);
6099
6100         fdput(f);
6101         return ret;
6102 }
6103
6104 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
6105 {
6106         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6107
6108         return __sev_issue_cmd(sev->fd, id, data, error);
6109 }
6110
6111 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
6112 {
6113         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6114         struct sev_data_launch_start *start;
6115         struct kvm_sev_launch_start params;
6116         void *dh_blob, *session_blob;
6117         int *error = &argp->error;
6118         int ret;
6119
6120         if (!sev_guest(kvm))
6121                 return -ENOTTY;
6122
6123         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6124                 return -EFAULT;
6125
6126         start = kzalloc(sizeof(*start), GFP_KERNEL);
6127         if (!start)
6128                 return -ENOMEM;
6129
6130         dh_blob = NULL;
6131         if (params.dh_uaddr) {
6132                 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
6133                 if (IS_ERR(dh_blob)) {
6134                         ret = PTR_ERR(dh_blob);
6135                         goto e_free;
6136                 }
6137
6138                 start->dh_cert_address = __sme_set(__pa(dh_blob));
6139                 start->dh_cert_len = params.dh_len;
6140         }
6141
6142         session_blob = NULL;
6143         if (params.session_uaddr) {
6144                 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
6145                 if (IS_ERR(session_blob)) {
6146                         ret = PTR_ERR(session_blob);
6147                         goto e_free_dh;
6148                 }
6149
6150                 start->session_address = __sme_set(__pa(session_blob));
6151                 start->session_len = params.session_len;
6152         }
6153
6154         start->handle = params.handle;
6155         start->policy = params.policy;
6156
6157         /* create memory encryption context */
6158         ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
6159         if (ret)
6160                 goto e_free_session;
6161
6162         /* Bind ASID to this guest */
6163         ret = sev_bind_asid(kvm, start->handle, error);
6164         if (ret)
6165                 goto e_free_session;
6166
6167         /* return handle to userspace */
6168         params.handle = start->handle;
6169         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
6170                 sev_unbind_asid(kvm, start->handle);
6171                 ret = -EFAULT;
6172                 goto e_free_session;
6173         }
6174
6175         sev->handle = start->handle;
6176         sev->fd = argp->sev_fd;
6177
6178 e_free_session:
6179         kfree(session_blob);
6180 e_free_dh:
6181         kfree(dh_blob);
6182 e_free:
6183         kfree(start);
6184         return ret;
6185 }
6186
6187 static int get_num_contig_pages(int idx, struct page **inpages,
6188                                 unsigned long npages)
6189 {
6190         unsigned long paddr, next_paddr;
6191         int i = idx + 1, pages = 1;
6192
6193         /* find the number of contiguous pages starting from idx */
6194         paddr = __sme_page_pa(inpages[idx]);
6195         while (i < npages) {
6196                 next_paddr = __sme_page_pa(inpages[i++]);
6197                 if ((paddr + PAGE_SIZE) == next_paddr) {
6198                         pages++;
6199                         paddr = next_paddr;
6200                         continue;
6201                 }
6202                 break;
6203         }
6204
6205         return pages;
6206 }
6207
6208 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
6209 {
6210         unsigned long vaddr, vaddr_end, next_vaddr, npages, size;
6211         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6212         struct kvm_sev_launch_update_data params;
6213         struct sev_data_launch_update_data *data;
6214         struct page **inpages;
6215         int i, ret, pages;
6216
6217         if (!sev_guest(kvm))
6218                 return -ENOTTY;
6219
6220         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6221                 return -EFAULT;
6222
6223         data = kzalloc(sizeof(*data), GFP_KERNEL);
6224         if (!data)
6225                 return -ENOMEM;
6226
6227         vaddr = params.uaddr;
6228         size = params.len;
6229         vaddr_end = vaddr + size;
6230
6231         /* Lock the user memory. */
6232         inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
6233         if (!inpages) {
6234                 ret = -ENOMEM;
6235                 goto e_free;
6236         }
6237
6238         /*
6239          * The LAUNCH_UPDATE command will perform in-place encryption of the
6240          * memory content (i.e it will write the same memory region with C=1).
6241          * It's possible that the cache may contain the data with C=0, i.e.,
6242          * unencrypted so invalidate it first.
6243          */
6244         sev_clflush_pages(inpages, npages);
6245
6246         for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
6247                 int offset, len;
6248
6249                 /*
6250                  * If the user buffer is not page-aligned, calculate the offset
6251                  * within the page.
6252                  */
6253                 offset = vaddr & (PAGE_SIZE - 1);
6254
6255                 /* Calculate the number of pages that can be encrypted in one go. */
6256                 pages = get_num_contig_pages(i, inpages, npages);
6257
6258                 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
6259
6260                 data->handle = sev->handle;
6261                 data->len = len;
6262                 data->address = __sme_page_pa(inpages[i]) + offset;
6263                 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
6264                 if (ret)
6265                         goto e_unpin;
6266
6267                 size -= len;
6268                 next_vaddr = vaddr + len;
6269         }
6270
6271 e_unpin:
6272         /* content of memory is updated, mark pages dirty */
6273         for (i = 0; i < npages; i++) {
6274                 set_page_dirty_lock(inpages[i]);
6275                 mark_page_accessed(inpages[i]);
6276         }
6277         /* unlock the user pages */
6278         sev_unpin_memory(kvm, inpages, npages);
6279 e_free:
6280         kfree(data);
6281         return ret;
6282 }
6283
6284 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
6285 {
6286         void __user *measure = (void __user *)(uintptr_t)argp->data;
6287         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6288         struct sev_data_launch_measure *data;
6289         struct kvm_sev_launch_measure params;
6290         void __user *p = NULL;
6291         void *blob = NULL;
6292         int ret;
6293
6294         if (!sev_guest(kvm))
6295                 return -ENOTTY;
6296
6297         if (copy_from_user(&params, measure, sizeof(params)))
6298                 return -EFAULT;
6299
6300         data = kzalloc(sizeof(*data), GFP_KERNEL);
6301         if (!data)
6302                 return -ENOMEM;
6303
6304         /* User wants to query the blob length */
6305         if (!params.len)
6306                 goto cmd;
6307
6308         p = (void __user *)(uintptr_t)params.uaddr;
6309         if (p) {
6310                 if (params.len > SEV_FW_BLOB_MAX_SIZE) {
6311                         ret = -EINVAL;
6312                         goto e_free;
6313                 }
6314
6315                 ret = -ENOMEM;
6316                 blob = kmalloc(params.len, GFP_KERNEL);
6317                 if (!blob)
6318                         goto e_free;
6319
6320                 data->address = __psp_pa(blob);
6321                 data->len = params.len;
6322         }
6323
6324 cmd:
6325         data->handle = sev->handle;
6326         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
6327
6328         /*
6329          * If we query the session length, FW responded with expected data.
6330          */
6331         if (!params.len)
6332                 goto done;
6333
6334         if (ret)
6335                 goto e_free_blob;
6336
6337         if (blob) {
6338                 if (copy_to_user(p, blob, params.len))
6339                         ret = -EFAULT;
6340         }
6341
6342 done:
6343         params.len = data->len;
6344         if (copy_to_user(measure, &params, sizeof(params)))
6345                 ret = -EFAULT;
6346 e_free_blob:
6347         kfree(blob);
6348 e_free:
6349         kfree(data);
6350         return ret;
6351 }
6352
6353 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
6354 {
6355         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6356         struct sev_data_launch_finish *data;
6357         int ret;
6358
6359         if (!sev_guest(kvm))
6360                 return -ENOTTY;
6361
6362         data = kzalloc(sizeof(*data), GFP_KERNEL);
6363         if (!data)
6364                 return -ENOMEM;
6365
6366         data->handle = sev->handle;
6367         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
6368
6369         kfree(data);
6370         return ret;
6371 }
6372
6373 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
6374 {
6375         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6376         struct kvm_sev_guest_status params;
6377         struct sev_data_guest_status *data;
6378         int ret;
6379
6380         if (!sev_guest(kvm))
6381                 return -ENOTTY;
6382
6383         data = kzalloc(sizeof(*data), GFP_KERNEL);
6384         if (!data)
6385                 return -ENOMEM;
6386
6387         data->handle = sev->handle;
6388         ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
6389         if (ret)
6390                 goto e_free;
6391
6392         params.policy = data->policy;
6393         params.state = data->state;
6394         params.handle = data->handle;
6395
6396         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
6397                 ret = -EFAULT;
6398 e_free:
6399         kfree(data);
6400         return ret;
6401 }
6402
6403 static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
6404                                unsigned long dst, int size,
6405                                int *error, bool enc)
6406 {
6407         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6408         struct sev_data_dbg *data;
6409         int ret;
6410
6411         data = kzalloc(sizeof(*data), GFP_KERNEL);
6412         if (!data)
6413                 return -ENOMEM;
6414
6415         data->handle = sev->handle;
6416         data->dst_addr = dst;
6417         data->src_addr = src;
6418         data->len = size;
6419
6420         ret = sev_issue_cmd(kvm,
6421                             enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
6422                             data, error);
6423         kfree(data);
6424         return ret;
6425 }
6426
6427 static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
6428                              unsigned long dst_paddr, int sz, int *err)
6429 {
6430         int offset;
6431
6432         /*
6433          * Its safe to read more than we are asked, caller should ensure that
6434          * destination has enough space.
6435          */
6436         src_paddr = round_down(src_paddr, 16);
6437         offset = src_paddr & 15;
6438         sz = round_up(sz + offset, 16);
6439
6440         return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
6441 }
6442
6443 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
6444                                   unsigned long __user dst_uaddr,
6445                                   unsigned long dst_paddr,
6446                                   int size, int *err)
6447 {
6448         struct page *tpage = NULL;
6449         int ret, offset;
6450
6451         /* if inputs are not 16-byte then use intermediate buffer */
6452         if (!IS_ALIGNED(dst_paddr, 16) ||
6453             !IS_ALIGNED(paddr,     16) ||
6454             !IS_ALIGNED(size,      16)) {
6455                 tpage = (void *)alloc_page(GFP_KERNEL);
6456                 if (!tpage)
6457                         return -ENOMEM;
6458
6459                 dst_paddr = __sme_page_pa(tpage);
6460         }
6461
6462         ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
6463         if (ret)
6464                 goto e_free;
6465
6466         if (tpage) {
6467                 offset = paddr & 15;
6468                 if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
6469                                  page_address(tpage) + offset, size))
6470                         ret = -EFAULT;
6471         }
6472
6473 e_free:
6474         if (tpage)
6475                 __free_page(tpage);
6476
6477         return ret;
6478 }
6479
6480 static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
6481                                   unsigned long __user vaddr,
6482                                   unsigned long dst_paddr,
6483                                   unsigned long __user dst_vaddr,
6484                                   int size, int *error)
6485 {
6486         struct page *src_tpage = NULL;
6487         struct page *dst_tpage = NULL;
6488         int ret, len = size;
6489
6490         /* If source buffer is not aligned then use an intermediate buffer */
6491         if (!IS_ALIGNED(vaddr, 16)) {
6492                 src_tpage = alloc_page(GFP_KERNEL);
6493                 if (!src_tpage)
6494                         return -ENOMEM;
6495
6496                 if (copy_from_user(page_address(src_tpage),
6497                                 (void __user *)(uintptr_t)vaddr, size)) {
6498                         __free_page(src_tpage);
6499                         return -EFAULT;
6500                 }
6501
6502                 paddr = __sme_page_pa(src_tpage);
6503         }
6504
6505         /*
6506          *  If destination buffer or length is not aligned then do read-modify-write:
6507          *   - decrypt destination in an intermediate buffer
6508          *   - copy the source buffer in an intermediate buffer
6509          *   - use the intermediate buffer as source buffer
6510          */
6511         if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
6512                 int dst_offset;
6513
6514                 dst_tpage = alloc_page(GFP_KERNEL);
6515                 if (!dst_tpage) {
6516                         ret = -ENOMEM;
6517                         goto e_free;
6518                 }
6519
6520                 ret = __sev_dbg_decrypt(kvm, dst_paddr,
6521                                         __sme_page_pa(dst_tpage), size, error);
6522                 if (ret)
6523                         goto e_free;
6524
6525                 /*
6526                  *  If source is kernel buffer then use memcpy() otherwise
6527                  *  copy_from_user().
6528                  */
6529                 dst_offset = dst_paddr & 15;
6530
6531                 if (src_tpage)
6532                         memcpy(page_address(dst_tpage) + dst_offset,
6533                                page_address(src_tpage), size);
6534                 else {
6535                         if (copy_from_user(page_address(dst_tpage) + dst_offset,
6536                                            (void __user *)(uintptr_t)vaddr, size)) {
6537                                 ret = -EFAULT;
6538                                 goto e_free;
6539                         }
6540                 }
6541
6542                 paddr = __sme_page_pa(dst_tpage);
6543                 dst_paddr = round_down(dst_paddr, 16);
6544                 len = round_up(size, 16);
6545         }
6546
6547         ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
6548
6549 e_free:
6550         if (src_tpage)
6551                 __free_page(src_tpage);
6552         if (dst_tpage)
6553                 __free_page(dst_tpage);
6554         return ret;
6555 }
6556
6557 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
6558 {
6559         unsigned long vaddr, vaddr_end, next_vaddr;
6560         unsigned long dst_vaddr, dst_vaddr_end;
6561         struct page **src_p, **dst_p;
6562         struct kvm_sev_dbg debug;
6563         unsigned long n;
6564         int ret, size;
6565
6566         if (!sev_guest(kvm))
6567                 return -ENOTTY;
6568
6569         if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
6570                 return -EFAULT;
6571
6572         vaddr = debug.src_uaddr;
6573         size = debug.len;
6574         vaddr_end = vaddr + size;
6575         dst_vaddr = debug.dst_uaddr;
6576         dst_vaddr_end = dst_vaddr + size;
6577
6578         for (; vaddr < vaddr_end; vaddr = next_vaddr) {
6579                 int len, s_off, d_off;
6580
6581                 /* lock userspace source and destination page */
6582                 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
6583                 if (!src_p)
6584                         return -EFAULT;
6585
6586                 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
6587                 if (!dst_p) {
6588                         sev_unpin_memory(kvm, src_p, n);
6589                         return -EFAULT;
6590                 }
6591
6592                 /*
6593                  * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the
6594                  * memory content (i.e it will write the same memory region with C=1).
6595                  * It's possible that the cache may contain the data with C=0, i.e.,
6596                  * unencrypted so invalidate it first.
6597                  */
6598                 sev_clflush_pages(src_p, 1);
6599                 sev_clflush_pages(dst_p, 1);
6600
6601                 /*
6602                  * Since user buffer may not be page aligned, calculate the
6603                  * offset within the page.
6604                  */
6605                 s_off = vaddr & ~PAGE_MASK;
6606                 d_off = dst_vaddr & ~PAGE_MASK;
6607                 len = min_t(size_t, (PAGE_SIZE - s_off), size);
6608
6609                 if (dec)
6610                         ret = __sev_dbg_decrypt_user(kvm,
6611                                                      __sme_page_pa(src_p[0]) + s_off,
6612                                                      dst_vaddr,
6613                                                      __sme_page_pa(dst_p[0]) + d_off,
6614                                                      len, &argp->error);
6615                 else
6616                         ret = __sev_dbg_encrypt_user(kvm,
6617                                                      __sme_page_pa(src_p[0]) + s_off,
6618                                                      vaddr,
6619                                                      __sme_page_pa(dst_p[0]) + d_off,
6620                                                      dst_vaddr,
6621                                                      len, &argp->error);
6622
6623                 sev_unpin_memory(kvm, src_p, 1);
6624                 sev_unpin_memory(kvm, dst_p, 1);
6625
6626                 if (ret)
6627                         goto err;
6628
6629                 next_vaddr = vaddr + len;
6630                 dst_vaddr = dst_vaddr + len;
6631                 size -= len;
6632         }
6633 err:
6634         return ret;
6635 }
6636
6637 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
6638 {
6639         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6640         struct sev_data_launch_secret *data;
6641         struct kvm_sev_launch_secret params;
6642         struct page **pages;
6643         void *blob, *hdr;
6644         unsigned long n;
6645         int ret, offset;
6646
6647         if (!sev_guest(kvm))
6648                 return -ENOTTY;
6649
6650         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6651                 return -EFAULT;
6652
6653         pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
6654         if (!pages)
6655                 return -ENOMEM;
6656
6657         /*
6658          * The secret must be copied into contiguous memory region, lets verify
6659          * that userspace memory pages are contiguous before we issue command.
6660          */
6661         if (get_num_contig_pages(0, pages, n) != n) {
6662                 ret = -EINVAL;
6663                 goto e_unpin_memory;
6664         }
6665
6666         ret = -ENOMEM;
6667         data = kzalloc(sizeof(*data), GFP_KERNEL);
6668         if (!data)
6669                 goto e_unpin_memory;
6670
6671         offset = params.guest_uaddr & (PAGE_SIZE - 1);
6672         data->guest_address = __sme_page_pa(pages[0]) + offset;
6673         data->guest_len = params.guest_len;
6674
6675         blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
6676         if (IS_ERR(blob)) {
6677                 ret = PTR_ERR(blob);
6678                 goto e_free;
6679         }
6680
6681         data->trans_address = __psp_pa(blob);
6682         data->trans_len = params.trans_len;
6683
6684         hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
6685         if (IS_ERR(hdr)) {
6686                 ret = PTR_ERR(hdr);
6687                 goto e_free_blob;
6688         }
6689         data->hdr_address = __psp_pa(hdr);
6690         data->hdr_len = params.hdr_len;
6691
6692         data->handle = sev->handle;
6693         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
6694
6695         kfree(hdr);
6696
6697 e_free_blob:
6698         kfree(blob);
6699 e_free:
6700         kfree(data);
6701 e_unpin_memory:
6702         sev_unpin_memory(kvm, pages, n);
6703         return ret;
6704 }
6705
6706 static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
6707 {
6708         struct kvm_sev_cmd sev_cmd;
6709         int r;
6710
6711         if (!svm_sev_enabled())
6712                 return -ENOTTY;
6713
6714         if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
6715                 return -EFAULT;
6716
6717         mutex_lock(&kvm->lock);
6718
6719         switch (sev_cmd.id) {
6720         case KVM_SEV_INIT:
6721                 r = sev_guest_init(kvm, &sev_cmd);
6722                 break;
6723         case KVM_SEV_LAUNCH_START:
6724                 r = sev_launch_start(kvm, &sev_cmd);
6725                 break;
6726         case KVM_SEV_LAUNCH_UPDATE_DATA:
6727                 r = sev_launch_update_data(kvm, &sev_cmd);
6728                 break;
6729         case KVM_SEV_LAUNCH_MEASURE:
6730                 r = sev_launch_measure(kvm, &sev_cmd);
6731                 break;
6732         case KVM_SEV_LAUNCH_FINISH:
6733                 r = sev_launch_finish(kvm, &sev_cmd);
6734                 break;
6735         case KVM_SEV_GUEST_STATUS:
6736                 r = sev_guest_status(kvm, &sev_cmd);
6737                 break;
6738         case KVM_SEV_DBG_DECRYPT:
6739                 r = sev_dbg_crypt(kvm, &sev_cmd, true);
6740                 break;
6741         case KVM_SEV_DBG_ENCRYPT:
6742                 r = sev_dbg_crypt(kvm, &sev_cmd, false);
6743                 break;
6744         case KVM_SEV_LAUNCH_SECRET:
6745                 r = sev_launch_secret(kvm, &sev_cmd);
6746                 break;
6747         default:
6748                 r = -EINVAL;
6749                 goto out;
6750         }
6751
6752         if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
6753                 r = -EFAULT;
6754
6755 out:
6756         mutex_unlock(&kvm->lock);
6757         return r;
6758 }
6759
6760 static int svm_register_enc_region(struct kvm *kvm,
6761                                    struct kvm_enc_region *range)
6762 {
6763         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6764         struct enc_region *region;
6765         int ret = 0;
6766
6767         if (!sev_guest(kvm))
6768                 return -ENOTTY;
6769
6770         region = kzalloc(sizeof(*region), GFP_KERNEL);
6771         if (!region)
6772                 return -ENOMEM;
6773
6774         region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
6775         if (!region->pages) {
6776                 ret = -ENOMEM;
6777                 goto e_free;
6778         }
6779
6780         /*
6781          * The guest may change the memory encryption attribute from C=0 -> C=1
6782          * or vice versa for this memory range. Lets make sure caches are
6783          * flushed to ensure that guest data gets written into memory with
6784          * correct C-bit.
6785          */
6786         sev_clflush_pages(region->pages, region->npages);
6787
6788         region->uaddr = range->addr;
6789         region->size = range->size;
6790
6791         mutex_lock(&kvm->lock);
6792         list_add_tail(&region->list, &sev->regions_list);
6793         mutex_unlock(&kvm->lock);
6794
6795         return ret;
6796
6797 e_free:
6798         kfree(region);
6799         return ret;
6800 }
6801
6802 static struct enc_region *
6803 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
6804 {
6805         struct kvm_sev_info *sev = &kvm->arch.sev_info;
6806         struct list_head *head = &sev->regions_list;
6807         struct enc_region *i;
6808
6809         list_for_each_entry(i, head, list) {
6810                 if (i->uaddr == range->addr &&
6811                     i->size == range->size)
6812                         return i;
6813         }
6814
6815         return NULL;
6816 }
6817
6818
6819 static int svm_unregister_enc_region(struct kvm *kvm,
6820                                      struct kvm_enc_region *range)
6821 {
6822         struct enc_region *region;
6823         int ret;
6824
6825         mutex_lock(&kvm->lock);
6826
6827         if (!sev_guest(kvm)) {
6828                 ret = -ENOTTY;
6829                 goto failed;
6830         }
6831
6832         region = find_enc_region(kvm, range);
6833         if (!region) {
6834                 ret = -EINVAL;
6835                 goto failed;
6836         }
6837
6838         __unregister_enc_region_locked(kvm, region);
6839
6840         mutex_unlock(&kvm->lock);
6841         return 0;
6842
6843 failed:
6844         mutex_unlock(&kvm->lock);
6845         return ret;
6846 }
6847
6848 static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
6849         .cpu_has_kvm_support = has_svm,
6850         .disabled_by_bios = is_disabled,
6851         .hardware_setup = svm_hardware_setup,
6852         .hardware_unsetup = svm_hardware_unsetup,
6853         .check_processor_compatibility = svm_check_processor_compat,
6854         .hardware_enable = svm_hardware_enable,
6855         .hardware_disable = svm_hardware_disable,
6856         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
6857         .cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
6858
6859         .vcpu_create = svm_create_vcpu,
6860         .vcpu_free = svm_free_vcpu,
6861         .vcpu_reset = svm_vcpu_reset,
6862
6863         .vm_init = avic_vm_init,
6864         .vm_destroy = svm_vm_destroy,
6865
6866         .prepare_guest_switch = svm_prepare_guest_switch,
6867         .vcpu_load = svm_vcpu_load,
6868         .vcpu_put = svm_vcpu_put,
6869         .vcpu_blocking = svm_vcpu_blocking,
6870         .vcpu_unblocking = svm_vcpu_unblocking,
6871
6872         .update_bp_intercept = update_bp_intercept,
6873         .get_msr_feature = svm_get_msr_feature,
6874         .get_msr = svm_get_msr,
6875         .set_msr = svm_set_msr,
6876         .get_segment_base = svm_get_segment_base,
6877         .get_segment = svm_get_segment,
6878         .set_segment = svm_set_segment,
6879         .get_cpl = svm_get_cpl,
6880         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
6881         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
6882         .decache_cr3 = svm_decache_cr3,
6883         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
6884         .set_cr0 = svm_set_cr0,
6885         .set_cr3 = svm_set_cr3,
6886         .set_cr4 = svm_set_cr4,
6887         .set_efer = svm_set_efer,
6888         .get_idt = svm_get_idt,
6889         .set_idt = svm_set_idt,
6890         .get_gdt = svm_get_gdt,
6891         .set_gdt = svm_set_gdt,
6892         .get_dr6 = svm_get_dr6,
6893         .set_dr6 = svm_set_dr6,
6894         .set_dr7 = svm_set_dr7,
6895         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
6896         .cache_reg = svm_cache_reg,
6897         .get_rflags = svm_get_rflags,
6898         .set_rflags = svm_set_rflags,
6899
6900         .tlb_flush = svm_flush_tlb,
6901
6902         .run = svm_vcpu_run,
6903         .handle_exit = handle_exit,
6904         .skip_emulated_instruction = skip_emulated_instruction,
6905         .set_interrupt_shadow = svm_set_interrupt_shadow,
6906         .get_interrupt_shadow = svm_get_interrupt_shadow,
6907         .patch_hypercall = svm_patch_hypercall,
6908         .set_irq = svm_set_irq,
6909         .set_nmi = svm_inject_nmi,
6910         .queue_exception = svm_queue_exception,
6911         .cancel_injection = svm_cancel_injection,
6912         .interrupt_allowed = svm_interrupt_allowed,
6913         .nmi_allowed = svm_nmi_allowed,
6914         .get_nmi_mask = svm_get_nmi_mask,
6915         .set_nmi_mask = svm_set_nmi_mask,
6916         .enable_nmi_window = enable_nmi_window,
6917         .enable_irq_window = enable_irq_window,
6918         .update_cr8_intercept = update_cr8_intercept,
6919         .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
6920         .get_enable_apicv = svm_get_enable_apicv,
6921         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
6922         .load_eoi_exitmap = svm_load_eoi_exitmap,
6923         .hwapic_irr_update = svm_hwapic_irr_update,
6924         .hwapic_isr_update = svm_hwapic_isr_update,
6925         .sync_pir_to_irr = kvm_lapic_find_highest_irr,
6926         .apicv_post_state_restore = avic_post_state_restore,
6927
6928         .set_tss_addr = svm_set_tss_addr,
6929         .get_tdp_level = get_npt_level,
6930         .get_mt_mask = svm_get_mt_mask,
6931
6932         .get_exit_info = svm_get_exit_info,
6933
6934         .get_lpage_level = svm_get_lpage_level,
6935
6936         .cpuid_update = svm_cpuid_update,
6937
6938         .rdtscp_supported = svm_rdtscp_supported,
6939         .invpcid_supported = svm_invpcid_supported,
6940         .mpx_supported = svm_mpx_supported,
6941         .xsaves_supported = svm_xsaves_supported,
6942         .umip_emulated = svm_umip_emulated,
6943
6944         .set_supported_cpuid = svm_set_supported_cpuid,
6945
6946         .has_wbinvd_exit = svm_has_wbinvd_exit,
6947
6948         .write_tsc_offset = svm_write_tsc_offset,
6949
6950         .set_tdp_cr3 = set_tdp_cr3,
6951
6952         .check_intercept = svm_check_intercept,
6953         .handle_external_intr = svm_handle_external_intr,
6954
6955         .sched_in = svm_sched_in,
6956
6957         .pmu_ops = &amd_pmu_ops,
6958         .deliver_posted_interrupt = svm_deliver_avic_intr,
6959         .update_pi_irte = svm_update_pi_irte,
6960         .setup_mce = svm_setup_mce,
6961
6962         .smi_allowed = svm_smi_allowed,
6963         .pre_enter_smm = svm_pre_enter_smm,
6964         .pre_leave_smm = svm_pre_leave_smm,
6965         .enable_smi_window = enable_smi_window,
6966
6967         .mem_enc_op = svm_mem_enc_op,
6968         .mem_enc_reg_region = svm_register_enc_region,
6969         .mem_enc_unreg_region = svm_unregister_enc_region,
6970 };
6971
6972 static int __init svm_init(void)
6973 {
6974         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
6975                         __alignof__(struct vcpu_svm), THIS_MODULE);
6976 }
6977
6978 static void __exit svm_exit(void)
6979 {
6980         kvm_exit();
6981 }
6982
6983 module_init(svm_init)
6984 module_exit(svm_exit)