arch/x86/kvm/vmx/vmx.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.  See
  15  * the COPYING file in the top-level directory.
  16  *
  17  */
  18
  19 #include <linux/frame.h>
  20 #include <linux/highmem.h>
  21 #include <linux/hrtimer.h>
  22 #include <linux/kernel.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/moduleparam.h>
  26 #include <linux/mod_devicetable.h>
  27 #include <linux/mm.h>
  28 #include <linux/sched.h>
  29 #include <linux/slab.h>
  30 #include <linux/tboot.h>
  31 #include <linux/trace_events.h>
  32
  33 #include <asm/apic.h>
  34 #include <asm/asm.h>
  35 #include <asm/cpu.h>
  36 #include <asm/debugreg.h>
  37 #include <asm/desc.h>
  38 #include <asm/fpu/internal.h>
  39 #include <asm/io.h>
  40 #include <asm/irq_remapping.h>
  41 #include <asm/kexec.h>
  42 #include <asm/perf_event.h>
  43 #include <asm/mce.h>
  44 #include <asm/mmu_context.h>
  45 #include <asm/mshyperv.h>
  46 #include <asm/spec-ctrl.h>
  47 #include <asm/virtext.h>
  48 #include <asm/vmx.h>
  49
  50 #include "capabilities.h"
  51 #include "cpuid.h"
  52 #include "evmcs.h"
  53 #include "irq.h"
  54 #include "kvm_cache_regs.h"
  55 #include "lapic.h"
  56 #include "mmu.h"
  57 #include "nested.h"
  58 #include "ops.h"
  59 #include "pmu.h"
  60 #include "trace.h"
  61 #include "vmcs.h"
  62 #include "vmcs12.h"
  63 #include "vmx.h"
  64 #include "x86.h"
  65
  66 MODULE_AUTHOR("Qumranet");
  67 MODULE_LICENSE("GPL");
  68
  69 static const struct x86_cpu_id vmx_cpu_id[] = {
  70         X86_FEATURE_MATCH(X86_FEATURE_VMX),
  71         {}
  72 };
  73 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
  74
  75 bool __read_mostly enable_vpid = 1;
  76 module_param_named(vpid, enable_vpid, bool, 0444);
  77
  78 static bool __read_mostly enable_vnmi = 1;
  79 module_param_named(vnmi, enable_vnmi, bool, S_IRUGO);
  80
  81 bool __read_mostly flexpriority_enabled = 1;
  82 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  83
  84 bool __read_mostly enable_ept = 1;
  85 module_param_named(ept, enable_ept, bool, S_IRUGO);
  86
  87 bool __read_mostly enable_unrestricted_guest = 1;
  88 module_param_named(unrestricted_guest,
  89                         enable_unrestricted_guest, bool, S_IRUGO);
  90
  91 bool __read_mostly enable_ept_ad_bits = 1;
  92 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
  93
  94 static bool __read_mostly emulate_invalid_guest_state = true;
  95 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
  96
  97 static bool __read_mostly fasteoi = 1;
  98 module_param(fasteoi, bool, S_IRUGO);
  99
 100 static bool __read_mostly enable_apicv = 1;
 101 module_param(enable_apicv, bool, S_IRUGO);
 102
 103 /*
 104  * If nested=1, nested virtualization is supported, i.e., guests may use
 105  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 106  * use VMX instructions.
 107  */
 108 static bool __read_mostly nested = 1;
 109 module_param(nested, bool, S_IRUGO);
 110
 111 static u64 __read_mostly host_xss;
 112
 113 bool __read_mostly enable_pml = 1;
 114 module_param_named(pml, enable_pml, bool, S_IRUGO);
 115
 116 #define MSR_BITMAP_MODE_X2APIC          1
 117 #define MSR_BITMAP_MODE_X2APIC_APICV    2
 118
 119 #define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL
 120
 121 /* Guest_tsc -> host_tsc conversion requires 64-bit division.  */
 122 static int __read_mostly cpu_preemption_timer_multi;
 123 static bool __read_mostly enable_preemption_timer = 1;
 124 #ifdef CONFIG_X86_64
 125 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
 126 #endif
 127
 128 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD)
 129 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
 130 #define KVM_VM_CR0_ALWAYS_ON                            \
 131         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST |      \
 132          X86_CR0_WP | X86_CR0_PG | X86_CR0_PE)
 133 #define KVM_CR4_GUEST_OWNED_BITS                                      \
 134         (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
 135          | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_TSD)
 136
 137 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE
 138 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
 139 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
 140
 141 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
 142
 143 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \
 144         RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \
 145         RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \
 146         RTIT_STATUS_BYTECNT))
 147
 148 #define MSR_IA32_RTIT_OUTPUT_BASE_MASK \
 149         (~((1UL << cpuid_query_maxphyaddr(vcpu)) - 1) | 0x7f)
 150
 151 /*
 152  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 153  * ple_gap:    upper bound on the amount of time between two successive
 154  *             executions of PAUSE in a loop. Also indicate if ple enabled.
 155  *             According to test, this time is usually smaller than 128 cycles.
 156  * ple_window: upper bound on the amount of time a guest is allowed to execute
 157  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 158  *             less than 2^12 cycles
 159  * Time is measured based on a counter that runs at the same rate as the TSC,
 160  * refer SDM volume 3b section 21.6.13 & 22.1.3.
 161  */
 162 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP;
 163 module_param(ple_gap, uint, 0444);
 164
 165 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
 166 module_param(ple_window, uint, 0444);
 167
 168 /* Default doubles per-vcpu window every exit. */
 169 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 170 module_param(ple_window_grow, uint, 0444);
 171
 172 /* Default resets per-vcpu window every exit to ple_window. */
 173 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 174 module_param(ple_window_shrink, uint, 0444);
 175
 176 /* Default is to compute the maximum so we can never overflow. */
 177 static unsigned int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 178 module_param(ple_window_max, uint, 0444);
 179
 180 /* Default is SYSTEM mode, 1 for host-guest mode */
 181 int __read_mostly pt_mode = PT_MODE_SYSTEM;
 182 module_param(pt_mode, int, S_IRUGO);
 183
 184 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
 185 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond);
 186 static DEFINE_MUTEX(vmx_l1d_flush_mutex);
 187
 188 /* Storage for pre module init parameter parsing */
 189 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO;
 190
 191 static const struct {
 192         const char *option;
 193         bool for_parse;
 194 } vmentry_l1d_param[] = {
 195         [VMENTER_L1D_FLUSH_AUTO]         = {"auto", true},
 196         [VMENTER_L1D_FLUSH_NEVER]        = {"never", true},
 197         [VMENTER_L1D_FLUSH_COND]         = {"cond", true},
 198         [VMENTER_L1D_FLUSH_ALWAYS]       = {"always", true},
 199         [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false},
 200         [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false},
 201 };
 202
 203 #define L1D_CACHE_ORDER 4
 204 static void *vmx_l1d_flush_pages;
 205
 206 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
 207 {
 208         struct page *page;
 209         unsigned int i;
 210
 211         if (!enable_ept) {
 212                 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED;
 213                 return 0;
 214         }
 215
 216         if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
 217                 u64 msr;
 218
 219                 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);
 220                 if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) {
 221                         l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
 222                         return 0;
 223                 }
 224         }
 225
 226         /* If set to auto use the default l1tf mitigation method */
 227         if (l1tf == VMENTER_L1D_FLUSH_AUTO) {
 228                 switch (l1tf_mitigation) {
 229                 case L1TF_MITIGATION_OFF:
 230                         l1tf = VMENTER_L1D_FLUSH_NEVER;
 231                         break;
 232                 case L1TF_MITIGATION_FLUSH_NOWARN:
 233                 case L1TF_MITIGATION_FLUSH:
 234                 case L1TF_MITIGATION_FLUSH_NOSMT:
 235                         l1tf = VMENTER_L1D_FLUSH_COND;
 236                         break;
 237                 case L1TF_MITIGATION_FULL:
 238                 case L1TF_MITIGATION_FULL_FORCE:
 239                         l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 240                         break;
 241                 }
 242         } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) {
 243                 l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 244         }
 245
 246         if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages &&
 247             !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) {
 248                 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
 249                 if (!page)
 250                         return -ENOMEM;
 251                 vmx_l1d_flush_pages = page_address(page);
 252
 253                 /*
 254                  * Initialize each page with a different pattern in
 255                  * order to protect against KSM in the nested
 256                  * virtualization case.
 257                  */
 258                 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) {
 259                         memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1,
 260                                PAGE_SIZE);
 261                 }
 262         }
 263
 264         l1tf_vmx_mitigation = l1tf;
 265
 266         if (l1tf != VMENTER_L1D_FLUSH_NEVER)
 267                 static_branch_enable(&vmx_l1d_should_flush);
 268         else
 269                 static_branch_disable(&vmx_l1d_should_flush);
 270
 271         if (l1tf == VMENTER_L1D_FLUSH_COND)
 272                 static_branch_enable(&vmx_l1d_flush_cond);
 273         else
 274                 static_branch_disable(&vmx_l1d_flush_cond);
 275         return 0;
 276 }
 277
 278 static int vmentry_l1d_flush_parse(const char *s)
 279 {
 280         unsigned int i;
 281
 282         if (s) {
 283                 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) {
 284                         if (vmentry_l1d_param[i].for_parse &&
 285                             sysfs_streq(s, vmentry_l1d_param[i].option))
 286                                 return i;
 287                 }
 288         }
 289         return -EINVAL;
 290 }
 291
 292 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
 293 {
 294         int l1tf, ret;
 295
 296         l1tf = vmentry_l1d_flush_parse(s);
 297         if (l1tf < 0)
 298                 return l1tf;
 299
 300         if (!boot_cpu_has(X86_BUG_L1TF))
 301                 return 0;
 302
 303         /*
 304          * Has vmx_init() run already? If not then this is the pre init
 305          * parameter parsing. In that case just store the value and let
 306          * vmx_init() do the proper setup after enable_ept has been
 307          * established.
 308          */
 309         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) {
 310                 vmentry_l1d_flush_param = l1tf;
 311                 return 0;
 312         }
 313
 314         mutex_lock(&vmx_l1d_flush_mutex);
 315         ret = vmx_setup_l1d_flush(l1tf);
 316         mutex_unlock(&vmx_l1d_flush_mutex);
 317         return ret;
 318 }
 319
 320 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
 321 {
 322         if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param)))
 323                 return sprintf(s, "???\n");
 324
 325         return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option);
 326 }
 327
 328 static const struct kernel_param_ops vmentry_l1d_flush_ops = {
 329         .set = vmentry_l1d_flush_set,
 330         .get = vmentry_l1d_flush_get,
 331 };
 332 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
 333
 334 static bool guest_state_valid(struct kvm_vcpu *vcpu);
 335 static u32 vmx_segment_access_rights(struct kvm_segment *var);
 336 static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
 337                                                           u32 msr, int type);
 338
 339 void vmx_vmexit(void);
 340
 341 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 342 DEFINE_PER_CPU(struct vmcs *, current_vmcs);
 343 /*
 344  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
 345  * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
 346  */
 347 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
 348
 349 /*
 350  * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
 351  * can find which vCPU should be waken up.
 352  */
 353 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
 354 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
 355
 356 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
 357 static DEFINE_SPINLOCK(vmx_vpid_lock);
 358
 359 struct vmcs_config vmcs_config;
 360 struct vmx_capability vmx_capability;
 361
 362 #define VMX_SEGMENT_FIELD(seg)                                  \
 363         [VCPU_SREG_##seg] = {                                   \
 364                 .selector = GUEST_##seg##_SELECTOR,             \
 365                 .base = GUEST_##seg##_BASE,                     \
 366                 .limit = GUEST_##seg##_LIMIT,                   \
 367                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
 368         }
 369
 370 static const struct kvm_vmx_segment_field {
 371         unsigned selector;
 372         unsigned base;
 373         unsigned limit;
 374         unsigned ar_bytes;
 375 } kvm_vmx_segment_fields[] = {
 376         VMX_SEGMENT_FIELD(CS),
 377         VMX_SEGMENT_FIELD(DS),
 378         VMX_SEGMENT_FIELD(ES),
 379         VMX_SEGMENT_FIELD(FS),
 380         VMX_SEGMENT_FIELD(GS),
 381         VMX_SEGMENT_FIELD(SS),
 382         VMX_SEGMENT_FIELD(TR),
 383         VMX_SEGMENT_FIELD(LDTR),
 384 };
 385
 386 u64 host_efer;
 387
 388 /*
 389  * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm
 390  * will emulate SYSCALL in legacy mode if the vendor string in guest
 391  * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To
 392  * support this emulation, IA32_STAR must always be included in
 393  * vmx_msr_index[], even in i386 builds.
 394  */
 395 const u32 vmx_msr_index[] = {
 396 #ifdef CONFIG_X86_64
 397         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
 398 #endif
 399         MSR_EFER, MSR_TSC_AUX, MSR_STAR,
 400 };
 401
 402 #if IS_ENABLED(CONFIG_HYPERV)
 403 static bool __read_mostly enlightened_vmcs = true;
 404 module_param(enlightened_vmcs, bool, 0444);
 405
 406 /* check_ept_pointer() should be under protection of ept_pointer_lock. */
 407 static void check_ept_pointer_match(struct kvm *kvm)
 408 {
 409         struct kvm_vcpu *vcpu;
 410         u64 tmp_eptp = INVALID_PAGE;
 411         int i;
 412
 413         kvm_for_each_vcpu(i, vcpu, kvm) {
 414                 if (!VALID_PAGE(tmp_eptp)) {
 415                         tmp_eptp = to_vmx(vcpu)->ept_pointer;
 416                 } else if (tmp_eptp != to_vmx(vcpu)->ept_pointer) {
 417                         to_kvm_vmx(kvm)->ept_pointers_match
 418                                 = EPT_POINTERS_MISMATCH;
 419                         return;
 420                 }
 421         }
 422
 423         to_kvm_vmx(kvm)->ept_pointers_match = EPT_POINTERS_MATCH;
 424 }
 425
 426 static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush,
 427                 void *data)
 428 {
 429         struct kvm_tlb_range *range = data;
 430
 431         return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn,
 432                         range->pages);
 433 }
 434
 435 static inline int __hv_remote_flush_tlb_with_range(struct kvm *kvm,
 436                 struct kvm_vcpu *vcpu, struct kvm_tlb_range *range)
 437 {
 438         u64 ept_pointer = to_vmx(vcpu)->ept_pointer;
 439
 440         /*
 441          * FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE hypercall needs address
 442          * of the base of EPT PML4 table, strip off EPT configuration
 443          * information.
 444          */
 445         if (range)
 446                 return hyperv_flush_guest_mapping_range(ept_pointer & PAGE_MASK,
 447                                 kvm_fill_hv_flush_list_func, (void *)range);
 448         else
 449                 return hyperv_flush_guest_mapping(ept_pointer & PAGE_MASK);
 450 }
 451
 452 static int hv_remote_flush_tlb_with_range(struct kvm *kvm,
 453                 struct kvm_tlb_range *range)
 454 {
 455         struct kvm_vcpu *vcpu;
 456         int ret = 0, i;
 457
 458         spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
 459
 460         if (to_kvm_vmx(kvm)->ept_pointers_match == EPT_POINTERS_CHECK)
 461                 check_ept_pointer_match(kvm);
 462
 463         if (to_kvm_vmx(kvm)->ept_pointers_match != EPT_POINTERS_MATCH) {
 464                 kvm_for_each_vcpu(i, vcpu, kvm) {
 465                         /* If ept_pointer is invalid pointer, bypass flush request. */
 466                         if (VALID_PAGE(to_vmx(vcpu)->ept_pointer))
 467                                 ret |= __hv_remote_flush_tlb_with_range(
 468                                         kvm, vcpu, range);
 469                 }
 470         } else {
 471                 ret = __hv_remote_flush_tlb_with_range(kvm,
 472                                 kvm_get_vcpu(kvm, 0), range);
 473         }
 474
 475         spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
 476         return ret;
 477 }
 478 static int hv_remote_flush_tlb(struct kvm *kvm)
 479 {
 480         return hv_remote_flush_tlb_with_range(kvm, NULL);
 481 }
 482
 483 #endif /* IS_ENABLED(CONFIG_HYPERV) */
 484
 485 /*
 486  * Comment's format: document - errata name - stepping - processor name.
 487  * Refer from
 488  * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
 489  */
 490 static u32 vmx_preemption_cpu_tfms[] = {
 491 /* 323344.pdf - BA86   - D0 - Xeon 7500 Series */
 492 0x000206E6,
 493 /* 323056.pdf - AAX65  - C2 - Xeon L3406 */
 494 /* 322814.pdf - AAT59  - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
 495 /* 322911.pdf - AAU65  - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
 496 0x00020652,
 497 /* 322911.pdf - AAU65  - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
 498 0x00020655,
 499 /* 322373.pdf - AAO95  - B1 - Xeon 3400 Series */
 500 /* 322166.pdf - AAN92  - B1 - i7-800 and i5-700 Desktop */
 501 /*
 502  * 320767.pdf - AAP86  - B1 -
 503  * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
 504  */
 505 0x000106E5,
 506 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
 507 0x000106A0,
 508 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
 509 0x000106A1,
 510 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
 511 0x000106A4,
 512  /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
 513  /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
 514  /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
 515 0x000106A5,
 516  /* Xeon E3-1220 V2 */
 517 0x000306A8,
 518 };
 519
 520 static inline bool cpu_has_broken_vmx_preemption_timer(void)
 521 {
 522         u32 eax = cpuid_eax(0x00000001), i;
 523
 524         /* Clear the reserved bits */
 525         eax &= ~(0x3U << 14 | 0xfU << 28);
 526         for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
 527                 if (eax == vmx_preemption_cpu_tfms[i])
 528                         return true;
 529
 530         return false;
 531 }
 532
 533 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
 534 {
 535         return flexpriority_enabled && lapic_in_kernel(vcpu);
 536 }
 537
 538 static inline bool report_flexpriority(void)
 539 {
 540         return flexpriority_enabled;
 541 }
 542
 543 static inline int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
 544 {
 545         int i;
 546
 547         for (i = 0; i < vmx->nmsrs; ++i)
 548                 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
 549                         return i;
 550         return -1;
 551 }
 552
 553 struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
 554 {
 555         int i;
 556
 557         i = __find_msr_index(vmx, msr);
 558         if (i >= 0)
 559                 return &vmx->guest_msrs[i];
 560         return NULL;
 561 }
 562
 563 void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
 564 {
 565         vmcs_clear(loaded_vmcs->vmcs);
 566         if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
 567                 vmcs_clear(loaded_vmcs->shadow_vmcs);
 568         loaded_vmcs->cpu = -1;
 569         loaded_vmcs->launched = 0;
 570 }
 571
 572 #ifdef CONFIG_KEXEC_CORE
 573 /*
 574  * This bitmap is used to indicate whether the vmclear
 575  * operation is enabled on all cpus. All disabled by
 576  * default.
 577  */
 578 static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;
 579
 580 static inline void crash_enable_local_vmclear(int cpu)
 581 {
 582         cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
 583 }
 584
 585 static inline void crash_disable_local_vmclear(int cpu)
 586 {
 587         cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
 588 }
 589
 590 static inline int crash_local_vmclear_enabled(int cpu)
 591 {
 592         return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
 593 }
 594
 595 static void crash_vmclear_local_loaded_vmcss(void)
 596 {
 597         int cpu = raw_smp_processor_id();
 598         struct loaded_vmcs *v;
 599
 600         if (!crash_local_vmclear_enabled(cpu))
 601                 return;
 602
 603         list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
 604                             loaded_vmcss_on_cpu_link)
 605                 vmcs_clear(v->vmcs);
 606 }
 607 #else
 608 static inline void crash_enable_local_vmclear(int cpu) { }
 609 static inline void crash_disable_local_vmclear(int cpu) { }
 610 #endif /* CONFIG_KEXEC_CORE */
 611
 612 static void __loaded_vmcs_clear(void *arg)
 613 {
 614         struct loaded_vmcs *loaded_vmcs = arg;
 615         int cpu = raw_smp_processor_id();
 616
 617         if (loaded_vmcs->cpu != cpu)
 618                 return; /* vcpu migration can race with cpu offline */
 619         if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
 620                 per_cpu(current_vmcs, cpu) = NULL;
 621         crash_disable_local_vmclear(cpu);
 622         list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
 623
 624         /*
 625          * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
 626          * is before setting loaded_vmcs->vcpu to -1 which is done in
 627          * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
 628          * then adds the vmcs into percpu list before it is deleted.
 629          */
 630         smp_wmb();
 631
 632         loaded_vmcs_init(loaded_vmcs);
 633         crash_enable_local_vmclear(cpu);
 634 }
 635
 636 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
 637 {
 638         int cpu = loaded_vmcs->cpu;
 639
 640         if (cpu != -1)
 641                 smp_call_function_single(cpu,
 642                          __loaded_vmcs_clear, loaded_vmcs, 1);
 643 }
 644
 645 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
 646                                        unsigned field)
 647 {
 648         bool ret;
 649         u32 mask = 1 << (seg * SEG_FIELD_NR + field);
 650
 651         if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
 652                 vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
 653                 vmx->segment_cache.bitmask = 0;
 654         }
 655         ret = vmx->segment_cache.bitmask & mask;
 656         vmx->segment_cache.bitmask |= mask;
 657         return ret;
 658 }
 659
 660 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
 661 {
 662         u16 *p = &vmx->segment_cache.seg[seg].selector;
 663
 664         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
 665                 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
 666         return *p;
 667 }
 668
 669 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
 670 {
 671         ulong *p = &vmx->segment_cache.seg[seg].base;
 672
 673         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
 674                 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
 675         return *p;
 676 }
 677
 678 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
 679 {
 680         u32 *p = &vmx->segment_cache.seg[seg].limit;
 681
 682         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
 683                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
 684         return *p;
 685 }
 686
 687 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
 688 {
 689         u32 *p = &vmx->segment_cache.seg[seg].ar;
 690
 691         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
 692                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
 693         return *p;
 694 }
 695
 696 void update_exception_bitmap(struct kvm_vcpu *vcpu)
 697 {
 698         u32 eb;
 699
 700         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
 701              (1u << DB_VECTOR) | (1u << AC_VECTOR);
 702         /*
 703          * Guest access to VMware backdoor ports could legitimately
 704          * trigger #GP because of TSS I/O permission bitmap.
 705          * We intercept those #GP and allow access to them anyway
 706          * as VMware does.
 707          */
 708         if (enable_vmware_backdoor)
 709                 eb |= (1u << GP_VECTOR);
 710         if ((vcpu->guest_debug &
 711              (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
 712             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
 713                 eb |= 1u << BP_VECTOR;
 714         if (to_vmx(vcpu)->rmode.vm86_active)
 715                 eb = ~0;
 716         if (enable_ept)
 717                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
 718
 719         /* When we are running a nested L2 guest and L1 specified for it a
 720          * certain exception bitmap, we must trap the same exceptions and pass
 721          * them to L1. When running L2, we will only handle the exceptions
 722          * specified above if L1 did not want them.
 723          */
 724         if (is_guest_mode(vcpu))
 725                 eb |= get_vmcs12(vcpu)->exception_bitmap;
 726
 727         vmcs_write32(EXCEPTION_BITMAP, eb);
 728 }
 729
 730 /*
 731  * Check if MSR is intercepted for currently loaded MSR bitmap.
 732  */
 733 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
 734 {
 735         unsigned long *msr_bitmap;
 736         int f = sizeof(unsigned long);
 737
 738         if (!cpu_has_vmx_msr_bitmap())
 739                 return true;
 740
 741         msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap;
 742
 743         if (msr <= 0x1fff) {
 744                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
 745         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
 746                 msr &= 0x1fff;
 747                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
 748         }
 749
 750         return true;
 751 }
 752
 753 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
 754                 unsigned long entry, unsigned long exit)
 755 {
 756         vm_entry_controls_clearbit(vmx, entry);
 757         vm_exit_controls_clearbit(vmx, exit);
 758 }
 759
 760 static int find_msr(struct vmx_msrs *m, unsigned int msr)
 761 {
 762         unsigned int i;
 763
 764         for (i = 0; i < m->nr; ++i) {
 765                 if (m->val[i].index == msr)
 766                         return i;
 767         }
 768         return -ENOENT;
 769 }
 770
 771 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
 772 {
 773         int i;
 774         struct msr_autoload *m = &vmx->msr_autoload;
 775
 776         switch (msr) {
 777         case MSR_EFER:
 778                 if (cpu_has_load_ia32_efer()) {
 779                         clear_atomic_switch_msr_special(vmx,
 780                                         VM_ENTRY_LOAD_IA32_EFER,
 781                                         VM_EXIT_LOAD_IA32_EFER);
 782                         return;
 783                 }
 784                 break;
 785         case MSR_CORE_PERF_GLOBAL_CTRL:
 786                 if (cpu_has_load_perf_global_ctrl()) {
 787                         clear_atomic_switch_msr_special(vmx,
 788                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
 789                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
 790                         return;
 791                 }
 792                 break;
 793         }
 794         i = find_msr(&m->guest, msr);
 795         if (i < 0)
 796                 goto skip_guest;
 797         --m->guest.nr;
 798         m->guest.val[i] = m->guest.val[m->guest.nr];
 799         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
 800
 801 skip_guest:
 802         i = find_msr(&m->host, msr);
 803         if (i < 0)
 804                 return;
 805
 806         --m->host.nr;
 807         m->host.val[i] = m->host.val[m->host.nr];
 808         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
 809 }
 810
 811 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
 812                 unsigned long entry, unsigned long exit,
 813                 unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
 814                 u64 guest_val, u64 host_val)
 815 {
 816         vmcs_write64(guest_val_vmcs, guest_val);
 817         if (host_val_vmcs != HOST_IA32_EFER)
 818                 vmcs_write64(host_val_vmcs, host_val);
 819         vm_entry_controls_setbit(vmx, entry);
 820         vm_exit_controls_setbit(vmx, exit);
 821 }
 822
 823 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
 824                                   u64 guest_val, u64 host_val, bool entry_only)
 825 {
 826         int i, j = 0;
 827         struct msr_autoload *m = &vmx->msr_autoload;
 828
 829         switch (msr) {
 830         case MSR_EFER:
 831                 if (cpu_has_load_ia32_efer()) {
 832                         add_atomic_switch_msr_special(vmx,
 833                                         VM_ENTRY_LOAD_IA32_EFER,
 834                                         VM_EXIT_LOAD_IA32_EFER,
 835                                         GUEST_IA32_EFER,
 836                                         HOST_IA32_EFER,
 837                                         guest_val, host_val);
 838                         return;
 839                 }
 840                 break;
 841         case MSR_CORE_PERF_GLOBAL_CTRL:
 842                 if (cpu_has_load_perf_global_ctrl()) {
 843                         add_atomic_switch_msr_special(vmx,
 844                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
 845                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
 846                                         GUEST_IA32_PERF_GLOBAL_CTRL,
 847                                         HOST_IA32_PERF_GLOBAL_CTRL,
 848                                         guest_val, host_val);
 849                         return;
 850                 }
 851                 break;
 852         case MSR_IA32_PEBS_ENABLE:
 853                 /* PEBS needs a quiescent period after being disabled (to write
 854                  * a record).  Disabling PEBS through VMX MSR swapping doesn't
 855                  * provide that period, so a CPU could write host's record into
 856                  * guest's memory.
 857                  */
 858                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
 859         }
 860
 861         i = find_msr(&m->guest, msr);
 862         if (!entry_only)
 863                 j = find_msr(&m->host, msr);
 864
 865         if (i == NR_AUTOLOAD_MSRS || j == NR_AUTOLOAD_MSRS) {
 866                 printk_once(KERN_WARNING "Not enough msr switch entries. "
 867                                 "Can't add msr %x\n", msr);
 868                 return;
 869         }
 870         if (i < 0) {
 871                 i = m->guest.nr++;
 872                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
 873         }
 874         m->guest.val[i].index = msr;
 875         m->guest.val[i].value = guest_val;
 876
 877         if (entry_only)
 878                 return;
 879
 880         if (j < 0) {
 881                 j = m->host.nr++;
 882                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
 883         }
 884         m->host.val[j].index = msr;
 885         m->host.val[j].value = host_val;
 886 }
 887
 888 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
 889 {
 890         u64 guest_efer = vmx->vcpu.arch.efer;
 891         u64 ignore_bits = 0;
 892
 893         if (!enable_ept) {
 894                 /*
 895                  * NX is needed to handle CR0.WP=1, CR4.SMEP=1.  Testing
 896                  * host CPUID is more efficient than testing guest CPUID
 897                  * or CR4.  Host SMEP is anyway a requirement for guest SMEP.
 898                  */
 899                 if (boot_cpu_has(X86_FEATURE_SMEP))
 900                         guest_efer |= EFER_NX;
 901                 else if (!(guest_efer & EFER_NX))
 902                         ignore_bits |= EFER_NX;
 903         }
 904
 905         /*
 906          * LMA and LME handled by hardware; SCE meaningless outside long mode.
 907          */
 908         ignore_bits |= EFER_SCE;
 909 #ifdef CONFIG_X86_64
 910         ignore_bits |= EFER_LMA | EFER_LME;
 911         /* SCE is meaningful only in long mode on Intel */
 912         if (guest_efer & EFER_LMA)
 913                 ignore_bits &= ~(u64)EFER_SCE;
 914 #endif
 915
 916         /*
 917          * On EPT, we can't emulate NX, so we must switch EFER atomically.
 918          * On CPUs that support "load IA32_EFER", always switch EFER
 919          * atomically, since it's faster than switching it manually.
 920          */
 921         if (cpu_has_load_ia32_efer() ||
 922             (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
 923                 if (!(guest_efer & EFER_LMA))
 924                         guest_efer &= ~EFER_LME;
 925                 if (guest_efer != host_efer)
 926                         add_atomic_switch_msr(vmx, MSR_EFER,
 927                                               guest_efer, host_efer, false);
 928                 else
 929                         clear_atomic_switch_msr(vmx, MSR_EFER);
 930                 return false;
 931         } else {
 932                 clear_atomic_switch_msr(vmx, MSR_EFER);
 933
 934                 guest_efer &= ~ignore_bits;
 935                 guest_efer |= host_efer & ignore_bits;
 936
 937                 vmx->guest_msrs[efer_offset].data = guest_efer;
 938                 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
 939
 940                 return true;
 941         }
 942 }
 943
 944 #ifdef CONFIG_X86_32
 945 /*
 946  * On 32-bit kernels, VM exits still load the FS and GS bases from the
 947  * VMCS rather than the segment table.  KVM uses this helper to figure
 948  * out the current bases to poke them into the VMCS before entry.
 949  */
 950 static unsigned long segment_base(u16 selector)
 951 {
 952         struct desc_struct *table;
 953         unsigned long v;
 954
 955         if (!(selector & ~SEGMENT_RPL_MASK))
 956                 return 0;
 957
 958         table = get_current_gdt_ro();
 959
 960         if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
 961                 u16 ldt_selector = kvm_read_ldt();
 962
 963                 if (!(ldt_selector & ~SEGMENT_RPL_MASK))
 964                         return 0;
 965
 966                 table = (struct desc_struct *)segment_base(ldt_selector);
 967         }
 968         v = get_desc_base(&table[selector >> 3]);
 969         return v;
 970 }
 971 #endif
 972
 973 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range)
 974 {
 975         u32 i;
 976
 977         wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
 978         wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
 979         wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
 980         wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
 981         for (i = 0; i < addr_range; i++) {
 982                 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
 983                 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
 984         }
 985 }
 986
 987 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range)
 988 {
 989         u32 i;
 990
 991         rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
 992         rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
 993         rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
 994         rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
 995         for (i = 0; i < addr_range; i++) {
 996                 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
 997                 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
 998         }
 999 }
1000
1001 static void pt_guest_enter(struct vcpu_vmx *vmx)
1002 {
1003         if (pt_mode == PT_MODE_SYSTEM)
1004                 return;
1005
1006         /*
1007          * GUEST_IA32_RTIT_CTL is already set in the VMCS.
1008          * Save host state before VM entry.
1009          */
1010         rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
1011         if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
1012                 wrmsrl(MSR_IA32_RTIT_CTL, 0);
1013                 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range);
1014                 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range);
1015         }
1016 }
1017
1018 static void pt_guest_exit(struct vcpu_vmx *vmx)
1019 {
1020         if (pt_mode == PT_MODE_SYSTEM)
1021                 return;
1022
1023         if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
1024                 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range);
1025                 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range);
1026         }
1027
1028         /* Reload host state (IA32_RTIT_CTL will be cleared on VM exit). */
1029         wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
1030 }
1031
1032 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1033 {
1034         struct vcpu_vmx *vmx = to_vmx(vcpu);
1035         struct vmcs_host_state *host_state;
1036 #ifdef CONFIG_X86_64
1037         int cpu = raw_smp_processor_id();
1038 #endif
1039         unsigned long fs_base, gs_base;
1040         u16 fs_sel, gs_sel;
1041         int i;
1042
1043         vmx->req_immediate_exit = false;
1044
1045         /*
1046          * Note that guest MSRs to be saved/restored can also be changed
1047          * when guest state is loaded. This happens when guest transitions
1048          * to/from long-mode by setting MSR_EFER.LMA.
1049          */
1050         if (!vmx->loaded_cpu_state || vmx->guest_msrs_dirty) {
1051                 vmx->guest_msrs_dirty = false;
1052                 for (i = 0; i < vmx->save_nmsrs; ++i)
1053                         kvm_set_shared_msr(vmx->guest_msrs[i].index,
1054                                            vmx->guest_msrs[i].data,
1055                                            vmx->guest_msrs[i].mask);
1056
1057         }
1058
1059         if (vmx->loaded_cpu_state)
1060                 return;
1061
1062         vmx->loaded_cpu_state = vmx->loaded_vmcs;
1063         host_state = &vmx->loaded_cpu_state->host_state;
1064
1065         /*
1066          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
1067          * allow segment selectors with cpl > 0 or ti == 1.
1068          */
1069         host_state->ldt_sel = kvm_read_ldt();
1070
1071 #ifdef CONFIG_X86_64
1072         savesegment(ds, host_state->ds_sel);
1073         savesegment(es, host_state->es_sel);
1074
1075         gs_base = cpu_kernelmode_gs_base(cpu);
1076         if (likely(is_64bit_mm(current->mm))) {
1077                 save_fsgs_for_kvm();
1078                 fs_sel = current->thread.fsindex;
1079                 gs_sel = current->thread.gsindex;
1080                 fs_base = current->thread.fsbase;
1081                 vmx->msr_host_kernel_gs_base = current->thread.gsbase;
1082         } else {
1083                 savesegment(fs, fs_sel);
1084                 savesegment(gs, gs_sel);
1085                 fs_base = read_msr(MSR_FS_BASE);
1086                 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
1087         }
1088
1089         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1090 #else
1091         savesegment(fs, fs_sel);
1092         savesegment(gs, gs_sel);
1093         fs_base = segment_base(fs_sel);
1094         gs_base = segment_base(gs_sel);
1095 #endif
1096
1097         if (unlikely(fs_sel != host_state->fs_sel)) {
1098                 if (!(fs_sel & 7))
1099                         vmcs_write16(HOST_FS_SELECTOR, fs_sel);
1100                 else
1101                         vmcs_write16(HOST_FS_SELECTOR, 0);
1102                 host_state->fs_sel = fs_sel;
1103         }
1104         if (unlikely(gs_sel != host_state->gs_sel)) {
1105                 if (!(gs_sel & 7))
1106                         vmcs_write16(HOST_GS_SELECTOR, gs_sel);
1107                 else
1108                         vmcs_write16(HOST_GS_SELECTOR, 0);
1109                 host_state->gs_sel = gs_sel;
1110         }
1111         if (unlikely(fs_base != host_state->fs_base)) {
1112                 vmcs_writel(HOST_FS_BASE, fs_base);
1113                 host_state->fs_base = fs_base;
1114         }
1115         if (unlikely(gs_base != host_state->gs_base)) {
1116                 vmcs_writel(HOST_GS_BASE, gs_base);
1117                 host_state->gs_base = gs_base;
1118         }
1119 }
1120
1121 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
1122 {
1123         struct vmcs_host_state *host_state;
1124
1125         if (!vmx->loaded_cpu_state)
1126                 return;
1127
1128         WARN_ON_ONCE(vmx->loaded_cpu_state != vmx->loaded_vmcs);
1129         host_state = &vmx->loaded_cpu_state->host_state;
1130
1131         ++vmx->vcpu.stat.host_state_reload;
1132         vmx->loaded_cpu_state = NULL;
1133
1134 #ifdef CONFIG_X86_64
1135         rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1136 #endif
1137         if (host_state->ldt_sel || (host_state->gs_sel & 7)) {
1138                 kvm_load_ldt(host_state->ldt_sel);
1139 #ifdef CONFIG_X86_64
1140                 load_gs_index(host_state->gs_sel);
1141 #else
1142                 loadsegment(gs, host_state->gs_sel);
1143 #endif
1144         }
1145         if (host_state->fs_sel & 7)
1146                 loadsegment(fs, host_state->fs_sel);
1147 #ifdef CONFIG_X86_64
1148         if (unlikely(host_state->ds_sel | host_state->es_sel)) {
1149                 loadsegment(ds, host_state->ds_sel);
1150                 loadsegment(es, host_state->es_sel);
1151         }
1152 #endif
1153         invalidate_tss_limit();
1154 #ifdef CONFIG_X86_64
1155         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1156 #endif
1157         load_fixmap_gdt(raw_smp_processor_id());
1158 }
1159
1160 #ifdef CONFIG_X86_64
1161 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
1162 {
1163         preempt_disable();
1164         if (vmx->loaded_cpu_state)
1165                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1166         preempt_enable();
1167         return vmx->msr_guest_kernel_gs_base;
1168 }
1169
1170 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
1171 {
1172         preempt_disable();
1173         if (vmx->loaded_cpu_state)
1174                 wrmsrl(MSR_KERNEL_GS_BASE, data);
1175         preempt_enable();
1176         vmx->msr_guest_kernel_gs_base = data;
1177 }
1178 #endif
1179
1180 static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
1181 {
1182         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
1183         struct pi_desc old, new;
1184         unsigned int dest;
1185
1186         /*
1187          * In case of hot-plug or hot-unplug, we may have to undo
1188          * vmx_vcpu_pi_put even if there is no assigned device.  And we
1189          * always keep PI.NDST up to date for simplicity: it makes the
1190          * code easier, and CPU migration is not a fast path.
1191          */
1192         if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu)
1193                 return;
1194
1195         /*
1196          * First handle the simple case where no cmpxchg is necessary; just
1197          * allow posting non-urgent interrupts.
1198          *
1199          * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change
1200          * PI.NDST: pi_post_block will do it for us and the wakeup_handler
1201          * expects the VCPU to be on the blocked_vcpu_list that matches
1202          * PI.NDST.
1203          */
1204         if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR ||
1205             vcpu->cpu == cpu) {
1206                 pi_clear_sn(pi_desc);
1207                 return;
1208         }
1209
1210         /* The full case.  */
1211         do {
1212                 old.control = new.control = pi_desc->control;
1213
1214                 dest = cpu_physical_id(cpu);
1215
1216                 if (x2apic_enabled())
1217                         new.ndst = dest;
1218                 else
1219                         new.ndst = (dest << 8) & 0xFF00;
1220
1221                 new.sn = 0;
1222         } while (cmpxchg64(&pi_desc->control, old.control,
1223                            new.control) != old.control);
1224 }
1225
1226 /*
1227  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
1228  * vcpu mutex is already taken.
1229  */
1230 void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1231 {
1232         struct vcpu_vmx *vmx = to_vmx(vcpu);
1233         bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
1234
1235         if (!already_loaded) {
1236                 loaded_vmcs_clear(vmx->loaded_vmcs);
1237                 local_irq_disable();
1238                 crash_disable_local_vmclear(cpu);
1239
1240                 /*
1241                  * Read loaded_vmcs->cpu should be before fetching
1242                  * loaded_vmcs->loaded_vmcss_on_cpu_link.
1243                  * See the comments in __loaded_vmcs_clear().
1244                  */
1245                 smp_rmb();
1246
1247                 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
1248                          &per_cpu(loaded_vmcss_on_cpu, cpu));
1249                 crash_enable_local_vmclear(cpu);
1250                 local_irq_enable();
1251         }
1252
1253         if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
1254                 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
1255                 vmcs_load(vmx->loaded_vmcs->vmcs);
1256                 indirect_branch_prediction_barrier();
1257         }
1258
1259         if (!already_loaded) {
1260                 void *gdt = get_current_gdt_ro();
1261                 unsigned long sysenter_esp;
1262
1263                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1264
1265                 /*
1266                  * Linux uses per-cpu TSS and GDT, so set these when switching
1267                  * processors.  See 22.2.4.
1268                  */
1269                 vmcs_writel(HOST_TR_BASE,
1270                             (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
1271                 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt);   /* 22.2.4 */
1272
1273                 /*
1274                  * VM exits change the host TR limit to 0x67 after a VM
1275                  * exit.  This is okay, since 0x67 covers everything except
1276                  * the IO bitmap and have have code to handle the IO bitmap
1277                  * being lost after a VM exit.
1278                  */
1279                 BUILD_BUG_ON(IO_BITMAP_OFFSET - 1 != 0x67);
1280
1281                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
1282                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
1283
1284                 vmx->loaded_vmcs->cpu = cpu;
1285         }
1286
1287         /* Setup TSC multiplier */
1288         if (kvm_has_tsc_control &&
1289             vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
1290                 decache_tsc_multiplier(vmx);
1291
1292         vmx_vcpu_pi_load(vcpu, cpu);
1293         vmx->host_pkru = read_pkru();
1294         vmx->host_debugctlmsr = get_debugctlmsr();
1295 }
1296
1297 static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
1298 {
1299         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
1300
1301         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
1302                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
1303                 !kvm_vcpu_apicv_active(vcpu))
1304                 return;
1305
1306         /* Set SN when the vCPU is preempted */
1307         if (vcpu->preempted)
1308                 pi_set_sn(pi_desc);
1309 }
1310
1311 void vmx_vcpu_put(struct kvm_vcpu *vcpu)
1312 {
1313         vmx_vcpu_pi_put(vcpu);
1314
1315         vmx_prepare_switch_to_host(to_vmx(vcpu));
1316 }
1317
1318 static bool emulation_required(struct kvm_vcpu *vcpu)
1319 {
1320         return emulate_invalid_guest_state && !guest_state_valid(vcpu);
1321 }
1322
1323 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
1324
1325 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
1326 {
1327         unsigned long rflags, save_rflags;
1328
1329         if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
1330                 __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
1331                 rflags = vmcs_readl(GUEST_RFLAGS);
1332                 if (to_vmx(vcpu)->rmode.vm86_active) {
1333                         rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
1334                         save_rflags = to_vmx(vcpu)->rmode.save_rflags;
1335                         rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
1336                 }
1337                 to_vmx(vcpu)->rflags = rflags;
1338         }
1339         return to_vmx(vcpu)->rflags;
1340 }
1341
1342 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1343 {
1344         unsigned long old_rflags = vmx_get_rflags(vcpu);
1345
1346         __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
1347         to_vmx(vcpu)->rflags = rflags;
1348         if (to_vmx(vcpu)->rmode.vm86_active) {
1349                 to_vmx(vcpu)->rmode.save_rflags = rflags;
1350                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
1351         }
1352         vmcs_writel(GUEST_RFLAGS, rflags);
1353
1354         if ((old_rflags ^ to_vmx(vcpu)->rflags) & X86_EFLAGS_VM)
1355                 to_vmx(vcpu)->emulation_required = emulation_required(vcpu);
1356 }
1357
1358 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
1359 {
1360         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1361         int ret = 0;
1362
1363         if (interruptibility & GUEST_INTR_STATE_STI)
1364                 ret |= KVM_X86_SHADOW_INT_STI;
1365         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
1366                 ret |= KVM_X86_SHADOW_INT_MOV_SS;
1367
1368         return ret;
1369 }
1370
1371 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
1372 {
1373         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1374         u32 interruptibility = interruptibility_old;
1375
1376         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
1377
1378         if (mask & KVM_X86_SHADOW_INT_MOV_SS)
1379                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
1380         else if (mask & KVM_X86_SHADOW_INT_STI)
1381                 interruptibility |= GUEST_INTR_STATE_STI;
1382
1383         if ((interruptibility != interruptibility_old))
1384                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
1385 }
1386
1387 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data)
1388 {
1389         struct vcpu_vmx *vmx = to_vmx(vcpu);
1390         unsigned long value;
1391
1392         /*
1393          * Any MSR write that attempts to change bits marked reserved will
1394          * case a #GP fault.
1395          */
1396         if (data & vmx->pt_desc.ctl_bitmask)
1397                 return 1;
1398
1399         /*
1400          * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will
1401          * result in a #GP unless the same write also clears TraceEn.
1402          */
1403         if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) &&
1404                 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN))
1405                 return 1;
1406
1407         /*
1408          * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit
1409          * and FabricEn would cause #GP, if
1410          * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0
1411          */
1412         if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) &&
1413                 !(data & RTIT_CTL_FABRIC_EN) &&
1414                 !intel_pt_validate_cap(vmx->pt_desc.caps,
1415                                         PT_CAP_single_range_output))
1416                 return 1;
1417
1418         /*
1419          * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that
1420          * utilize encodings marked reserved will casue a #GP fault.
1421          */
1422         value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods);
1423         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) &&
1424                         !test_bit((data & RTIT_CTL_MTC_RANGE) >>
1425                         RTIT_CTL_MTC_RANGE_OFFSET, &value))
1426                 return 1;
1427         value = intel_pt_validate_cap(vmx->pt_desc.caps,
1428                                                 PT_CAP_cycle_thresholds);
1429         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
1430                         !test_bit((data & RTIT_CTL_CYC_THRESH) >>
1431                         RTIT_CTL_CYC_THRESH_OFFSET, &value))
1432                 return 1;
1433         value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods);
1434         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
1435                         !test_bit((data & RTIT_CTL_PSB_FREQ) >>
1436                         RTIT_CTL_PSB_FREQ_OFFSET, &value))
1437                 return 1;
1438
1439         /*
1440          * If ADDRx_CFG is reserved or the encodings is >2 will
1441          * cause a #GP fault.
1442          */
1443         value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET;
1444         if ((value && (vmx->pt_desc.addr_range < 1)) || (value > 2))
1445                 return 1;
1446         value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET;
1447         if ((value && (vmx->pt_desc.addr_range < 2)) || (value > 2))
1448                 return 1;
1449         value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET;
1450         if ((value && (vmx->pt_desc.addr_range < 3)) || (value > 2))
1451                 return 1;
1452         value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET;
1453         if ((value && (vmx->pt_desc.addr_range < 4)) || (value > 2))
1454                 return 1;
1455
1456         return 0;
1457 }
1458
1459
1460 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
1461 {
1462         unsigned long rip;
1463
1464         rip = kvm_rip_read(vcpu);
1465         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1466         kvm_rip_write(vcpu, rip);
1467
1468         /* skipping an emulated instruction also counts */
1469         vmx_set_interrupt_shadow(vcpu, 0);
1470 }
1471
1472 static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
1473 {
1474         /*
1475          * Ensure that we clear the HLT state in the VMCS.  We don't need to
1476          * explicitly skip the instruction because if the HLT state is set,
1477          * then the instruction is already executing and RIP has already been
1478          * advanced.
1479          */
1480         if (kvm_hlt_in_guest(vcpu->kvm) &&
1481                         vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
1482                 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
1483 }
1484
1485 static void vmx_queue_exception(struct kvm_vcpu *vcpu)
1486 {
1487         struct vcpu_vmx *vmx = to_vmx(vcpu);
1488         unsigned nr = vcpu->arch.exception.nr;
1489         bool has_error_code = vcpu->arch.exception.has_error_code;
1490         u32 error_code = vcpu->arch.exception.error_code;
1491         u32 intr_info = nr | INTR_INFO_VALID_MASK;
1492
1493         kvm_deliver_exception_payload(vcpu);
1494
1495         if (has_error_code) {
1496                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
1497                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
1498         }
1499
1500         if (vmx->rmode.vm86_active) {
1501                 int inc_eip = 0;
1502                 if (kvm_exception_is_soft(nr))
1503                         inc_eip = vcpu->arch.event_exit_inst_len;
1504                 if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
1505                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1506                 return;
1507         }
1508
1509         WARN_ON_ONCE(vmx->emulation_required);
1510
1511         if (kvm_exception_is_soft(nr)) {
1512                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
1513                              vmx->vcpu.arch.event_exit_inst_len);
1514                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
1515         } else
1516                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
1517
1518         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
1519
1520         vmx_clear_hlt(vcpu);
1521 }
1522
1523 static bool vmx_rdtscp_supported(void)
1524 {
1525         return cpu_has_vmx_rdtscp();
1526 }
1527
1528 static bool vmx_invpcid_supported(void)
1529 {
1530         return cpu_has_vmx_invpcid();
1531 }
1532
1533 /*
1534  * Swap MSR entry in host/guest MSR entry array.
1535  */
1536 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
1537 {
1538         struct shared_msr_entry tmp;
1539
1540         tmp = vmx->guest_msrs[to];
1541         vmx->guest_msrs[to] = vmx->guest_msrs[from];
1542         vmx->guest_msrs[from] = tmp;
1543 }
1544
1545 /*
1546  * Set up the vmcs to automatically save and restore system
1547  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
1548  * mode, as fiddling with msrs is very expensive.
1549  */
1550 static void setup_msrs(struct vcpu_vmx *vmx)
1551 {
1552         int save_nmsrs, index;
1553
1554         save_nmsrs = 0;
1555 #ifdef CONFIG_X86_64
1556         /*
1557          * The SYSCALL MSRs are only needed on long mode guests, and only
1558          * when EFER.SCE is set.
1559          */
1560         if (is_long_mode(&vmx->vcpu) && (vmx->vcpu.arch.efer & EFER_SCE)) {
1561                 index = __find_msr_index(vmx, MSR_STAR);
1562                 if (index >= 0)
1563                         move_msr_up(vmx, index, save_nmsrs++);
1564                 index = __find_msr_index(vmx, MSR_LSTAR);
1565                 if (index >= 0)
1566                         move_msr_up(vmx, index, save_nmsrs++);
1567                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
1568                 if (index >= 0)
1569                         move_msr_up(vmx, index, save_nmsrs++);
1570         }
1571 #endif
1572         index = __find_msr_index(vmx, MSR_EFER);
1573         if (index >= 0 && update_transition_efer(vmx, index))
1574                 move_msr_up(vmx, index, save_nmsrs++);
1575         index = __find_msr_index(vmx, MSR_TSC_AUX);
1576         if (index >= 0 && guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP))
1577                 move_msr_up(vmx, index, save_nmsrs++);
1578
1579         vmx->save_nmsrs = save_nmsrs;
1580         vmx->guest_msrs_dirty = true;
1581
1582         if (cpu_has_vmx_msr_bitmap())
1583                 vmx_update_msr_bitmap(&vmx->vcpu);
1584 }
1585
1586 static u64 vmx_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
1587 {
1588         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1589
1590         if (is_guest_mode(vcpu) &&
1591             (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING))
1592                 return vcpu->arch.tsc_offset - vmcs12->tsc_offset;
1593
1594         return vcpu->arch.tsc_offset;
1595 }
1596
1597 static u64 vmx_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1598 {
1599         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1600         u64 g_tsc_offset = 0;
1601
1602         /*
1603          * We're here if L1 chose not to trap WRMSR to TSC. According
1604          * to the spec, this should set L1's TSC; The offset that L1
1605          * set for L2 remains unchanged, and still needs to be added
1606          * to the newly set TSC to get L2's TSC.
1607          */
1608         if (is_guest_mode(vcpu) &&
1609             (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING))
1610                 g_tsc_offset = vmcs12->tsc_offset;
1611
1612         trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1613                                    vcpu->arch.tsc_offset - g_tsc_offset,
1614                                    offset);
1615         vmcs_write64(TSC_OFFSET, offset + g_tsc_offset);
1616         return offset + g_tsc_offset;
1617 }
1618
1619 /*
1620  * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
1621  * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
1622  * all guests if the "nested" module option is off, and can also be disabled
1623  * for a single guest by disabling its VMX cpuid bit.
1624  */
1625 bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
1626 {
1627         return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
1628 }
1629
1630 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
1631                                                  uint64_t val)
1632 {
1633         uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
1634
1635         return !(val & ~valid_bits);
1636 }
1637
1638 static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
1639 {
1640         switch (msr->index) {
1641         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1642                 if (!nested)
1643                         return 1;
1644                 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data);
1645         default:
1646                 return 1;
1647         }
1648
1649         return 0;
1650 }
1651
1652 /*
1653  * Reads an msr value (of 'msr_index') into 'pdata'.
1654  * Returns 0 on success, non-0 otherwise.
1655  * Assumes vcpu_load() was already called.
1656  */
1657 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1658 {
1659         struct vcpu_vmx *vmx = to_vmx(vcpu);
1660         struct shared_msr_entry *msr;
1661         u32 index;
1662
1663         switch (msr_info->index) {
1664 #ifdef CONFIG_X86_64
1665         case MSR_FS_BASE:
1666                 msr_info->data = vmcs_readl(GUEST_FS_BASE);
1667                 break;
1668         case MSR_GS_BASE:
1669                 msr_info->data = vmcs_readl(GUEST_GS_BASE);
1670                 break;
1671         case MSR_KERNEL_GS_BASE:
1672                 msr_info->data = vmx_read_guest_kernel_gs_base(vmx);
1673                 break;
1674 #endif
1675         case MSR_EFER:
1676                 return kvm_get_msr_common(vcpu, msr_info);
1677         case MSR_IA32_SPEC_CTRL:
1678                 if (!msr_info->host_initiated &&
1679                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
1680                         return 1;
1681
1682                 msr_info->data = to_vmx(vcpu)->spec_ctrl;
1683                 break;
1684         case MSR_IA32_ARCH_CAPABILITIES:
1685                 if (!msr_info->host_initiated &&
1686                     !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
1687                         return 1;
1688                 msr_info->data = to_vmx(vcpu)->arch_capabilities;
1689                 break;
1690         case MSR_IA32_SYSENTER_CS:
1691                 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
1692                 break;
1693         case MSR_IA32_SYSENTER_EIP:
1694                 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
1695                 break;
1696         case MSR_IA32_SYSENTER_ESP:
1697                 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
1698                 break;
1699         case MSR_IA32_BNDCFGS:
1700                 if (!kvm_mpx_supported() ||
1701                     (!msr_info->host_initiated &&
1702                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
1703                         return 1;
1704                 msr_info->data = vmcs_read64(GUEST_BNDCFGS);
1705                 break;
1706         case MSR_IA32_MCG_EXT_CTL:
1707                 if (!msr_info->host_initiated &&
1708                     !(vmx->msr_ia32_feature_control &
1709                       FEATURE_CONTROL_LMCE))
1710                         return 1;
1711                 msr_info->data = vcpu->arch.mcg_ext_ctl;
1712                 break;
1713         case MSR_IA32_FEATURE_CONTROL:
1714                 msr_info->data = vmx->msr_ia32_feature_control;
1715                 break;
1716         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1717                 if (!nested_vmx_allowed(vcpu))
1718                         return 1;
1719                 return vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index,
1720                                        &msr_info->data);
1721         case MSR_IA32_XSS:
1722                 if (!vmx_xsaves_supported())
1723                         return 1;
1724                 msr_info->data = vcpu->arch.ia32_xss;
1725                 break;
1726         case MSR_IA32_RTIT_CTL:
1727                 if (pt_mode != PT_MODE_HOST_GUEST)
1728                         return 1;
1729                 msr_info->data = vmx->pt_desc.guest.ctl;
1730                 break;
1731         case MSR_IA32_RTIT_STATUS:
1732                 if (pt_mode != PT_MODE_HOST_GUEST)
1733                         return 1;
1734                 msr_info->data = vmx->pt_desc.guest.status;
1735                 break;
1736         case MSR_IA32_RTIT_CR3_MATCH:
1737                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1738                         !intel_pt_validate_cap(vmx->pt_desc.caps,
1739                                                 PT_CAP_cr3_filtering))
1740                         return 1;
1741                 msr_info->data = vmx->pt_desc.guest.cr3_match;
1742                 break;
1743         case MSR_IA32_RTIT_OUTPUT_BASE:
1744                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1745                         (!intel_pt_validate_cap(vmx->pt_desc.caps,
1746                                         PT_CAP_topa_output) &&
1747                          !intel_pt_validate_cap(vmx->pt_desc.caps,
1748                                         PT_CAP_single_range_output)))
1749                         return 1;
1750                 msr_info->data = vmx->pt_desc.guest.output_base;
1751                 break;
1752         case MSR_IA32_RTIT_OUTPUT_MASK:
1753                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1754                         (!intel_pt_validate_cap(vmx->pt_desc.caps,
1755                                         PT_CAP_topa_output) &&
1756                          !intel_pt_validate_cap(vmx->pt_desc.caps,
1757                                         PT_CAP_single_range_output)))
1758                         return 1;
1759                 msr_info->data = vmx->pt_desc.guest.output_mask;
1760                 break;
1761         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
1762                 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
1763                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1764                         (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps,
1765                                         PT_CAP_num_address_ranges)))
1766                         return 1;
1767                 if (index % 2)
1768                         msr_info->data = vmx->pt_desc.guest.addr_b[index / 2];
1769                 else
1770                         msr_info->data = vmx->pt_desc.guest.addr_a[index / 2];
1771                 break;
1772         case MSR_TSC_AUX:
1773                 if (!msr_info->host_initiated &&
1774                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1775                         return 1;
1776                 /* Else, falls through */
1777         default:
1778                 msr = find_msr_entry(vmx, msr_info->index);
1779                 if (msr) {
1780                         msr_info->data = msr->data;
1781                         break;
1782                 }
1783                 return kvm_get_msr_common(vcpu, msr_info);
1784         }
1785
1786         return 0;
1787 }
1788
1789 /*
1790  * Writes msr value into into the appropriate "register".
1791  * Returns 0 on success, non-0 otherwise.
1792  * Assumes vcpu_load() was already called.
1793  */
1794 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1795 {
1796         struct vcpu_vmx *vmx = to_vmx(vcpu);
1797         struct shared_msr_entry *msr;
1798         int ret = 0;
1799         u32 msr_index = msr_info->index;
1800         u64 data = msr_info->data;
1801         u32 index;
1802
1803         switch (msr_index) {
1804         case MSR_EFER:
1805                 ret = kvm_set_msr_common(vcpu, msr_info);
1806                 break;
1807 #ifdef CONFIG_X86_64
1808         case MSR_FS_BASE:
1809                 vmx_segment_cache_clear(vmx);
1810                 vmcs_writel(GUEST_FS_BASE, data);
1811                 break;
1812         case MSR_GS_BASE:
1813                 vmx_segment_cache_clear(vmx);
1814                 vmcs_writel(GUEST_GS_BASE, data);
1815                 break;
1816         case MSR_KERNEL_GS_BASE:
1817                 vmx_write_guest_kernel_gs_base(vmx, data);
1818                 break;
1819 #endif
1820         case MSR_IA32_SYSENTER_CS:
1821                 vmcs_write32(GUEST_SYSENTER_CS, data);
1822                 break;
1823         case MSR_IA32_SYSENTER_EIP:
1824                 vmcs_writel(GUEST_SYSENTER_EIP, data);
1825                 break;
1826         case MSR_IA32_SYSENTER_ESP:
1827                 vmcs_writel(GUEST_SYSENTER_ESP, data);
1828                 break;
1829         case MSR_IA32_BNDCFGS:
1830                 if (!kvm_mpx_supported() ||
1831                     (!msr_info->host_initiated &&
1832                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
1833                         return 1;
1834                 if (is_noncanonical_address(data & PAGE_MASK, vcpu) ||
1835                     (data & MSR_IA32_BNDCFGS_RSVD))
1836                         return 1;
1837                 vmcs_write64(GUEST_BNDCFGS, data);
1838                 break;
1839         case MSR_IA32_SPEC_CTRL:
1840                 if (!msr_info->host_initiated &&
1841                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
1842                         return 1;
1843
1844                 /* The STIBP bit doesn't fault even if it's not advertised */
1845                 if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD))
1846                         return 1;
1847
1848                 vmx->spec_ctrl = data;
1849
1850                 if (!data)
1851                         break;
1852
1853                 /*
1854                  * For non-nested:
1855                  * When it's written (to non-zero) for the first time, pass
1856                  * it through.
1857                  *
1858                  * For nested:
1859                  * The handling of the MSR bitmap for L2 guests is done in
1860                  * nested_vmx_merge_msr_bitmap. We should not touch the
1861                  * vmcs02.msr_bitmap here since it gets completely overwritten
1862                  * in the merging. We update the vmcs01 here for L1 as well
1863                  * since it will end up touching the MSR anyway now.
1864                  */
1865                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap,
1866                                               MSR_IA32_SPEC_CTRL,
1867                                               MSR_TYPE_RW);
1868                 break;
1869         case MSR_IA32_PRED_CMD:
1870                 if (!msr_info->host_initiated &&
1871                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
1872                         return 1;
1873
1874                 if (data & ~PRED_CMD_IBPB)
1875                         return 1;
1876
1877                 if (!data)
1878                         break;
1879
1880                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
1881
1882                 /*
1883                  * For non-nested:
1884                  * When it's written (to non-zero) for the first time, pass
1885                  * it through.
1886                  *
1887                  * For nested:
1888                  * The handling of the MSR bitmap for L2 guests is done in
1889                  * nested_vmx_merge_msr_bitmap. We should not touch the
1890                  * vmcs02.msr_bitmap here since it gets completely overwritten
1891                  * in the merging.
1892                  */
1893                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, MSR_IA32_PRED_CMD,
1894                                               MSR_TYPE_W);
1895                 break;
1896         case MSR_IA32_ARCH_CAPABILITIES:
1897                 if (!msr_info->host_initiated)
1898                         return 1;
1899                 vmx->arch_capabilities = data;
1900                 break;
1901         case MSR_IA32_CR_PAT:
1902                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
1903                         if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
1904                                 return 1;
1905                         vmcs_write64(GUEST_IA32_PAT, data);
1906                         vcpu->arch.pat = data;
1907                         break;
1908                 }
1909                 ret = kvm_set_msr_common(vcpu, msr_info);
1910                 break;
1911         case MSR_IA32_TSC_ADJUST:
1912                 ret = kvm_set_msr_common(vcpu, msr_info);
1913                 break;
1914         case MSR_IA32_MCG_EXT_CTL:
1915                 if ((!msr_info->host_initiated &&
1916                      !(to_vmx(vcpu)->msr_ia32_feature_control &
1917                        FEATURE_CONTROL_LMCE)) ||
1918                     (data & ~MCG_EXT_CTL_LMCE_EN))
1919                         return 1;
1920                 vcpu->arch.mcg_ext_ctl = data;
1921                 break;
1922         case MSR_IA32_FEATURE_CONTROL:
1923                 if (!vmx_feature_control_msr_valid(vcpu, data) ||
1924                     (to_vmx(vcpu)->msr_ia32_feature_control &
1925                      FEATURE_CONTROL_LOCKED && !msr_info->host_initiated))
1926                         return 1;
1927                 vmx->msr_ia32_feature_control = data;
1928                 if (msr_info->host_initiated && data == 0)
1929                         vmx_leave_nested(vcpu);
1930                 break;
1931         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1932                 if (!msr_info->host_initiated)
1933                         return 1; /* they are read-only */
1934                 if (!nested_vmx_allowed(vcpu))
1935                         return 1;
1936                 return vmx_set_vmx_msr(vcpu, msr_index, data);
1937         case MSR_IA32_XSS:
1938                 if (!vmx_xsaves_supported())
1939                         return 1;
1940                 /*
1941                  * The only supported bit as of Skylake is bit 8, but
1942                  * it is not supported on KVM.
1943                  */
1944                 if (data != 0)
1945                         return 1;
1946                 vcpu->arch.ia32_xss = data;
1947                 if (vcpu->arch.ia32_xss != host_xss)
1948                         add_atomic_switch_msr(vmx, MSR_IA32_XSS,
1949                                 vcpu->arch.ia32_xss, host_xss, false);
1950                 else
1951                         clear_atomic_switch_msr(vmx, MSR_IA32_XSS);
1952                 break;
1953         case MSR_IA32_RTIT_CTL:
1954                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1955                         vmx_rtit_ctl_check(vcpu, data) ||
1956                         vmx->nested.vmxon)
1957                         return 1;
1958                 vmcs_write64(GUEST_IA32_RTIT_CTL, data);
1959                 vmx->pt_desc.guest.ctl = data;
1960                 pt_update_intercept_for_msr(vmx);
1961                 break;
1962         case MSR_IA32_RTIT_STATUS:
1963                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1964                         (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) ||
1965                         (data & MSR_IA32_RTIT_STATUS_MASK))
1966                         return 1;
1967                 vmx->pt_desc.guest.status = data;
1968                 break;
1969         case MSR_IA32_RTIT_CR3_MATCH:
1970                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1971                         (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) ||
1972                         !intel_pt_validate_cap(vmx->pt_desc.caps,
1973                                                 PT_CAP_cr3_filtering))
1974                         return 1;
1975                 vmx->pt_desc.guest.cr3_match = data;
1976                 break;
1977         case MSR_IA32_RTIT_OUTPUT_BASE:
1978                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1979                         (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) ||
1980                         (!intel_pt_validate_cap(vmx->pt_desc.caps,
1981                                         PT_CAP_topa_output) &&
1982                          !intel_pt_validate_cap(vmx->pt_desc.caps,
1983                                         PT_CAP_single_range_output)) ||
1984                         (data & MSR_IA32_RTIT_OUTPUT_BASE_MASK))
1985                         return 1;
1986                 vmx->pt_desc.guest.output_base = data;
1987                 break;
1988         case MSR_IA32_RTIT_OUTPUT_MASK:
1989                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
1990                         (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) ||
1991                         (!intel_pt_validate_cap(vmx->pt_desc.caps,
1992                                         PT_CAP_topa_output) &&
1993                          !intel_pt_validate_cap(vmx->pt_desc.caps,
1994                                         PT_CAP_single_range_output)))
1995                         return 1;
1996                 vmx->pt_desc.guest.output_mask = data;
1997                 break;
1998         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
1999                 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
2000                 if ((pt_mode != PT_MODE_HOST_GUEST) ||
2001                         (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) ||
2002                         (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps,
2003                                         PT_CAP_num_address_ranges)))
2004                         return 1;
2005                 if (index % 2)
2006                         vmx->pt_desc.guest.addr_b[index / 2] = data;
2007                 else
2008                         vmx->pt_desc.guest.addr_a[index / 2] = data;
2009                 break;
2010         case MSR_TSC_AUX:
2011                 if (!msr_info->host_initiated &&
2012                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
2013                         return 1;
2014                 /* Check reserved bit, higher 32 bits should be zero */
2015                 if ((data >> 32) != 0)
2016                         return 1;
2017                 /* Else, falls through */
2018         default:
2019                 msr = find_msr_entry(vmx, msr_index);
2020                 if (msr) {
2021                         u64 old_msr_data = msr->data;
2022                         msr->data = data;
2023                         if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
2024                                 preempt_disable();
2025                                 ret = kvm_set_shared_msr(msr->index, msr->data,
2026                                                          msr->mask);
2027                                 preempt_enable();
2028                                 if (ret)
2029                                         msr->data = old_msr_data;
2030                         }
2031                         break;
2032                 }
2033                 ret = kvm_set_msr_common(vcpu, msr_info);
2034         }
2035
2036         return ret;
2037 }
2038
2039 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2040 {
2041         __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
2042         switch (reg) {
2043         case VCPU_REGS_RSP:
2044                 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
2045                 break;
2046         case VCPU_REGS_RIP:
2047                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
2048                 break;
2049         case VCPU_EXREG_PDPTR:
2050                 if (enable_ept)
2051                         ept_save_pdptrs(vcpu);
2052                 break;
2053         default:
2054                 break;
2055         }
2056 }
2057
2058 static __init int cpu_has_kvm_support(void)
2059 {
2060         return cpu_has_vmx();
2061 }
2062
2063 static __init int vmx_disabled_by_bios(void)
2064 {
2065         u64 msr;
2066
2067         rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
2068         if (msr & FEATURE_CONTROL_LOCKED) {
2069                 /* launched w/ TXT and VMX disabled */
2070                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
2071                         && tboot_enabled())
2072                         return 1;
2073                 /* launched w/o TXT and VMX only enabled w/ TXT */
2074                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
2075                         && (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
2076                         && !tboot_enabled()) {
2077                         printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
2078                                 "activate TXT before enabling KVM\n");
2079                         return 1;
2080                 }
2081                 /* launched w/o TXT and VMX disabled */
2082                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
2083                         && !tboot_enabled())
2084                         return 1;
2085         }
2086
2087         return 0;
2088 }
2089
2090 static void kvm_cpu_vmxon(u64 addr)
2091 {
2092         cr4_set_bits(X86_CR4_VMXE);
2093         intel_pt_handle_vmx(1);
2094
2095         asm volatile ("vmxon %0" : : "m"(addr));
2096 }
2097
2098 static int hardware_enable(void)
2099 {
2100         int cpu = raw_smp_processor_id();
2101         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
2102         u64 old, test_bits;
2103
2104         if (cr4_read_shadow() & X86_CR4_VMXE)
2105                 return -EBUSY;
2106
2107         /*
2108          * This can happen if we hot-added a CPU but failed to allocate
2109          * VP assist page for it.
2110          */
2111         if (static_branch_unlikely(&enable_evmcs) &&
2112             !hv_get_vp_assist_page(cpu))
2113                 return -EFAULT;
2114
2115         INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
2116         INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
2117         spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
2118
2119         /*
2120          * Now we can enable the vmclear operation in kdump
2121          * since the loaded_vmcss_on_cpu list on this cpu
2122          * has been initialized.
2123          *
2124          * Though the cpu is not in VMX operation now, there
2125          * is no problem to enable the vmclear operation
2126          * for the loaded_vmcss_on_cpu list is empty!
2127          */
2128         crash_enable_local_vmclear(cpu);
2129
2130         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
2131
2132         test_bits = FEATURE_CONTROL_LOCKED;
2133         test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
2134         if (tboot_enabled())
2135                 test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;
2136
2137         if ((old & test_bits) != test_bits) {
2138                 /* enable and lock */
2139                 wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
2140         }
2141         kvm_cpu_vmxon(phys_addr);
2142         if (enable_ept)
2143                 ept_sync_global();
2144
2145         return 0;
2146 }
2147
2148 static void vmclear_local_loaded_vmcss(void)
2149 {
2150         int cpu = raw_smp_processor_id();
2151         struct loaded_vmcs *v, *n;
2152
2153         list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
2154                                  loaded_vmcss_on_cpu_link)
2155                 __loaded_vmcs_clear(v);
2156 }
2157
2158
2159 /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
2160  * tricks.
2161  */
2162 static void kvm_cpu_vmxoff(void)
2163 {
2164         asm volatile (__ex("vmxoff"));
2165
2166         intel_pt_handle_vmx(0);
2167         cr4_clear_bits(X86_CR4_VMXE);
2168 }
2169
2170 static void hardware_disable(void)
2171 {
2172         vmclear_local_loaded_vmcss();
2173         kvm_cpu_vmxoff();
2174 }
2175
2176 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
2177                                       u32 msr, u32 *result)
2178 {
2179         u32 vmx_msr_low, vmx_msr_high;
2180         u32 ctl = ctl_min | ctl_opt;
2181
2182         rdmsr(msr, vmx_msr_low, vmx_msr_high);
2183
2184         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
2185         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
2186
2187         /* Ensure minimum (required) set of control bits are supported. */
2188         if (ctl_min & ~ctl)
2189                 return -EIO;
2190
2191         *result = ctl;
2192         return 0;
2193 }
2194
2195 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf,
2196                                     struct vmx_capability *vmx_cap)
2197 {
2198         u32 vmx_msr_low, vmx_msr_high;
2199         u32 min, opt, min2, opt2;
2200         u32 _pin_based_exec_control = 0;
2201         u32 _cpu_based_exec_control = 0;
2202         u32 _cpu_based_2nd_exec_control = 0;
2203         u32 _vmexit_control = 0;
2204         u32 _vmentry_control = 0;
2205
2206         memset(vmcs_conf, 0, sizeof(*vmcs_conf));
2207         min = CPU_BASED_HLT_EXITING |
2208 #ifdef CONFIG_X86_64
2209               CPU_BASED_CR8_LOAD_EXITING |
2210               CPU_BASED_CR8_STORE_EXITING |
2211 #endif
2212               CPU_BASED_CR3_LOAD_EXITING |
2213               CPU_BASED_CR3_STORE_EXITING |
2214               CPU_BASED_UNCOND_IO_EXITING |
2215               CPU_BASED_MOV_DR_EXITING |
2216               CPU_BASED_USE_TSC_OFFSETING |
2217               CPU_BASED_MWAIT_EXITING |
2218               CPU_BASED_MONITOR_EXITING |
2219               CPU_BASED_INVLPG_EXITING |
2220               CPU_BASED_RDPMC_EXITING;
2221
2222         opt = CPU_BASED_TPR_SHADOW |
2223               CPU_BASED_USE_MSR_BITMAPS |
2224               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2225         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
2226                                 &_cpu_based_exec_control) < 0)
2227                 return -EIO;
2228 #ifdef CONFIG_X86_64
2229         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2230                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
2231                                            ~CPU_BASED_CR8_STORE_EXITING;
2232 #endif
2233         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
2234                 min2 = 0;
2235                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2236                         SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2237                         SECONDARY_EXEC_WBINVD_EXITING |
2238                         SECONDARY_EXEC_ENABLE_VPID |
2239                         SECONDARY_EXEC_ENABLE_EPT |
2240                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
2241                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
2242                         SECONDARY_EXEC_DESC |
2243                         SECONDARY_EXEC_RDTSCP |
2244                         SECONDARY_EXEC_ENABLE_INVPCID |
2245                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
2246                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2247                         SECONDARY_EXEC_SHADOW_VMCS |
2248                         SECONDARY_EXEC_XSAVES |
2249                         SECONDARY_EXEC_RDSEED_EXITING |
2250                         SECONDARY_EXEC_RDRAND_EXITING |
2251                         SECONDARY_EXEC_ENABLE_PML |
2252                         SECONDARY_EXEC_TSC_SCALING |
2253                         SECONDARY_EXEC_PT_USE_GPA |
2254                         SECONDARY_EXEC_PT_CONCEAL_VMX |
2255                         SECONDARY_EXEC_ENABLE_VMFUNC |
2256                         SECONDARY_EXEC_ENCLS_EXITING;
2257                 if (adjust_vmx_controls(min2, opt2,
2258                                         MSR_IA32_VMX_PROCBASED_CTLS2,
2259                                         &_cpu_based_2nd_exec_control) < 0)
2260                         return -EIO;
2261         }
2262 #ifndef CONFIG_X86_64
2263         if (!(_cpu_based_2nd_exec_control &
2264                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
2265                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
2266 #endif
2267
2268         if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2269                 _cpu_based_2nd_exec_control &= ~(
2270                                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2271                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2272                                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
2273
2274         rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP,
2275                 &vmx_cap->ept, &vmx_cap->vpid);
2276
2277         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
2278                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
2279                    enabled */
2280                 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
2281                                              CPU_BASED_CR3_STORE_EXITING |
2282                                              CPU_BASED_INVLPG_EXITING);
2283         } else if (vmx_cap->ept) {
2284                 vmx_cap->ept = 0;
2285                 pr_warn_once("EPT CAP should not exist if not support "
2286                                 "1-setting enable EPT VM-execution control\n");
2287         }
2288         if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) &&
2289                 vmx_cap->vpid) {
2290                 vmx_cap->vpid = 0;
2291                 pr_warn_once("VPID CAP should not exist if not support "
2292                                 "1-setting enable VPID VM-execution control\n");
2293         }
2294
2295         min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
2296 #ifdef CONFIG_X86_64
2297         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
2298 #endif
2299         opt = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
2300               VM_EXIT_SAVE_IA32_PAT |
2301               VM_EXIT_LOAD_IA32_PAT |
2302               VM_EXIT_LOAD_IA32_EFER |
2303               VM_EXIT_CLEAR_BNDCFGS |
2304               VM_EXIT_PT_CONCEAL_PIP |
2305               VM_EXIT_CLEAR_IA32_RTIT_CTL;
2306         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
2307                                 &_vmexit_control) < 0)
2308                 return -EIO;
2309
2310         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
2311         opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
2312                  PIN_BASED_VMX_PREEMPTION_TIMER;
2313         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
2314                                 &_pin_based_exec_control) < 0)
2315                 return -EIO;
2316
2317         if (cpu_has_broken_vmx_preemption_timer())
2318                 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
2319         if (!(_cpu_based_2nd_exec_control &
2320                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
2321                 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
2322
2323         min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
2324         opt = VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL |
2325               VM_ENTRY_LOAD_IA32_PAT |
2326               VM_ENTRY_LOAD_IA32_EFER |
2327               VM_ENTRY_LOAD_BNDCFGS |
2328               VM_ENTRY_PT_CONCEAL_PIP |
2329               VM_ENTRY_LOAD_IA32_RTIT_CTL;
2330         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
2331                                 &_vmentry_control) < 0)
2332                 return -EIO;
2333
2334         /*
2335          * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they
2336          * can't be used due to an errata where VM Exit may incorrectly clear
2337          * IA32_PERF_GLOBAL_CTRL[34:32].  Workaround the errata by using the
2338          * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL.
2339          */
2340         if (boot_cpu_data.x86 == 0x6) {
2341                 switch (boot_cpu_data.x86_model) {
2342                 case 26: /* AAK155 */
2343                 case 30: /* AAP115 */
2344                 case 37: /* AAT100 */
2345                 case 44: /* BC86,AAY89,BD102 */
2346                 case 46: /* BA97 */
2347                         _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
2348                         _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
2349                         pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
2350                                         "does not work properly. Using workaround\n");
2351                         break;
2352                 default:
2353                         break;
2354                 }
2355         }
2356
2357
2358         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
2359
2360         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
2361         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
2362                 return -EIO;
2363
2364 #ifdef CONFIG_X86_64
2365         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
2366         if (vmx_msr_high & (1u<<16))
2367                 return -EIO;
2368 #endif
2369
2370         /* Require Write-Back (WB) memory type for VMCS accesses. */
2371         if (((vmx_msr_high >> 18) & 15) != 6)
2372                 return -EIO;
2373
2374         vmcs_conf->size = vmx_msr_high & 0x1fff;
2375         vmcs_conf->order = get_order(vmcs_conf->size);
2376         vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
2377
2378         vmcs_conf->revision_id = vmx_msr_low;
2379
2380         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
2381         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
2382         vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
2383         vmcs_conf->vmexit_ctrl         = _vmexit_control;
2384         vmcs_conf->vmentry_ctrl        = _vmentry_control;
2385
2386         if (static_branch_unlikely(&enable_evmcs))
2387                 evmcs_sanitize_exec_ctrls(vmcs_conf);
2388
2389         return 0;
2390 }
2391
2392 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu)
2393 {
2394         int node = cpu_to_node(cpu);
2395         struct page *pages;
2396         struct vmcs *vmcs;
2397
2398         pages = __alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);
2399         if (!pages)
2400                 return NULL;
2401         vmcs = page_address(pages);
2402         memset(vmcs, 0, vmcs_config.size);
2403
2404         /* KVM supports Enlightened VMCS v1 only */
2405         if (static_branch_unlikely(&enable_evmcs))
2406                 vmcs->hdr.revision_id = KVM_EVMCS_VERSION;
2407         else
2408                 vmcs->hdr.revision_id = vmcs_config.revision_id;
2409
2410         if (shadow)
2411                 vmcs->hdr.shadow_vmcs = 1;
2412         return vmcs;
2413 }
2414
2415 void free_vmcs(struct vmcs *vmcs)
2416 {
2417         free_pages((unsigned long)vmcs, vmcs_config.order);
2418 }
2419
2420 /*
2421  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
2422  */
2423 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
2424 {
2425         if (!loaded_vmcs->vmcs)
2426                 return;
2427         loaded_vmcs_clear(loaded_vmcs);
2428         free_vmcs(loaded_vmcs->vmcs);
2429         loaded_vmcs->vmcs = NULL;
2430         if (loaded_vmcs->msr_bitmap)
2431                 free_page((unsigned long)loaded_vmcs->msr_bitmap);
2432         WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
2433 }
2434
2435 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
2436 {
2437         loaded_vmcs->vmcs = alloc_vmcs(false);
2438         if (!loaded_vmcs->vmcs)
2439                 return -ENOMEM;
2440
2441         loaded_vmcs->shadow_vmcs = NULL;
2442         loaded_vmcs_init(loaded_vmcs);
2443
2444         if (cpu_has_vmx_msr_bitmap()) {
2445                 loaded_vmcs->msr_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
2446                 if (!loaded_vmcs->msr_bitmap)
2447                         goto out_vmcs;
2448                 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
2449
2450                 if (IS_ENABLED(CONFIG_HYPERV) &&
2451                     static_branch_unlikely(&enable_evmcs) &&
2452                     (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) {
2453                         struct hv_enlightened_vmcs *evmcs =
2454                                 (struct hv_enlightened_vmcs *)loaded_vmcs->vmcs;
2455
2456                         evmcs->hv_enlightenments_control.msr_bitmap = 1;
2457                 }
2458         }
2459
2460         memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state));
2461
2462         return 0;
2463
2464 out_vmcs:
2465         free_loaded_vmcs(loaded_vmcs);
2466         return -ENOMEM;
2467 }
2468
2469 static void free_kvm_area(void)
2470 {
2471         int cpu;
2472
2473         for_each_possible_cpu(cpu) {
2474                 free_vmcs(per_cpu(vmxarea, cpu));
2475                 per_cpu(vmxarea, cpu) = NULL;
2476         }
2477 }
2478
2479 static __init int alloc_kvm_area(void)
2480 {
2481         int cpu;
2482
2483         for_each_possible_cpu(cpu) {
2484                 struct vmcs *vmcs;
2485
2486                 vmcs = alloc_vmcs_cpu(false, cpu);
2487                 if (!vmcs) {
2488                         free_kvm_area();
2489                         return -ENOMEM;
2490                 }
2491
2492                 /*
2493                  * When eVMCS is enabled, alloc_vmcs_cpu() sets
2494                  * vmcs->revision_id to KVM_EVMCS_VERSION instead of
2495                  * revision_id reported by MSR_IA32_VMX_BASIC.
2496                  *
2497                  * However, even though not explicitly documented by
2498                  * TLFS, VMXArea passed as VMXON argument should
2499                  * still be marked with revision_id reported by
2500                  * physical CPU.
2501                  */
2502                 if (static_branch_unlikely(&enable_evmcs))
2503                         vmcs->hdr.revision_id = vmcs_config.revision_id;
2504
2505                 per_cpu(vmxarea, cpu) = vmcs;
2506         }
2507         return 0;
2508 }
2509
2510 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
2511                 struct kvm_segment *save)
2512 {
2513         if (!emulate_invalid_guest_state) {
2514                 /*
2515                  * CS and SS RPL should be equal during guest entry according
2516                  * to VMX spec, but in reality it is not always so. Since vcpu
2517                  * is in the middle of the transition from real mode to
2518                  * protected mode it is safe to assume that RPL 0 is a good
2519                  * default value.
2520                  */
2521                 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
2522                         save->selector &= ~SEGMENT_RPL_MASK;
2523                 save->dpl = save->selector & SEGMENT_RPL_MASK;
2524                 save->s = 1;
2525         }
2526         vmx_set_segment(vcpu, save, seg);
2527 }
2528
2529 static void enter_pmode(struct kvm_vcpu *vcpu)
2530 {
2531         unsigned long flags;
2532         struct vcpu_vmx *vmx = to_vmx(vcpu);
2533
2534         /*
2535          * Update real mode segment cache. It may be not up-to-date if sement
2536          * register was written while vcpu was in a guest mode.
2537          */
2538         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2539         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2540         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2541         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
2542         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
2543         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
2544
2545         vmx->rmode.vm86_active = 0;
2546
2547         vmx_segment_cache_clear(vmx);
2548
2549         vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2550
2551         flags = vmcs_readl(GUEST_RFLAGS);
2552         flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
2553         flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
2554         vmcs_writel(GUEST_RFLAGS, flags);
2555
2556         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
2557                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
2558
2559         update_exception_bitmap(vcpu);
2560
2561         fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2562         fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2563         fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2564         fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2565         fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
2566         fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
2567 }
2568
2569 static void fix_rmode_seg(int seg, struct kvm_segment *save)
2570 {
2571         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2572         struct kvm_segment var = *save;
2573
2574         var.dpl = 0x3;
2575         if (seg == VCPU_SREG_CS)
2576                 var.type = 0x3;
2577
2578         if (!emulate_invalid_guest_state) {
2579                 var.selector = var.base >> 4;
2580                 var.base = var.base & 0xffff0;
2581                 var.limit = 0xffff;
2582                 var.g = 0;
2583                 var.db = 0;
2584                 var.present = 1;
2585                 var.s = 1;
2586                 var.l = 0;
2587                 var.unusable = 0;
2588                 var.type = 0x3;
2589                 var.avl = 0;
2590                 if (save->base & 0xf)
2591                         printk_once(KERN_WARNING "kvm: segment base is not "
2592                                         "paragraph aligned when entering "
2593                                         "protected mode (seg=%d)", seg);
2594         }
2595
2596         vmcs_write16(sf->selector, var.selector);
2597         vmcs_writel(sf->base, var.base);
2598         vmcs_write32(sf->limit, var.limit);
2599         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
2600 }
2601
2602 static void enter_rmode(struct kvm_vcpu *vcpu)
2603 {
2604         unsigned long flags;
2605         struct vcpu_vmx *vmx = to_vmx(vcpu);
2606         struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm);
2607
2608         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2609         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2610         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2611         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2612         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
2613         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
2614         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
2615
2616         vmx->rmode.vm86_active = 1;
2617
2618         /*
2619          * Very old userspace does not call KVM_SET_TSS_ADDR before entering
2620          * vcpu. Warn the user that an update is overdue.
2621          */
2622         if (!kvm_vmx->tss_addr)
2623                 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
2624                              "called before entering vcpu\n");
2625
2626         vmx_segment_cache_clear(vmx);
2627
2628         vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr);
2629         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
2630         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2631
2632         flags = vmcs_readl(GUEST_RFLAGS);
2633         vmx->rmode.save_rflags = flags;
2634
2635         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
2636
2637         vmcs_writel(GUEST_RFLAGS, flags);
2638         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
2639         update_exception_bitmap(vcpu);
2640
2641         fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2642         fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2643         fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2644         fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2645         fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
2646         fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
2647
2648         kvm_mmu_reset_context(vcpu);
2649 }
2650
2651 void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
2652 {
2653         struct vcpu_vmx *vmx = to_vmx(vcpu);
2654         struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
2655
2656         if (!msr)
2657                 return;
2658
2659         vcpu->arch.efer = efer;
2660         if (efer & EFER_LMA) {
2661                 vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2662                 msr->data = efer;
2663         } else {
2664                 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2665
2666                 msr->data = efer & ~EFER_LME;
2667         }
2668         setup_msrs(vmx);
2669 }
2670
2671 #ifdef CONFIG_X86_64
2672
2673 static void enter_lmode(struct kvm_vcpu *vcpu)
2674 {
2675         u32 guest_tr_ar;
2676
2677         vmx_segment_cache_clear(to_vmx(vcpu));
2678
2679         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
2680         if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
2681                 pr_debug_ratelimited("%s: tss fixup for long mode. \n",
2682                                      __func__);
2683                 vmcs_write32(GUEST_TR_AR_BYTES,
2684                              (guest_tr_ar & ~VMX_AR_TYPE_MASK)
2685                              | VMX_AR_TYPE_BUSY_64_TSS);
2686         }
2687         vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
2688 }
2689
2690 static void exit_lmode(struct kvm_vcpu *vcpu)
2691 {
2692         vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2693         vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
2694 }
2695
2696 #endif
2697
2698 static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
2699 {
2700         int vpid = to_vmx(vcpu)->vpid;
2701
2702         if (!vpid_sync_vcpu_addr(vpid, addr))
2703                 vpid_sync_context(vpid);
2704
2705         /*
2706          * If VPIDs are not supported or enabled, then the above is a no-op.
2707          * But we don't really need a TLB flush in that case anyway, because
2708          * each VM entry/exit includes an implicit flush when VPID is 0.
2709          */
2710 }
2711
2712 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2713 {
2714         ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
2715
2716         vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
2717         vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
2718 }
2719
2720 static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
2721 {
2722         if (enable_unrestricted_guest || (enable_ept && is_paging(vcpu)))
2723                 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
2724         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
2725 }
2726
2727 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
2728 {
2729         ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
2730
2731         vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
2732         vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
2733 }
2734
2735 static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
2736 {
2737         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
2738
2739         if (!test_bit(VCPU_EXREG_PDPTR,
2740                       (unsigned long *)&vcpu->arch.regs_dirty))
2741                 return;
2742
2743         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
2744                 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
2745                 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
2746                 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
2747                 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
2748         }
2749 }
2750
2751 void ept_save_pdptrs(struct kvm_vcpu *vcpu)
2752 {
2753         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
2754
2755         if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
2756                 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
2757                 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
2758                 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
2759                 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
2760         }
2761
2762         __set_bit(VCPU_EXREG_PDPTR,
2763                   (unsigned long *)&vcpu->arch.regs_avail);
2764         __set_bit(VCPU_EXREG_PDPTR,
2765                   (unsigned long *)&vcpu->arch.regs_dirty);
2766 }
2767
2768 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
2769                                         unsigned long cr0,
2770                                         struct kvm_vcpu *vcpu)
2771 {
2772         if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
2773                 vmx_decache_cr3(vcpu);
2774         if (!(cr0 & X86_CR0_PG)) {
2775                 /* From paging/starting to nonpaging */
2776                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
2777                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
2778                              (CPU_BASED_CR3_LOAD_EXITING |
2779                               CPU_BASED_CR3_STORE_EXITING));
2780                 vcpu->arch.cr0 = cr0;
2781                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
2782         } else if (!is_paging(vcpu)) {
2783                 /* From nonpaging to paging */
2784                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
2785                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
2786                              ~(CPU_BASED_CR3_LOAD_EXITING |
2787                                CPU_BASED_CR3_STORE_EXITING));
2788                 vcpu->arch.cr0 = cr0;
2789                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
2790         }
2791
2792         if (!(cr0 & X86_CR0_WP))
2793                 *hw_cr0 &= ~X86_CR0_WP;
2794 }
2795
2796 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
2797 {
2798         struct vcpu_vmx *vmx = to_vmx(vcpu);
2799         unsigned long hw_cr0;
2800
2801         hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF);
2802         if (enable_unrestricted_guest)
2803                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
2804         else {
2805                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
2806
2807                 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
2808                         enter_pmode(vcpu);
2809
2810                 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
2811                         enter_rmode(vcpu);
2812         }
2813
2814 #ifdef CONFIG_X86_64
2815         if (vcpu->arch.efer & EFER_LME) {
2816                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
2817                         enter_lmode(vcpu);
2818                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
2819                         exit_lmode(vcpu);
2820         }
2821 #endif
2822
2823         if (enable_ept && !enable_unrestricted_guest)
2824                 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
2825
2826         vmcs_writel(CR0_READ_SHADOW, cr0);
2827         vmcs_writel(GUEST_CR0, hw_cr0);
2828         vcpu->arch.cr0 = cr0;
2829
2830         /* depends on vcpu->arch.cr0 to be set to a new value */
2831         vmx->emulation_required = emulation_required(vcpu);
2832 }
2833
2834 static int get_ept_level(struct kvm_vcpu *vcpu)
2835 {
2836         if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48))
2837                 return 5;
2838         return 4;
2839 }
2840
2841 u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa)
2842 {
2843         u64 eptp = VMX_EPTP_MT_WB;
2844
2845         eptp |= (get_ept_level(vcpu) == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
2846
2847         if (enable_ept_ad_bits &&
2848             (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
2849                 eptp |= VMX_EPTP_AD_ENABLE_BIT;
2850         eptp |= (root_hpa & PAGE_MASK);
2851
2852         return eptp;
2853 }
2854
2855 void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
2856 {
2857         struct kvm *kvm = vcpu->kvm;
2858         unsigned long guest_cr3;
2859         u64 eptp;
2860
2861         guest_cr3 = cr3;
2862         if (enable_ept) {
2863                 eptp = construct_eptp(vcpu, cr3);
2864                 vmcs_write64(EPT_POINTER, eptp);
2865
2866                 if (kvm_x86_ops->tlb_remote_flush) {
2867                         spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
2868                         to_vmx(vcpu)->ept_pointer = eptp;
2869                         to_kvm_vmx(kvm)->ept_pointers_match
2870                                 = EPT_POINTERS_CHECK;
2871                         spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
2872                 }
2873
2874                 if (enable_unrestricted_guest || is_paging(vcpu) ||
2875                     is_guest_mode(vcpu))
2876                         guest_cr3 = kvm_read_cr3(vcpu);
2877                 else
2878                         guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr;
2879                 ept_load_pdptrs(vcpu);
2880         }
2881
2882         vmcs_writel(GUEST_CR3, guest_cr3);
2883 }
2884
2885 int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
2886 {
2887         /*
2888          * Pass through host's Machine Check Enable value to hw_cr4, which
2889          * is in force while we are in guest mode.  Do not let guests control
2890          * this bit, even if host CR4.MCE == 0.
2891          */
2892         unsigned long hw_cr4;
2893
2894         hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE);
2895         if (enable_unrestricted_guest)
2896                 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST;
2897         else if (to_vmx(vcpu)->rmode.vm86_active)
2898                 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON;
2899         else
2900                 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON;
2901
2902         if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) {
2903                 if (cr4 & X86_CR4_UMIP) {
2904                         vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
2905                                 SECONDARY_EXEC_DESC);
2906                         hw_cr4 &= ~X86_CR4_UMIP;
2907                 } else if (!is_guest_mode(vcpu) ||
2908                         !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC))
2909                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
2910                                         SECONDARY_EXEC_DESC);
2911         }
2912
2913         if (cr4 & X86_CR4_VMXE) {
2914                 /*
2915                  * To use VMXON (and later other VMX instructions), a guest
2916                  * must first be able to turn on cr4.VMXE (see handle_vmon()).
2917                  * So basically the check on whether to allow nested VMX
2918                  * is here.  We operate under the default treatment of SMM,
2919                  * so VMX cannot be enabled under SMM.
2920                  */
2921                 if (!nested_vmx_allowed(vcpu) || is_smm(vcpu))
2922                         return 1;
2923         }
2924
2925         if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
2926                 return 1;
2927
2928         vcpu->arch.cr4 = cr4;
2929
2930         if (!enable_unrestricted_guest) {
2931                 if (enable_ept) {
2932                         if (!is_paging(vcpu)) {
2933                                 hw_cr4 &= ~X86_CR4_PAE;
2934                                 hw_cr4 |= X86_CR4_PSE;
2935                         } else if (!(cr4 & X86_CR4_PAE)) {
2936                                 hw_cr4 &= ~X86_CR4_PAE;
2937                         }
2938                 }
2939
2940                 /*
2941                  * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
2942                  * hardware.  To emulate this behavior, SMEP/SMAP/PKU needs
2943                  * to be manually disabled when guest switches to non-paging
2944                  * mode.
2945                  *
2946                  * If !enable_unrestricted_guest, the CPU is always running
2947                  * with CR0.PG=1 and CR4 needs to be modified.
2948                  * If enable_unrestricted_guest, the CPU automatically
2949                  * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
2950                  */
2951                 if (!is_paging(vcpu))
2952                         hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
2953         }
2954
2955         vmcs_writel(CR4_READ_SHADOW, cr4);
2956         vmcs_writel(GUEST_CR4, hw_cr4);
2957         return 0;
2958 }
2959
2960 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
2961 {
2962         struct vcpu_vmx *vmx = to_vmx(vcpu);
2963         u32 ar;
2964
2965         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
2966                 *var = vmx->rmode.segs[seg];
2967                 if (seg == VCPU_SREG_TR
2968                     || var->selector == vmx_read_guest_seg_selector(vmx, seg))
2969                         return;
2970                 var->base = vmx_read_guest_seg_base(vmx, seg);
2971                 var->selector = vmx_read_guest_seg_selector(vmx, seg);
2972                 return;
2973         }
2974         var->base = vmx_read_guest_seg_base(vmx, seg);
2975         var->limit = vmx_read_guest_seg_limit(vmx, seg);
2976         var->selector = vmx_read_guest_seg_selector(vmx, seg);
2977         ar = vmx_read_guest_seg_ar(vmx, seg);
2978         var->unusable = (ar >> 16) & 1;
2979         var->type = ar & 15;
2980         var->s = (ar >> 4) & 1;
2981         var->dpl = (ar >> 5) & 3;
2982         /*
2983          * Some userspaces do not preserve unusable property. Since usable
2984          * segment has to be present according to VMX spec we can use present
2985          * property to amend userspace bug by making unusable segment always
2986          * nonpresent. vmx_segment_access_rights() already marks nonpresent
2987          * segment as unusable.
2988          */
2989         var->present = !var->unusable;
2990         var->avl = (ar >> 12) & 1;
2991         var->l = (ar >> 13) & 1;
2992         var->db = (ar >> 14) & 1;
2993         var->g = (ar >> 15) & 1;
2994 }
2995
2996 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
2997 {
2998         struct kvm_segment s;
2999
3000         if (to_vmx(vcpu)->rmode.vm86_active) {
3001                 vmx_get_segment(vcpu, &s, seg);
3002                 return s.base;
3003         }
3004         return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
3005 }
3006
3007 int vmx_get_cpl(struct kvm_vcpu *vcpu)
3008 {
3009         struct vcpu_vmx *vmx = to_vmx(vcpu);
3010
3011         if (unlikely(vmx->rmode.vm86_active))
3012                 return 0;
3013         else {
3014                 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
3015                 return VMX_AR_DPL(ar);
3016         }
3017 }
3018
3019 static u32 vmx_segment_access_rights(struct kvm_segment *var)
3020 {
3021         u32 ar;
3022
3023         if (var->unusable || !var->present)
3024                 ar = 1 << 16;
3025         else {
3026                 ar = var->type & 15;
3027                 ar |= (var->s & 1) << 4;
3028                 ar |= (var->dpl & 3) << 5;
3029                 ar |= (var->present & 1) << 7;
3030                 ar |= (var->avl & 1) << 12;
3031                 ar |= (var->l & 1) << 13;
3032                 ar |= (var->db & 1) << 14;
3033                 ar |= (var->g & 1) << 15;
3034         }
3035
3036         return ar;
3037 }
3038
3039 void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
3040 {
3041         struct vcpu_vmx *vmx = to_vmx(vcpu);
3042         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3043
3044         vmx_segment_cache_clear(vmx);
3045
3046         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3047                 vmx->rmode.segs[seg] = *var;
3048                 if (seg == VCPU_SREG_TR)
3049                         vmcs_write16(sf->selector, var->selector);
3050                 else if (var->s)
3051                         fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
3052                 goto out;
3053         }
3054
3055         vmcs_writel(sf->base, var->base);
3056         vmcs_write32(sf->limit, var->limit);
3057         vmcs_write16(sf->selector, var->selector);
3058
3059         /*
3060          *   Fix the "Accessed" bit in AR field of segment registers for older
3061          * qemu binaries.
3062          *   IA32 arch specifies that at the time of processor reset the
3063          * "Accessed" bit in the AR field of segment registers is 1. And qemu
3064          * is setting it to 0 in the userland code. This causes invalid guest
3065          * state vmexit when "unrestricted guest" mode is turned on.
3066          *    Fix for this setup issue in cpu_reset is being pushed in the qemu
3067          * tree. Newer qemu binaries with that qemu fix would not need this
3068          * kvm hack.
3069          */
3070         if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
3071                 var->type |= 0x1; /* Accessed */
3072
3073         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
3074
3075 out:
3076         vmx->emulation_required = emulation_required(vcpu);
3077 }
3078
3079 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
3080 {
3081         u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
3082
3083         *db = (ar >> 14) & 1;
3084         *l = (ar >> 13) & 1;
3085 }
3086
3087 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3088 {
3089         dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
3090         dt->address = vmcs_readl(GUEST_IDTR_BASE);
3091 }
3092
3093 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3094 {
3095         vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
3096         vmcs_writel(GUEST_IDTR_BASE, dt->address);
3097 }
3098
3099 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3100 {
3101         dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
3102         dt->address = vmcs_readl(GUEST_GDTR_BASE);
3103 }
3104
3105 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3106 {
3107         vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
3108         vmcs_writel(GUEST_GDTR_BASE, dt->address);
3109 }
3110
3111 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
3112 {
3113         struct kvm_segment var;
3114         u32 ar;
3115
3116         vmx_get_segment(vcpu, &var, seg);
3117         var.dpl = 0x3;
3118         if (seg == VCPU_SREG_CS)
3119                 var.type = 0x3;
3120         ar = vmx_segment_access_rights(&var);
3121
3122         if (var.base != (var.selector << 4))
3123                 return false;
3124         if (var.limit != 0xffff)
3125                 return false;
3126         if (ar != 0xf3)
3127                 return false;
3128
3129         return true;
3130 }
3131
3132 static bool code_segment_valid(struct kvm_vcpu *vcpu)
3133 {
3134         struct kvm_segment cs;
3135         unsigned int cs_rpl;
3136
3137         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3138         cs_rpl = cs.selector & SEGMENT_RPL_MASK;
3139
3140         if (cs.unusable)
3141                 return false;
3142         if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
3143                 return false;
3144         if (!cs.s)
3145                 return false;
3146         if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
3147                 if (cs.dpl > cs_rpl)
3148                         return false;
3149         } else {
3150                 if (cs.dpl != cs_rpl)
3151                         return false;
3152         }
3153         if (!cs.present)
3154                 return false;
3155
3156         /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
3157         return true;
3158 }
3159
3160 static bool stack_segment_valid(struct kvm_vcpu *vcpu)
3161 {
3162         struct kvm_segment ss;
3163         unsigned int ss_rpl;
3164
3165         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3166         ss_rpl = ss.selector & SEGMENT_RPL_MASK;
3167
3168         if (ss.unusable)
3169                 return true;
3170         if (ss.type != 3 && ss.type != 7)
3171                 return false;
3172         if (!ss.s)
3173                 return false;
3174         if (ss.dpl != ss_rpl) /* DPL != RPL */
3175                 return false;
3176         if (!ss.present)
3177                 return false;
3178
3179         return true;
3180 }
3181
3182 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
3183 {
3184         struct kvm_segment var;
3185         unsigned int rpl;
3186
3187         vmx_get_segment(vcpu, &var, seg);
3188         rpl = var.selector & SEGMENT_RPL_MASK;
3189
3190         if (var.unusable)
3191                 return true;
3192         if (!var.s)
3193                 return false;
3194         if (!var.present)
3195                 return false;
3196         if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
3197                 if (var.dpl < rpl) /* DPL < RPL */
3198                         return false;
3199         }
3200
3201         /* TODO: Add other members to kvm_segment_field to allow checking for other access
3202          * rights flags
3203          */
3204         return true;
3205 }
3206
3207 static bool tr_valid(struct kvm_vcpu *vcpu)
3208 {
3209         struct kvm_segment tr;
3210
3211         vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
3212
3213         if (tr.unusable)
3214                 return false;
3215         if (tr.selector & SEGMENT_TI_MASK)      /* TI = 1 */
3216                 return false;
3217         if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
3218                 return false;
3219         if (!tr.present)
3220                 return false;
3221
3222         return true;
3223 }
3224
3225 static bool ldtr_valid(struct kvm_vcpu *vcpu)
3226 {
3227         struct kvm_segment ldtr;
3228
3229         vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
3230
3231         if (ldtr.unusable)
3232                 return true;
3233         if (ldtr.selector & SEGMENT_TI_MASK)    /* TI = 1 */
3234                 return false;
3235         if (ldtr.type != 2)
3236                 return false;
3237         if (!ldtr.present)
3238                 return false;
3239
3240         return true;
3241 }
3242
3243 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
3244 {
3245         struct kvm_segment cs, ss;
3246
3247         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3248         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3249
3250         return ((cs.selector & SEGMENT_RPL_MASK) ==
3251                  (ss.selector & SEGMENT_RPL_MASK));
3252 }
3253
3254 /*
3255  * Check if guest state is valid. Returns true if valid, false if
3256  * not.
3257  * We assume that registers are always usable
3258  */
3259 static bool guest_state_valid(struct kvm_vcpu *vcpu)
3260 {
3261         if (enable_unrestricted_guest)
3262                 return true;
3263
3264         /* real mode guest state checks */
3265         if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
3266                 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
3267                         return false;
3268                 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
3269                         return false;
3270                 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
3271                         return false;
3272                 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
3273                         return false;
3274                 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
3275                         return false;
3276                 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
3277                         return false;
3278         } else {
3279         /* protected mode guest state checks */
3280                 if (!cs_ss_rpl_check(vcpu))
3281                         return false;
3282                 if (!code_segment_valid(vcpu))
3283                         return false;
3284                 if (!stack_segment_valid(vcpu))
3285                         return false;
3286                 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
3287                         return false;
3288                 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
3289                         return false;
3290                 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
3291                         return false;
3292                 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
3293                         return false;
3294                 if (!tr_valid(vcpu))
3295                         return false;
3296                 if (!ldtr_valid(vcpu))
3297                         return false;
3298         }
3299         /* TODO:
3300          * - Add checks on RIP
3301          * - Add checks on RFLAGS
3302          */
3303
3304         return true;
3305 }
3306
3307 static int init_rmode_tss(struct kvm *kvm)
3308 {
3309         gfn_t fn;
3310         u16 data = 0;
3311         int idx, r;
3312
3313         idx = srcu_read_lock(&kvm->srcu);
3314         fn = to_kvm_vmx(kvm)->tss_addr >> PAGE_SHIFT;
3315         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
3316         if (r < 0)
3317                 goto out;
3318         data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
3319         r = kvm_write_guest_page(kvm, fn++, &data,
3320                         TSS_IOPB_BASE_OFFSET, sizeof(u16));
3321         if (r < 0)
3322                 goto out;
3323         r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
3324         if (r < 0)
3325                 goto out;
3326         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
3327         if (r < 0)
3328                 goto out;
3329         data = ~0;
3330         r = kvm_write_guest_page(kvm, fn, &data,
3331                                  RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
3332                                  sizeof(u8));
3333 out:
3334         srcu_read_unlock(&kvm->srcu, idx);
3335         return r;
3336 }
3337
3338 static int init_rmode_identity_map(struct kvm *kvm)
3339 {
3340         struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
3341         int i, idx, r = 0;
3342         kvm_pfn_t identity_map_pfn;
3343         u32 tmp;
3344
3345         /* Protect kvm_vmx->ept_identity_pagetable_done. */
3346         mutex_lock(&kvm->slots_lock);
3347
3348         if (likely(kvm_vmx->ept_identity_pagetable_done))
3349                 goto out2;
3350
3351         if (!kvm_vmx->ept_identity_map_addr)
3352                 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR;
3353         identity_map_pfn = kvm_vmx->ept_identity_map_addr >> PAGE_SHIFT;
3354
3355         r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
3356                                     kvm_vmx->ept_identity_map_addr, PAGE_SIZE);
3357         if (r < 0)
3358                 goto out2;
3359
3360         idx = srcu_read_lock(&kvm->srcu);
3361         r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
3362         if (r < 0)
3363                 goto out;
3364         /* Set up identity-mapping pagetable for EPT in real mode */
3365         for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
3366                 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
3367                         _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
3368                 r = kvm_write_guest_page(kvm, identity_map_pfn,
3369                                 &tmp, i * sizeof(tmp), sizeof(tmp));
3370                 if (r < 0)
3371                         goto out;
3372         }
3373         kvm_vmx->ept_identity_pagetable_done = true;
3374
3375 out:
3376         srcu_read_unlock(&kvm->srcu, idx);
3377
3378 out2:
3379         mutex_unlock(&kvm->slots_lock);
3380         return r;
3381 }
3382
3383 static void seg_setup(int seg)
3384 {
3385         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3386         unsigned int ar;
3387
3388         vmcs_write16(sf->selector, 0);
3389         vmcs_writel(sf->base, 0);
3390         vmcs_write32(sf->limit, 0xffff);
3391         ar = 0x93;
3392         if (seg == VCPU_SREG_CS)
3393                 ar |= 0x08; /* code segment */
3394
3395         vmcs_write32(sf->ar_bytes, ar);
3396 }
3397
3398 static int alloc_apic_access_page(struct kvm *kvm)
3399 {
3400         struct page *page;
3401         int r = 0;
3402
3403         mutex_lock(&kvm->slots_lock);
3404         if (kvm->arch.apic_access_page_done)
3405                 goto out;
3406         r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
3407                                     APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
3408         if (r)
3409                 goto out;
3410
3411         page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
3412         if (is_error_page(page)) {
3413                 r = -EFAULT;
3414                 goto out;
3415         }
3416
3417         /*
3418          * Do not pin the page in memory, so that memory hot-unplug
3419          * is able to migrate it.
3420          */
3421         put_page(page);
3422         kvm->arch.apic_access_page_done = true;
3423 out:
3424         mutex_unlock(&kvm->slots_lock);
3425         return r;
3426 }
3427
3428 int allocate_vpid(void)
3429 {
3430         int vpid;
3431
3432         if (!enable_vpid)
3433                 return 0;
3434         spin_lock(&vmx_vpid_lock);
3435         vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
3436         if (vpid < VMX_NR_VPIDS)
3437                 __set_bit(vpid, vmx_vpid_bitmap);
3438         else
3439                 vpid = 0;
3440         spin_unlock(&vmx_vpid_lock);
3441         return vpid;
3442 }
3443
3444 void free_vpid(int vpid)
3445 {
3446         if (!enable_vpid || vpid == 0)
3447                 return;
3448         spin_lock(&vmx_vpid_lock);
3449         __clear_bit(vpid, vmx_vpid_bitmap);
3450         spin_unlock(&vmx_vpid_lock);
3451 }
3452
3453 static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
3454                                                           u32 msr, int type)
3455 {
3456         int f = sizeof(unsigned long);
3457
3458         if (!cpu_has_vmx_msr_bitmap())
3459                 return;
3460
3461         if (static_branch_unlikely(&enable_evmcs))
3462                 evmcs_touch_msr_bitmap();
3463
3464         /*
3465          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
3466          * have the write-low and read-high bitmap offsets the wrong way round.
3467          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
3468          */
3469         if (msr <= 0x1fff) {
3470                 if (type & MSR_TYPE_R)
3471                         /* read-low */
3472                         __clear_bit(msr, msr_bitmap + 0x000 / f);
3473
3474                 if (type & MSR_TYPE_W)
3475                         /* write-low */
3476                         __clear_bit(msr, msr_bitmap + 0x800 / f);
3477
3478         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
3479                 msr &= 0x1fff;
3480                 if (type & MSR_TYPE_R)
3481                         /* read-high */
3482                         __clear_bit(msr, msr_bitmap + 0x400 / f);
3483
3484                 if (type & MSR_TYPE_W)
3485                         /* write-high */
3486                         __clear_bit(msr, msr_bitmap + 0xc00 / f);
3487
3488         }
3489 }
3490
3491 static __always_inline void vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
3492                                                          u32 msr, int type)
3493 {
3494         int f = sizeof(unsigned long);
3495
3496         if (!cpu_has_vmx_msr_bitmap())
3497                 return;
3498
3499         if (static_branch_unlikely(&enable_evmcs))
3500                 evmcs_touch_msr_bitmap();
3501
3502         /*
3503          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
3504          * have the write-low and read-high bitmap offsets the wrong way round.
3505          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
3506          */
3507         if (msr <= 0x1fff) {
3508                 if (type & MSR_TYPE_R)
3509                         /* read-low */
3510                         __set_bit(msr, msr_bitmap + 0x000 / f);
3511
3512                 if (type & MSR_TYPE_W)
3513                         /* write-low */
3514                         __set_bit(msr, msr_bitmap + 0x800 / f);
3515
3516         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
3517                 msr &= 0x1fff;
3518                 if (type & MSR_TYPE_R)
3519                         /* read-high */
3520                         __set_bit(msr, msr_bitmap + 0x400 / f);
3521
3522                 if (type & MSR_TYPE_W)
3523                         /* write-high */
3524                         __set_bit(msr, msr_bitmap + 0xc00 / f);
3525
3526         }
3527 }
3528
3529 static __always_inline void vmx_set_intercept_for_msr(unsigned long *msr_bitmap,
3530                                                       u32 msr, int type, bool value)
3531 {
3532         if (value)
3533                 vmx_enable_intercept_for_msr(msr_bitmap, msr, type);
3534         else
3535                 vmx_disable_intercept_for_msr(msr_bitmap, msr, type);
3536 }
3537
3538 static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu)
3539 {
3540         u8 mode = 0;
3541
3542         if (cpu_has_secondary_exec_ctrls() &&
3543             (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) &
3544              SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
3545                 mode |= MSR_BITMAP_MODE_X2APIC;
3546                 if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
3547                         mode |= MSR_BITMAP_MODE_X2APIC_APICV;
3548         }
3549
3550         return mode;
3551 }
3552
3553 static void vmx_update_msr_bitmap_x2apic(unsigned long *msr_bitmap,
3554                                          u8 mode)
3555 {
3556         int msr;
3557
3558         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
3559                 unsigned word = msr / BITS_PER_LONG;
3560                 msr_bitmap[word] = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0;
3561                 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
3562         }
3563
3564         if (mode & MSR_BITMAP_MODE_X2APIC) {
3565                 /*
3566                  * TPR reads and writes can be virtualized even if virtual interrupt
3567                  * delivery is not in use.
3568                  */
3569                 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW);
3570                 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) {
3571                         vmx_enable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_R);
3572                         vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_EOI), MSR_TYPE_W);
3573                         vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W);
3574                 }
3575         }
3576 }
3577
3578 void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu)
3579 {
3580         struct vcpu_vmx *vmx = to_vmx(vcpu);
3581         unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
3582         u8 mode = vmx_msr_bitmap_mode(vcpu);
3583         u8 changed = mode ^ vmx->msr_bitmap_mode;
3584
3585         if (!changed)
3586                 return;
3587
3588         if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV))
3589                 vmx_update_msr_bitmap_x2apic(msr_bitmap, mode);
3590
3591         vmx->msr_bitmap_mode = mode;
3592 }
3593
3594 void pt_update_intercept_for_msr(struct vcpu_vmx *vmx)
3595 {
3596         unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
3597         bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN);
3598         u32 i;
3599
3600         vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_STATUS,
3601                                                         MSR_TYPE_RW, flag);
3602         vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_OUTPUT_BASE,
3603                                                         MSR_TYPE_RW, flag);
3604         vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_OUTPUT_MASK,
3605                                                         MSR_TYPE_RW, flag);
3606         vmx_set_intercept_for_msr(msr_bitmap, MSR_IA32_RTIT_CR3_MATCH,
3607                                                         MSR_TYPE_RW, flag);
3608         for (i = 0; i < vmx->pt_desc.addr_range; i++) {
3609                 vmx_set_intercept_for_msr(msr_bitmap,
3610                         MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag);
3611                 vmx_set_intercept_for_msr(msr_bitmap,
3612                         MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag);
3613         }
3614 }
3615
3616 static bool vmx_get_enable_apicv(struct kvm_vcpu *vcpu)
3617 {
3618         return enable_apicv;
3619 }
3620
3621 static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
3622 {
3623         struct vcpu_vmx *vmx = to_vmx(vcpu);
3624         void *vapic_page;
3625         u32 vppr;
3626         int rvi;
3627
3628         if (WARN_ON_ONCE(!is_guest_mode(vcpu)) ||
3629                 !nested_cpu_has_vid(get_vmcs12(vcpu)) ||
3630                 WARN_ON_ONCE(!vmx->nested.virtual_apic_page))
3631                 return false;
3632
3633         rvi = vmx_get_rvi();
3634
3635         vapic_page = kmap(vmx->nested.virtual_apic_page);
3636         vppr = *((u32 *)(vapic_page + APIC_PROCPRI));
3637         kunmap(vmx->nested.virtual_apic_page);
3638
3639         return ((rvi & 0xf0) > (vppr & 0xf0));
3640 }
3641
3642 static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
3643                                                      bool nested)
3644 {
3645 #ifdef CONFIG_SMP
3646         int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR;
3647
3648         if (vcpu->mode == IN_GUEST_MODE) {
3649                 /*
3650                  * The vector of interrupt to be delivered to vcpu had
3651                  * been set in PIR before this function.
3652                  *
3653                  * Following cases will be reached in this block, and
3654                  * we always send a notification event in all cases as
3655                  * explained below.
3656                  *
3657                  * Case 1: vcpu keeps in non-root mode. Sending a
3658                  * notification event posts the interrupt to vcpu.
3659                  *
3660                  * Case 2: vcpu exits to root mode and is still
3661                  * runnable. PIR will be synced to vIRR before the
3662                  * next vcpu entry. Sending a notification event in
3663                  * this case has no effect, as vcpu is not in root
3664                  * mode.
3665                  *
3666                  * Case 3: vcpu exits to root mode and is blocked.
3667                  * vcpu_block() has already synced PIR to vIRR and
3668                  * never blocks vcpu if vIRR is not cleared. Therefore,
3669                  * a blocked vcpu here does not wait for any requested
3670                  * interrupts in PIR, and sending a notification event
3671                  * which has no effect is safe here.
3672                  */
3673
3674                 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
3675                 return true;
3676         }
3677 #endif
3678         return false;
3679 }
3680
3681 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
3682                                                 int vector)
3683 {
3684         struct vcpu_vmx *vmx = to_vmx(vcpu);
3685
3686         if (is_guest_mode(vcpu) &&
3687             vector == vmx->nested.posted_intr_nv) {
3688                 /*
3689                  * If a posted intr is not recognized by hardware,
3690                  * we will accomplish it in the next vmentry.
3691                  */
3692                 vmx->nested.pi_pending = true;
3693                 kvm_make_request(KVM_REQ_EVENT, vcpu);
3694                 /* the PIR and ON have been set by L1. */
3695                 if (!kvm_vcpu_trigger_posted_interrupt(vcpu, true))
3696                         kvm_vcpu_kick(vcpu);
3697                 return 0;
3698         }
3699         return -1;
3700 }
3701 /*
3702  * Send interrupt to vcpu via posted interrupt way.
3703  * 1. If target vcpu is running(non-root mode), send posted interrupt
3704  * notification to vcpu and hardware will sync PIR to vIRR atomically.
3705  * 2. If target vcpu isn't running(root mode), kick it to pick up the
3706  * interrupt from PIR in next vmentry.
3707  */
3708 static void vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
3709 {
3710         struct vcpu_vmx *vmx = to_vmx(vcpu);
3711         int r;
3712
3713         r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
3714         if (!r)
3715                 return;
3716
3717         if (pi_test_and_set_pir(vector, &vmx->pi_desc))
3718                 return;
3719
3720         /* If a previous notification has sent the IPI, nothing to do.  */
3721         if (pi_test_and_set_on(&vmx->pi_desc))
3722                 return;
3723
3724         if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false))
3725                 kvm_vcpu_kick(vcpu);
3726 }
3727
3728 /*
3729  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
3730  * will not change in the lifetime of the guest.
3731  * Note that host-state that does change is set elsewhere. E.g., host-state
3732  * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
3733  */
3734 void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
3735 {
3736         u32 low32, high32;
3737         unsigned long tmpl;
3738         struct desc_ptr dt;
3739         unsigned long cr0, cr3, cr4;
3740
3741         cr0 = read_cr0();
3742         WARN_ON(cr0 & X86_CR0_TS);
3743         vmcs_writel(HOST_CR0, cr0);  /* 22.2.3 */
3744
3745         /*
3746          * Save the most likely value for this task's CR3 in the VMCS.
3747          * We can't use __get_current_cr3_fast() because we're not atomic.
3748          */
3749         cr3 = __read_cr3();
3750         vmcs_writel(HOST_CR3, cr3);             /* 22.2.3  FIXME: shadow tables */
3751         vmx->loaded_vmcs->host_state.cr3 = cr3;
3752
3753         /* Save the most likely value for this task's CR4 in the VMCS. */
3754         cr4 = cr4_read_shadow();
3755         vmcs_writel(HOST_CR4, cr4);                     /* 22.2.3, 22.2.5 */
3756         vmx->loaded_vmcs->host_state.cr4 = cr4;
3757
3758         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
3759 #ifdef CONFIG_X86_64
3760         /*
3761          * Load null selectors, so we can avoid reloading them in
3762          * vmx_prepare_switch_to_host(), in case userspace uses
3763          * the null selectors too (the expected case).
3764          */
3765         vmcs_write16(HOST_DS_SELECTOR, 0);
3766         vmcs_write16(HOST_ES_SELECTOR, 0);
3767 #else
3768         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
3769         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
3770 #endif
3771         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
3772         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
3773
3774         store_idt(&dt);
3775         vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
3776         vmx->host_idt_base = dt.address;
3777
3778         vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */
3779
3780         rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
3781         vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
3782         rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
3783         vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */
3784
3785         if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
3786                 rdmsr(MSR_IA32_CR_PAT, low32, high32);
3787                 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
3788         }
3789
3790         if (cpu_has_load_ia32_efer())
3791                 vmcs_write64(HOST_IA32_EFER, host_efer);
3792 }
3793
3794 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
3795 {
3796         vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
3797         if (enable_ept)
3798                 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
3799         if (is_guest_mode(&vmx->vcpu))
3800                 vmx->vcpu.arch.cr4_guest_owned_bits &=
3801                         ~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
3802         vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
3803 }
3804
3805 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
3806 {
3807         u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
3808
3809         if (!kvm_vcpu_apicv_active(&vmx->vcpu))
3810                 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
3811
3812         if (!enable_vnmi)
3813                 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS;
3814
3815         /* Enable the preemption timer dynamically */
3816         pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
3817         return pin_based_exec_ctrl;
3818 }
3819
3820 static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
3821 {
3822         struct vcpu_vmx *vmx = to_vmx(vcpu);
3823
3824         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
3825         if (cpu_has_secondary_exec_ctrls()) {
3826                 if (kvm_vcpu_apicv_active(vcpu))
3827                         vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
3828                                       SECONDARY_EXEC_APIC_REGISTER_VIRT |
3829                                       SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
3830                 else
3831                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
3832                                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
3833                                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
3834         }
3835
3836         if (cpu_has_vmx_msr_bitmap())
3837                 vmx_update_msr_bitmap(vcpu);
3838 }
3839
3840 u32 vmx_exec_control(struct vcpu_vmx *vmx)
3841 {
3842         u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
3843
3844         if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
3845                 exec_control &= ~CPU_BASED_MOV_DR_EXITING;
3846
3847         if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
3848                 exec_control &= ~CPU_BASED_TPR_SHADOW;
3849 #ifdef CONFIG_X86_64
3850                 exec_control |= CPU_BASED_CR8_STORE_EXITING |
3851                                 CPU_BASED_CR8_LOAD_EXITING;
3852 #endif
3853         }
3854         if (!enable_ept)
3855                 exec_control |= CPU_BASED_CR3_STORE_EXITING |
3856                                 CPU_BASED_CR3_LOAD_EXITING  |
3857                                 CPU_BASED_INVLPG_EXITING;
3858         if (kvm_mwait_in_guest(vmx->vcpu.kvm))
3859                 exec_control &= ~(CPU_BASED_MWAIT_EXITING |
3860                                 CPU_BASED_MONITOR_EXITING);
3861         if (kvm_hlt_in_guest(vmx->vcpu.kvm))
3862                 exec_control &= ~CPU_BASED_HLT_EXITING;
3863         return exec_control;
3864 }
3865
3866
3867 static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
3868 {
3869         struct kvm_vcpu *vcpu = &vmx->vcpu;
3870
3871         u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
3872
3873         if (pt_mode == PT_MODE_SYSTEM)
3874                 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX);
3875         if (!cpu_need_virtualize_apic_accesses(vcpu))
3876                 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
3877         if (vmx->vpid == 0)
3878                 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
3879         if (!enable_ept) {
3880                 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
3881                 enable_unrestricted_guest = 0;
3882         }
3883         if (!enable_unrestricted_guest)
3884                 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
3885         if (kvm_pause_in_guest(vmx->vcpu.kvm))
3886                 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
3887         if (!kvm_vcpu_apicv_active(vcpu))
3888                 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
3889                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
3890         exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
3891
3892         /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP,
3893          * in vmx_set_cr4.  */
3894         exec_control &= ~SECONDARY_EXEC_DESC;
3895
3896         /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
3897            (handle_vmptrld).
3898            We can NOT enable shadow_vmcs here because we don't have yet
3899            a current VMCS12
3900         */
3901         exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
3902
3903         if (!enable_pml)
3904                 exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
3905
3906         if (vmx_xsaves_supported()) {
3907                 /* Exposing XSAVES only when XSAVE is exposed */
3908                 bool xsaves_enabled =
3909                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
3910                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVES);
3911
3912                 if (!xsaves_enabled)
3913                         exec_control &= ~SECONDARY_EXEC_XSAVES;
3914
3915                 if (nested) {
3916                         if (xsaves_enabled)
3917                                 vmx->nested.msrs.secondary_ctls_high |=
3918                                         SECONDARY_EXEC_XSAVES;
3919                         else
3920                                 vmx->nested.msrs.secondary_ctls_high &=
3921                                         ~SECONDARY_EXEC_XSAVES;
3922                 }
3923         }
3924
3925         if (vmx_rdtscp_supported()) {
3926                 bool rdtscp_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP);
3927                 if (!rdtscp_enabled)
3928                         exec_control &= ~SECONDARY_EXEC_RDTSCP;
3929
3930                 if (nested) {
3931                         if (rdtscp_enabled)
3932                                 vmx->nested.msrs.secondary_ctls_high |=
3933                                         SECONDARY_EXEC_RDTSCP;
3934                         else
3935                                 vmx->nested.msrs.secondary_ctls_high &=
3936                                         ~SECONDARY_EXEC_RDTSCP;
3937                 }
3938         }
3939
3940         if (vmx_invpcid_supported()) {
3941                 /* Exposing INVPCID only when PCID is exposed */
3942                 bool invpcid_enabled =
3943                         guest_cpuid_has(vcpu, X86_FEATURE_INVPCID) &&
3944                         guest_cpuid_has(vcpu, X86_FEATURE_PCID);
3945
3946                 if (!invpcid_enabled) {
3947                         exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
3948                         guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID);
3949                 }
3950
3951                 if (nested) {
3952                         if (invpcid_enabled)
3953                                 vmx->nested.msrs.secondary_ctls_high |=
3954                                         SECONDARY_EXEC_ENABLE_INVPCID;
3955                         else
3956                                 vmx->nested.msrs.secondary_ctls_high &=
3957                                         ~SECONDARY_EXEC_ENABLE_INVPCID;
3958                 }
3959         }
3960
3961         if (vmx_rdrand_supported()) {
3962                 bool rdrand_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDRAND);
3963                 if (rdrand_enabled)
3964                         exec_control &= ~SECONDARY_EXEC_RDRAND_EXITING;
3965
3966                 if (nested) {
3967                         if (rdrand_enabled)
3968                                 vmx->nested.msrs.secondary_ctls_high |=
3969                                         SECONDARY_EXEC_RDRAND_EXITING;
3970                         else
3971                                 vmx->nested.msrs.secondary_ctls_high &=
3972                                         ~SECONDARY_EXEC_RDRAND_EXITING;
3973                 }
3974         }
3975
3976         if (vmx_rdseed_supported()) {
3977                 bool rdseed_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDSEED);
3978                 if (rdseed_enabled)
3979                         exec_control &= ~SECONDARY_EXEC_RDSEED_EXITING;
3980
3981                 if (nested) {
3982                         if (rdseed_enabled)
3983                                 vmx->nested.msrs.secondary_ctls_high |=
3984                                         SECONDARY_EXEC_RDSEED_EXITING;
3985                         else
3986                                 vmx->nested.msrs.secondary_ctls_high &=
3987                                         ~SECONDARY_EXEC_RDSEED_EXITING;
3988                 }
3989         }
3990
3991         vmx->secondary_exec_control = exec_control;
3992 }
3993
3994 static void ept_set_mmio_spte_mask(void)
3995 {
3996         /*
3997          * EPT Misconfigurations can be generated if the value of bits 2:0
3998          * of an EPT paging-structure entry is 110b (write/execute).
3999          */
4000         kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK,
4001                                    VMX_EPT_MISCONFIG_WX_VALUE);
4002 }
4003
4004 #define VMX_XSS_EXIT_BITMAP 0
4005
4006 /*
4007  * Sets up the vmcs for emulated real mode.
4008  */
4009 static void vmx_vcpu_setup(struct vcpu_vmx *vmx)
4010 {
4011         int i;
4012
4013         if (nested)
4014                 nested_vmx_vcpu_setup();
4015
4016         if (cpu_has_vmx_msr_bitmap())
4017                 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
4018
4019         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
4020
4021         /* Control */
4022         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
4023         vmx->hv_deadline_tsc = -1;
4024
4025         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
4026
4027         if (cpu_has_secondary_exec_ctrls()) {
4028                 vmx_compute_secondary_exec_control(vmx);
4029                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
4030                              vmx->secondary_exec_control);
4031         }
4032
4033         if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
4034                 vmcs_write64(EOI_EXIT_BITMAP0, 0);
4035                 vmcs_write64(EOI_EXIT_BITMAP1, 0);
4036                 vmcs_write64(EOI_EXIT_BITMAP2, 0);
4037                 vmcs_write64(EOI_EXIT_BITMAP3, 0);
4038
4039                 vmcs_write16(GUEST_INTR_STATUS, 0);
4040
4041                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
4042                 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
4043         }
4044
4045         if (!kvm_pause_in_guest(vmx->vcpu.kvm)) {
4046                 vmcs_write32(PLE_GAP, ple_gap);
4047                 vmx->ple_window = ple_window;
4048                 vmx->ple_window_dirty = true;
4049         }
4050
4051         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
4052         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
4053         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
4054
4055         vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
4056         vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
4057         vmx_set_constant_host_state(vmx);
4058         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
4059         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
4060
4061         if (cpu_has_vmx_vmfunc())
4062                 vmcs_write64(VM_FUNCTION_CONTROL, 0);
4063
4064         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
4065         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
4066         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
4067         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
4068         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
4069
4070         if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
4071                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
4072
4073         for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) {
4074                 u32 index = vmx_msr_index[i];
4075                 u32 data_low, data_high;
4076                 int j = vmx->nmsrs;
4077
4078                 if (rdmsr_safe(index, &data_low, &data_high) < 0)
4079                         continue;
4080                 if (wrmsr_safe(index, data_low, data_high) < 0)
4081                         continue;
4082                 vmx->guest_msrs[j].index = i;
4083                 vmx->guest_msrs[j].data = 0;
4084                 vmx->guest_msrs[j].mask = -1ull;
4085                 ++vmx->nmsrs;
4086         }
4087
4088         vmx->arch_capabilities = kvm_get_arch_capabilities();
4089
4090         vm_exit_controls_init(vmx, vmx_vmexit_ctrl());
4091
4092         /* 22.2.1, 20.8.1 */
4093         vm_entry_controls_init(vmx, vmx_vmentry_ctrl());
4094
4095         vmx->vcpu.arch.cr0_guest_owned_bits = X86_CR0_TS;
4096         vmcs_writel(CR0_GUEST_HOST_MASK, ~X86_CR0_TS);
4097
4098         set_cr4_guest_host_mask(vmx);
4099
4100         if (vmx_xsaves_supported())
4101                 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
4102
4103         if (enable_pml) {
4104                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
4105                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
4106         }
4107
4108         if (cpu_has_vmx_encls_vmexit())
4109                 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
4110
4111         if (pt_mode == PT_MODE_HOST_GUEST) {
4112                 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc));
4113                 /* Bit[6~0] are forced to 1, writes are ignored. */
4114                 vmx->pt_desc.guest.output_mask = 0x7F;
4115                 vmcs_write64(GUEST_IA32_RTIT_CTL, 0);
4116         }
4117 }
4118
4119 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
4120 {
4121         struct vcpu_vmx *vmx = to_vmx(vcpu);
4122         struct msr_data apic_base_msr;
4123         u64 cr0;
4124
4125         vmx->rmode.vm86_active = 0;
4126         vmx->spec_ctrl = 0;
4127
4128         vcpu->arch.microcode_version = 0x100000000ULL;
4129         vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
4130         kvm_set_cr8(vcpu, 0);
4131
4132         if (!init_event) {
4133                 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE |
4134                                      MSR_IA32_APICBASE_ENABLE;
4135                 if (kvm_vcpu_is_reset_bsp(vcpu))
4136                         apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
4137                 apic_base_msr.host_initiated = true;
4138                 kvm_set_apic_base(vcpu, &apic_base_msr);
4139         }
4140
4141         vmx_segment_cache_clear(vmx);
4142
4143         seg_setup(VCPU_SREG_CS);
4144         vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
4145         vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
4146
4147         seg_setup(VCPU_SREG_DS);
4148         seg_setup(VCPU_SREG_ES);
4149         seg_setup(VCPU_SREG_FS);
4150         seg_setup(VCPU_SREG_GS);
4151         seg_setup(VCPU_SREG_SS);
4152
4153         vmcs_write16(GUEST_TR_SELECTOR, 0);
4154         vmcs_writel(GUEST_TR_BASE, 0);
4155         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
4156         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
4157
4158         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
4159         vmcs_writel(GUEST_LDTR_BASE, 0);
4160         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
4161         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
4162
4163         if (!init_event) {
4164                 vmcs_write32(GUEST_SYSENTER_CS, 0);
4165                 vmcs_writel(GUEST_SYSENTER_ESP, 0);
4166                 vmcs_writel(GUEST_SYSENTER_EIP, 0);
4167                 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
4168         }
4169
4170         kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
4171         kvm_rip_write(vcpu, 0xfff0);
4172
4173         vmcs_writel(GUEST_GDTR_BASE, 0);
4174         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
4175
4176         vmcs_writel(GUEST_IDTR_BASE, 0);
4177         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
4178
4179         vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
4180         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
4181         vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
4182         if (kvm_mpx_supported())
4183                 vmcs_write64(GUEST_BNDCFGS, 0);
4184
4185         setup_msrs(vmx);
4186
4187         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
4188
4189         if (cpu_has_vmx_tpr_shadow() && !init_event) {
4190                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
4191                 if (cpu_need_tpr_shadow(vcpu))
4192                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
4193                                      __pa(vcpu->arch.apic->regs));
4194                 vmcs_write32(TPR_THRESHOLD, 0);
4195         }
4196
4197         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
4198
4199         if (vmx->vpid != 0)
4200                 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
4201
4202         cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
4203         vmx->vcpu.arch.cr0 = cr0;
4204         vmx_set_cr0(vcpu, cr0); /* enter rmode */
4205         vmx_set_cr4(vcpu, 0);
4206         vmx_set_efer(vcpu, 0);
4207
4208         update_exception_bitmap(vcpu);
4209
4210         vpid_sync_context(vmx->vpid);
4211         if (init_event)
4212                 vmx_clear_hlt(vcpu);
4213 }
4214
4215 static void enable_irq_window(struct kvm_vcpu *vcpu)
4216 {
4217         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
4218                       CPU_BASED_VIRTUAL_INTR_PENDING);
4219 }
4220
4221 static void enable_nmi_window(struct kvm_vcpu *vcpu)
4222 {
4223         if (!enable_vnmi ||
4224             vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
4225                 enable_irq_window(vcpu);
4226                 return;
4227         }
4228
4229         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
4230                       CPU_BASED_VIRTUAL_NMI_PENDING);
4231 }
4232
4233 static void vmx_inject_irq(struct kvm_vcpu *vcpu)
4234 {
4235         struct vcpu_vmx *vmx = to_vmx(vcpu);
4236         uint32_t intr;
4237         int irq = vcpu->arch.interrupt.nr;
4238
4239         trace_kvm_inj_virq(irq);
4240
4241         ++vcpu->stat.irq_injections;
4242         if (vmx->rmode.vm86_active) {
4243                 int inc_eip = 0;
4244                 if (vcpu->arch.interrupt.soft)
4245                         inc_eip = vcpu->arch.event_exit_inst_len;
4246                 if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
4247                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4248                 return;
4249         }
4250         intr = irq | INTR_INFO_VALID_MASK;
4251         if (vcpu->arch.interrupt.soft) {
4252                 intr |= INTR_TYPE_SOFT_INTR;
4253                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
4254                              vmx->vcpu.arch.event_exit_inst_len);
4255         } else
4256                 intr |= INTR_TYPE_EXT_INTR;
4257         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
4258
4259         vmx_clear_hlt(vcpu);
4260 }
4261
4262 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
4263 {
4264         struct vcpu_vmx *vmx = to_vmx(vcpu);
4265
4266         if (!enable_vnmi) {
4267                 /*
4268                  * Tracking the NMI-blocked state in software is built upon
4269                  * finding the next open IRQ window. This, in turn, depends on
4270                  * well-behaving guests: They have to keep IRQs disabled at
4271                  * least as long as the NMI handler runs. Otherwise we may
4272                  * cause NMI nesting, maybe breaking the guest. But as this is
4273                  * highly unlikely, we can live with the residual risk.
4274                  */
4275                 vmx->loaded_vmcs->soft_vnmi_blocked = 1;
4276                 vmx->loaded_vmcs->vnmi_blocked_time = 0;
4277         }
4278
4279         ++vcpu->stat.nmi_injections;
4280         vmx->loaded_vmcs->nmi_known_unmasked = false;
4281
4282         if (vmx->rmode.vm86_active) {
4283                 if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
4284                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4285                 return;
4286         }
4287
4288         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
4289                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
4290
4291         vmx_clear_hlt(vcpu);
4292 }
4293
4294 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
4295 {
4296         struct vcpu_vmx *vmx = to_vmx(vcpu);
4297         bool masked;
4298
4299         if (!enable_vnmi)
4300                 return vmx->loaded_vmcs->soft_vnmi_blocked;
4301         if (vmx->loaded_vmcs->nmi_known_unmasked)
4302                 return false;
4303         masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
4304         vmx->loaded_vmcs->nmi_known_unmasked = !masked;
4305         return masked;
4306 }
4307
4308 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
4309 {
4310         struct vcpu_vmx *vmx = to_vmx(vcpu);
4311
4312         if (!enable_vnmi) {
4313                 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) {
4314                         vmx->loaded_vmcs->soft_vnmi_blocked = masked;
4315                         vmx->loaded_vmcs->vnmi_blocked_time = 0;
4316                 }
4317         } else {
4318                 vmx->loaded_vmcs->nmi_known_unmasked = !masked;
4319                 if (masked)
4320                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
4321                                       GUEST_INTR_STATE_NMI);
4322                 else
4323                         vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
4324                                         GUEST_INTR_STATE_NMI);
4325         }
4326 }
4327
4328 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
4329 {
4330         if (to_vmx(vcpu)->nested.nested_run_pending)
4331                 return 0;
4332
4333         if (!enable_vnmi &&
4334             to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked)
4335                 return 0;
4336
4337         return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4338                   (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
4339                    | GUEST_INTR_STATE_NMI));
4340 }
4341
4342 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
4343 {
4344         return (!to_vmx(vcpu)->nested.nested_run_pending &&
4345                 vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
4346                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
4347                         (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
4348 }
4349
4350 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
4351 {
4352         int ret;
4353
4354         if (enable_unrestricted_guest)
4355                 return 0;
4356
4357         ret = x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
4358                                     PAGE_SIZE * 3);
4359         if (ret)
4360                 return ret;
4361         to_kvm_vmx(kvm)->tss_addr = addr;
4362         return init_rmode_tss(kvm);
4363 }
4364
4365 static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
4366 {
4367         to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr;
4368         return 0;
4369 }
4370
4371 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
4372 {
4373         switch (vec) {
4374         case BP_VECTOR:
4375                 /*
4376                  * Update instruction length as we may reinject the exception
4377                  * from user space while in guest debugging mode.
4378                  */
4379                 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
4380                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
4381                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
4382                         return false;
4383                 /* fall through */
4384         case DB_VECTOR:
4385                 if (vcpu->guest_debug &
4386                         (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
4387                         return false;
4388                 /* fall through */
4389         case DE_VECTOR:
4390         case OF_VECTOR:
4391         case BR_VECTOR:
4392         case UD_VECTOR:
4393         case DF_VECTOR:
4394         case SS_VECTOR:
4395         case GP_VECTOR:
4396         case MF_VECTOR:
4397                 return true;
4398         break;
4399         }
4400         return false;
4401 }
4402
4403 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
4404                                   int vec, u32 err_code)
4405 {
4406         /*
4407          * Instruction with address size override prefix opcode 0x67
4408          * Cause the #SS fault with 0 error code in VM86 mode.
4409          */
4410         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
4411                 if (kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE) {
4412                         if (vcpu->arch.halt_request) {
4413                                 vcpu->arch.halt_request = 0;
4414                                 return kvm_vcpu_halt(vcpu);
4415                         }
4416                         return 1;
4417                 }
4418                 return 0;
4419         }
4420
4421         /*
4422          * Forward all other exceptions that are valid in real mode.
4423          * FIXME: Breaks guest debugging in real mode, needs to be fixed with
4424          *        the required debugging infrastructure rework.
4425          */
4426         kvm_queue_exception(vcpu, vec);
4427         return 1;
4428 }
4429
4430 /*
4431  * Trigger machine check on the host. We assume all the MSRs are already set up
4432  * by the CPU and that we still run on the same CPU as the MCE occurred on.
4433  * We pass a fake environment to the machine check handler because we want
4434  * the guest to be always treated like user space, no matter what context
4435  * it used internally.
4436  */
4437 static void kvm_machine_check(void)
4438 {
4439 #if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
4440         struct pt_regs regs = {
4441                 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
4442                 .flags = X86_EFLAGS_IF,
4443         };
4444
4445         do_machine_check(&regs, 0);
4446 #endif
4447 }
4448
4449 static int handle_machine_check(struct kvm_vcpu *vcpu)
4450 {
4451         /* already handled by vcpu_run */
4452         return 1;
4453 }
4454
4455 static int handle_exception(struct kvm_vcpu *vcpu)
4456 {
4457         struct vcpu_vmx *vmx = to_vmx(vcpu);
4458         struct kvm_run *kvm_run = vcpu->run;
4459         u32 intr_info, ex_no, error_code;
4460         unsigned long cr2, rip, dr6;
4461         u32 vect_info;
4462         enum emulation_result er;
4463
4464         vect_info = vmx->idt_vectoring_info;
4465         intr_info = vmx->exit_intr_info;
4466
4467         if (is_machine_check(intr_info))
4468                 return handle_machine_check(vcpu);
4469
4470         if (is_nmi(intr_info))
4471                 return 1;  /* already handled by vmx_vcpu_run() */
4472
4473         if (is_invalid_opcode(intr_info))
4474                 return handle_ud(vcpu);
4475
4476         error_code = 0;
4477         if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
4478                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
4479
4480         if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) {
4481                 WARN_ON_ONCE(!enable_vmware_backdoor);
4482                 er = kvm_emulate_instruction(vcpu,
4483                         EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL);
4484                 if (er == EMULATE_USER_EXIT)
4485                         return 0;
4486                 else if (er != EMULATE_DONE)
4487                         kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
4488                 return 1;
4489         }
4490
4491         /*
4492          * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
4493          * MMIO, it is better to report an internal error.
4494          * See the comments in vmx_handle_exit.
4495          */
4496         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
4497             !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
4498                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4499                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
4500                 vcpu->run->internal.ndata = 3;
4501                 vcpu->run->internal.data[0] = vect_info;
4502                 vcpu->run->internal.data[1] = intr_info;
4503                 vcpu->run->internal.data[2] = error_code;
4504                 return 0;
4505         }
4506
4507         if (is_page_fault(intr_info)) {
4508                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
4509                 /* EPT won't cause page fault directly */
4510                 WARN_ON_ONCE(!vcpu->arch.apf.host_apf_reason && enable_ept);
4511                 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
4512         }
4513
4514         ex_no = intr_info & INTR_INFO_VECTOR_MASK;
4515
4516         if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
4517                 return handle_rmode_exception(vcpu, ex_no, error_code);
4518
4519         switch (ex_no) {
4520         case AC_VECTOR:
4521                 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
4522                 return 1;
4523         case DB_VECTOR:
4524                 dr6 = vmcs_readl(EXIT_QUALIFICATION);
4525                 if (!(vcpu->guest_debug &
4526                       (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
4527                         vcpu->arch.dr6 &= ~15;
4528                         vcpu->arch.dr6 |= dr6 | DR6_RTM;
4529                         if (is_icebp(intr_info))
4530                                 skip_emulated_instruction(vcpu);
4531
4532                         kvm_queue_exception(vcpu, DB_VECTOR);
4533                         return 1;
4534                 }
4535                 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
4536                 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
4537                 /* fall through */
4538         case BP_VECTOR:
4539                 /*
4540                  * Update instruction length as we may reinject #BP from
4541                  * user space while in guest debugging mode. Reading it for
4542                  * #DB as well causes no harm, it is not used in that case.
4543                  */
4544                 vmx->vcpu.arch.event_exit_inst_len =
4545                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
4546                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
4547                 rip = kvm_rip_read(vcpu);
4548                 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
4549                 kvm_run->debug.arch.exception = ex_no;
4550                 break;
4551         default:
4552                 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
4553                 kvm_run->ex.exception = ex_no;
4554                 kvm_run->ex.error_code = error_code;
4555                 break;
4556         }
4557         return 0;
4558 }
4559
4560 static int handle_external_interrupt(struct kvm_vcpu *vcpu)
4561 {
4562         ++vcpu->stat.irq_exits;
4563         return 1;
4564 }
4565
4566 static int handle_triple_fault(struct kvm_vcpu *vcpu)
4567 {
4568         vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
4569         vcpu->mmio_needed = 0;
4570         return 0;
4571 }
4572
4573 static int handle_io(struct kvm_vcpu *vcpu)
4574 {
4575         unsigned long exit_qualification;
4576         int size, in, string;
4577         unsigned port;
4578
4579         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4580         string = (exit_qualification & 16) != 0;
4581
4582         ++vcpu->stat.io_exits;
4583
4584         if (string)
4585                 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
4586
4587         port = exit_qualification >> 16;
4588         size = (exit_qualification & 7) + 1;
4589         in = (exit_qualification & 8) != 0;
4590
4591         return kvm_fast_pio(vcpu, size, port, in);
4592 }
4593
4594 static void
4595 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4596 {
4597         /*
4598          * Patch in the VMCALL instruction:
4599          */
4600         hypercall[0] = 0x0f;
4601         hypercall[1] = 0x01;
4602         hypercall[2] = 0xc1;
4603 }
4604
4605 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
4606 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
4607 {
4608         if (is_guest_mode(vcpu)) {
4609                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4610                 unsigned long orig_val = val;
4611
4612                 /*
4613                  * We get here when L2 changed cr0 in a way that did not change
4614                  * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
4615                  * but did change L0 shadowed bits. So we first calculate the
4616                  * effective cr0 value that L1 would like to write into the
4617                  * hardware. It consists of the L2-owned bits from the new
4618                  * value combined with the L1-owned bits from L1's guest_cr0.
4619                  */
4620                 val = (val & ~vmcs12->cr0_guest_host_mask) |
4621                         (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
4622
4623                 if (!nested_guest_cr0_valid(vcpu, val))
4624                         return 1;
4625
4626                 if (kvm_set_cr0(vcpu, val))
4627                         return 1;
4628                 vmcs_writel(CR0_READ_SHADOW, orig_val);
4629                 return 0;
4630         } else {
4631                 if (to_vmx(vcpu)->nested.vmxon &&
4632                     !nested_host_cr0_valid(vcpu, val))
4633                         return 1;
4634
4635                 return kvm_set_cr0(vcpu, val);
4636         }
4637 }
4638
4639 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
4640 {
4641         if (is_guest_mode(vcpu)) {
4642                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4643                 unsigned long orig_val = val;
4644
4645                 /* analogously to handle_set_cr0 */
4646                 val = (val & ~vmcs12->cr4_guest_host_mask) |
4647                         (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
4648                 if (kvm_set_cr4(vcpu, val))
4649                         return 1;
4650                 vmcs_writel(CR4_READ_SHADOW, orig_val);
4651                 return 0;
4652         } else
4653                 return kvm_set_cr4(vcpu, val);
4654 }
4655
4656 static int handle_desc(struct kvm_vcpu *vcpu)
4657 {
4658         WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP));
4659         return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
4660 }
4661
4662 static int handle_cr(struct kvm_vcpu *vcpu)
4663 {
4664         unsigned long exit_qualification, val;
4665         int cr;
4666         int reg;
4667         int err;
4668         int ret;
4669
4670         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4671         cr = exit_qualification & 15;
4672         reg = (exit_qualification >> 8) & 15;
4673         switch ((exit_qualification >> 4) & 3) {
4674         case 0: /* mov to cr */
4675                 val = kvm_register_readl(vcpu, reg);
4676                 trace_kvm_cr_write(cr, val);
4677                 switch (cr) {
4678                 case 0:
4679                         err = handle_set_cr0(vcpu, val);
4680                         return kvm_complete_insn_gp(vcpu, err);
4681                 case 3:
4682                         WARN_ON_ONCE(enable_unrestricted_guest);
4683                         err = kvm_set_cr3(vcpu, val);
4684                         return kvm_complete_insn_gp(vcpu, err);
4685                 case 4:
4686                         err = handle_set_cr4(vcpu, val);
4687                         return kvm_complete_insn_gp(vcpu, err);
4688                 case 8: {
4689                                 u8 cr8_prev = kvm_get_cr8(vcpu);
4690                                 u8 cr8 = (u8)val;
4691                                 err = kvm_set_cr8(vcpu, cr8);
4692                                 ret = kvm_complete_insn_gp(vcpu, err);
4693                                 if (lapic_in_kernel(vcpu))
4694                                         return ret;
4695                                 if (cr8_prev <= cr8)
4696                                         return ret;
4697                                 /*
4698                                  * TODO: we might be squashing a
4699                                  * KVM_GUESTDBG_SINGLESTEP-triggered
4700                                  * KVM_EXIT_DEBUG here.
4701                                  */
4702                                 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
4703                                 return 0;
4704                         }
4705                 }
4706                 break;
4707         case 2: /* clts */
4708                 WARN_ONCE(1, "Guest should always own CR0.TS");
4709                 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
4710                 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
4711                 return kvm_skip_emulated_instruction(vcpu);
4712         case 1: /*mov from cr*/
4713                 switch (cr) {
4714                 case 3:
4715                         WARN_ON_ONCE(enable_unrestricted_guest);
4716                         val = kvm_read_cr3(vcpu);
4717                         kvm_register_write(vcpu, reg, val);
4718                         trace_kvm_cr_read(cr, val);
4719                         return kvm_skip_emulated_instruction(vcpu);
4720                 case 8:
4721                         val = kvm_get_cr8(vcpu);
4722                         kvm_register_write(vcpu, reg, val);
4723                         trace_kvm_cr_read(cr, val);
4724                         return kvm_skip_emulated_instruction(vcpu);
4725                 }
4726                 break;
4727         case 3: /* lmsw */
4728                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
4729                 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
4730                 kvm_lmsw(vcpu, val);
4731
4732                 return kvm_skip_emulated_instruction(vcpu);
4733         default:
4734                 break;
4735         }
4736         vcpu->run->exit_reason = 0;
4737         vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
4738                (int)(exit_qualification >> 4) & 3, cr);
4739         return 0;
4740 }
4741
4742 static int handle_dr(struct kvm_vcpu *vcpu)
4743 {
4744         unsigned long exit_qualification;
4745         int dr, dr7, reg;
4746
4747         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4748         dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
4749
4750         /* First, if DR does not exist, trigger UD */
4751         if (!kvm_require_dr(vcpu, dr))
4752                 return 1;
4753
4754         /* Do not handle if the CPL > 0, will trigger GP on re-entry */
4755         if (!kvm_require_cpl(vcpu, 0))
4756                 return 1;
4757         dr7 = vmcs_readl(GUEST_DR7);
4758         if (dr7 & DR7_GD) {
4759                 /*
4760                  * As the vm-exit takes precedence over the debug trap, we
4761                  * need to emulate the latter, either for the host or the
4762                  * guest debugging itself.
4763                  */
4764                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
4765                         vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
4766                         vcpu->run->debug.arch.dr7 = dr7;
4767                         vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
4768                         vcpu->run->debug.arch.exception = DB_VECTOR;
4769                         vcpu->run->exit_reason = KVM_EXIT_DEBUG;
4770                         return 0;
4771                 } else {
4772                         vcpu->arch.dr6 &= ~15;
4773                         vcpu->arch.dr6 |= DR6_BD | DR6_RTM;
4774                         kvm_queue_exception(vcpu, DB_VECTOR);
4775                         return 1;
4776                 }
4777         }
4778
4779         if (vcpu->guest_debug == 0) {
4780                 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
4781                                 CPU_BASED_MOV_DR_EXITING);
4782
4783                 /*
4784                  * No more DR vmexits; force a reload of the debug registers
4785                  * and reenter on this instruction.  The next vmexit will
4786                  * retrieve the full state of the debug registers.
4787                  */
4788                 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
4789                 return 1;
4790         }
4791
4792         reg = DEBUG_REG_ACCESS_REG(exit_qualification);
4793         if (exit_qualification & TYPE_MOV_FROM_DR) {
4794                 unsigned long val;
4795
4796                 if (kvm_get_dr(vcpu, dr, &val))
4797                         return 1;
4798                 kvm_register_write(vcpu, reg, val);
4799         } else
4800                 if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
4801                         return 1;
4802
4803         return kvm_skip_emulated_instruction(vcpu);
4804 }
4805
4806 static u64 vmx_get_dr6(struct kvm_vcpu *vcpu)
4807 {
4808         return vcpu->arch.dr6;
4809 }
4810
4811 static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
4812 {
4813 }
4814
4815 static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
4816 {
4817         get_debugreg(vcpu->arch.db[0], 0);
4818         get_debugreg(vcpu->arch.db[1], 1);
4819         get_debugreg(vcpu->arch.db[2], 2);
4820         get_debugreg(vcpu->arch.db[3], 3);
4821         get_debugreg(vcpu->arch.dr6, 6);
4822         vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);
4823
4824         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
4825         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, CPU_BASED_MOV_DR_EXITING);
4826 }
4827
4828 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
4829 {
4830         vmcs_writel(GUEST_DR7, val);
4831 }
4832
4833 static int handle_cpuid(struct kvm_vcpu *vcpu)
4834 {
4835         return kvm_emulate_cpuid(vcpu);
4836 }
4837
4838 static int handle_rdmsr(struct kvm_vcpu *vcpu)
4839 {
4840         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
4841         struct msr_data msr_info;
4842
4843         msr_info.index = ecx;
4844         msr_info.host_initiated = false;
4845         if (vmx_get_msr(vcpu, &msr_info)) {
4846                 trace_kvm_msr_read_ex(ecx);
4847                 kvm_inject_gp(vcpu, 0);
4848                 return 1;
4849         }
4850
4851         trace_kvm_msr_read(ecx, msr_info.data);
4852
4853         /* FIXME: handling of bits 32:63 of rax, rdx */
4854         vcpu->arch.regs[VCPU_REGS_RAX] = msr_info.data & -1u;
4855         vcpu->arch.regs[VCPU_REGS_RDX] = (msr_info.data >> 32) & -1u;
4856         return kvm_skip_emulated_instruction(vcpu);
4857 }
4858
4859 static int handle_wrmsr(struct kvm_vcpu *vcpu)
4860 {
4861         struct msr_data msr;
4862         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
4863         u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
4864                 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
4865
4866         msr.data = data;
4867         msr.index = ecx;
4868         msr.host_initiated = false;
4869         if (kvm_set_msr(vcpu, &msr) != 0) {
4870                 trace_kvm_msr_write_ex(ecx, data);
4871                 kvm_inject_gp(vcpu, 0);
4872                 return 1;
4873         }
4874
4875         trace_kvm_msr_write(ecx, data);
4876         return kvm_skip_emulated_instruction(vcpu);
4877 }
4878
4879 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
4880 {
4881         kvm_apic_update_ppr(vcpu);
4882         return 1;
4883 }
4884
4885 static int handle_interrupt_window(struct kvm_vcpu *vcpu)
4886 {
4887         vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
4888                         CPU_BASED_VIRTUAL_INTR_PENDING);
4889
4890         kvm_make_request(KVM_REQ_EVENT, vcpu);
4891
4892         ++vcpu->stat.irq_window_exits;
4893         return 1;
4894 }
4895
4896 static int handle_halt(struct kvm_vcpu *vcpu)
4897 {
4898         return kvm_emulate_halt(vcpu);
4899 }
4900
4901 static int handle_vmcall(struct kvm_vcpu *vcpu)
4902 {
4903         return kvm_emulate_hypercall(vcpu);
4904 }
4905
4906 static int handle_invd(struct kvm_vcpu *vcpu)
4907 {
4908         return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
4909 }
4910
4911 static int handle_invlpg(struct kvm_vcpu *vcpu)
4912 {
4913         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4914
4915         kvm_mmu_invlpg(vcpu, exit_qualification);
4916         return kvm_skip_emulated_instruction(vcpu);
4917 }
4918
4919 static int handle_rdpmc(struct kvm_vcpu *vcpu)
4920 {
4921         int err;
4922
4923         err = kvm_rdpmc(vcpu);
4924         return kvm_complete_insn_gp(vcpu, err);
4925 }
4926
4927 static int handle_wbinvd(struct kvm_vcpu *vcpu)
4928 {
4929         return kvm_emulate_wbinvd(vcpu);
4930 }
4931
4932 static int handle_xsetbv(struct kvm_vcpu *vcpu)
4933 {
4934         u64 new_bv = kvm_read_edx_eax(vcpu);
4935         u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);
4936
4937         if (kvm_set_xcr(vcpu, index, new_bv) == 0)
4938                 return kvm_skip_emulated_instruction(vcpu);
4939         return 1;
4940 }
4941
4942 static int handle_xsaves(struct kvm_vcpu *vcpu)
4943 {
4944         kvm_skip_emulated_instruction(vcpu);
4945         WARN(1, "this should never happen\n");
4946         return 1;
4947 }
4948
4949 static int handle_xrstors(struct kvm_vcpu *vcpu)
4950 {
4951         kvm_skip_emulated_instruction(vcpu);
4952         WARN(1, "this should never happen\n");
4953         return 1;
4954 }
4955
4956 static int handle_apic_access(struct kvm_vcpu *vcpu)
4957 {
4958         if (likely(fasteoi)) {
4959                 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4960                 int access_type, offset;
4961
4962                 access_type = exit_qualification & APIC_ACCESS_TYPE;
4963                 offset = exit_qualification & APIC_ACCESS_OFFSET;
4964                 /*
4965                  * Sane guest uses MOV to write EOI, with written value
4966                  * not cared. So make a short-circuit here by avoiding
4967                  * heavy instruction emulation.
4968                  */
4969                 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
4970                     (offset == APIC_EOI)) {
4971                         kvm_lapic_set_eoi(vcpu);
4972                         return kvm_skip_emulated_instruction(vcpu);
4973                 }
4974         }
4975         return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
4976 }
4977
4978 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
4979 {
4980         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4981         int vector = exit_qualification & 0xff;
4982
4983         /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
4984         kvm_apic_set_eoi_accelerated(vcpu, vector);
4985         return 1;
4986 }
4987
4988 static int handle_apic_write(struct kvm_vcpu *vcpu)
4989 {
4990         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4991         u32 offset = exit_qualification & 0xfff;
4992
4993         /* APIC-write VM exit is trap-like and thus no need to adjust IP */
4994         kvm_apic_write_nodecode(vcpu, offset);
4995         return 1;
4996 }
4997
4998 static int handle_task_switch(struct kvm_vcpu *vcpu)
4999 {
5000         struct vcpu_vmx *vmx = to_vmx(vcpu);
5001         unsigned long exit_qualification;
5002         bool has_error_code = false;
5003         u32 error_code = 0;
5004         u16 tss_selector;
5005         int reason, type, idt_v, idt_index;
5006
5007         idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
5008         idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
5009         type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
5010
5011         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5012
5013         reason = (u32)exit_qualification >> 30;
5014         if (reason == TASK_SWITCH_GATE && idt_v) {
5015                 switch (type) {
5016                 case INTR_TYPE_NMI_INTR:
5017                         vcpu->arch.nmi_injected = false;
5018                         vmx_set_nmi_mask(vcpu, true);
5019                         break;
5020                 case INTR_TYPE_EXT_INTR:
5021                 case INTR_TYPE_SOFT_INTR:
5022                         kvm_clear_interrupt_queue(vcpu);
5023                         break;
5024                 case INTR_TYPE_HARD_EXCEPTION:
5025                         if (vmx->idt_vectoring_info &
5026                             VECTORING_INFO_DELIVER_CODE_MASK) {
5027                                 has_error_code = true;
5028                                 error_code =
5029                                         vmcs_read32(IDT_VECTORING_ERROR_CODE);
5030                         }
5031                         /* fall through */
5032                 case INTR_TYPE_SOFT_EXCEPTION:
5033                         kvm_clear_exception_queue(vcpu);
5034                         break;
5035                 default:
5036                         break;
5037                 }
5038         }
5039         tss_selector = exit_qualification;
5040
5041         if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
5042                        type != INTR_TYPE_EXT_INTR &&
5043                        type != INTR_TYPE_NMI_INTR))
5044                 skip_emulated_instruction(vcpu);
5045
5046         if (kvm_task_switch(vcpu, tss_selector,
5047                             type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
5048                             has_error_code, error_code) == EMULATE_FAIL) {
5049                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5050                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
5051                 vcpu->run->internal.ndata = 0;
5052                 return 0;
5053         }
5054
5055         /*
5056          * TODO: What about debug traps on tss switch?
5057          *       Are we supposed to inject them and update dr6?
5058          */
5059
5060         return 1;
5061 }
5062
5063 static int handle_ept_violation(struct kvm_vcpu *vcpu)
5064 {
5065         unsigned long exit_qualification;
5066         gpa_t gpa;
5067         u64 error_code;
5068
5069         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5070
5071         /*
5072          * EPT violation happened while executing iret from NMI,
5073          * "blocked by NMI" bit has to be set before next VM entry.
5074          * There are errata that may cause this bit to not be set:
5075          * AAK134, BY25.
5076          */
5077         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
5078                         enable_vnmi &&
5079                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
5080                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
5081
5082         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5083         trace_kvm_page_fault(gpa, exit_qualification);
5084
5085         /* Is it a read fault? */
5086         error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
5087                      ? PFERR_USER_MASK : 0;
5088         /* Is it a write fault? */
5089         error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
5090                       ? PFERR_WRITE_MASK : 0;
5091         /* Is it a fetch fault? */
5092         error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
5093                       ? PFERR_FETCH_MASK : 0;
5094         /* ept page table entry is present? */
5095         error_code |= (exit_qualification &
5096                        (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE |
5097                         EPT_VIOLATION_EXECUTABLE))
5098                       ? PFERR_PRESENT_MASK : 0;
5099
5100         error_code |= (exit_qualification & 0x100) != 0 ?
5101                PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
5102
5103         vcpu->arch.exit_qualification = exit_qualification;
5104         return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
5105 }
5106
5107 static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
5108 {
5109         gpa_t gpa;
5110
5111         /*
5112          * A nested guest cannot optimize MMIO vmexits, because we have an
5113          * nGPA here instead of the required GPA.
5114          */
5115         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5116         if (!is_guest_mode(vcpu) &&
5117             !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
5118                 trace_kvm_fast_mmio(gpa);
5119                 /*
5120                  * Doing kvm_skip_emulated_instruction() depends on undefined
5121                  * behavior: Intel's manual doesn't mandate
5122                  * VM_EXIT_INSTRUCTION_LEN to be set in VMCS when EPT MISCONFIG
5123                  * occurs and while on real hardware it was observed to be set,
5124                  * other hypervisors (namely Hyper-V) don't set it, we end up
5125                  * advancing IP with some random value. Disable fast mmio when
5126                  * running nested and keep it for real hardware in hope that
5127                  * VM_EXIT_INSTRUCTION_LEN will always be set correctly.
5128                  */
5129                 if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
5130                         return kvm_skip_emulated_instruction(vcpu);
5131                 else
5132                         return kvm_emulate_instruction(vcpu, EMULTYPE_SKIP) ==
5133                                                                 EMULATE_DONE;
5134         }
5135
5136         return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
5137 }
5138
5139 static int handle_nmi_window(struct kvm_vcpu *vcpu)
5140 {
5141         WARN_ON_ONCE(!enable_vnmi);
5142         vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
5143                         CPU_BASED_VIRTUAL_NMI_PENDING);
5144         ++vcpu->stat.nmi_window_exits;
5145         kvm_make_request(KVM_REQ_EVENT, vcpu);
5146
5147         return 1;
5148 }
5149
5150 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
5151 {
5152         struct vcpu_vmx *vmx = to_vmx(vcpu);
5153         enum emulation_result err = EMULATE_DONE;
5154         int ret = 1;
5155         u32 cpu_exec_ctrl;
5156         bool intr_window_requested;
5157         unsigned count = 130;
5158
5159         /*
5160          * We should never reach the point where we are emulating L2
5161          * due to invalid guest state as that means we incorrectly
5162          * allowed a nested VMEntry with an invalid vmcs12.
5163          */
5164         WARN_ON_ONCE(vmx->emulation_required && vmx->nested.nested_run_pending);
5165
5166         cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
5167         intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
5168
5169         while (vmx->emulation_required && count-- != 0) {
5170                 if (intr_window_requested && vmx_interrupt_allowed(vcpu))
5171                         return handle_interrupt_window(&vmx->vcpu);
5172
5173                 if (kvm_test_request(KVM_REQ_EVENT, vcpu))
5174                         return 1;
5175
5176                 err = kvm_emulate_instruction(vcpu, 0);
5177
5178                 if (err == EMULATE_USER_EXIT) {
5179                         ++vcpu->stat.mmio_exits;
5180                         ret = 0;
5181                         goto out;
5182                 }
5183
5184                 if (err != EMULATE_DONE)
5185                         goto emulation_error;
5186
5187                 if (vmx->emulation_required && !vmx->rmode.vm86_active &&
5188                     vcpu->arch.exception.pending)
5189                         goto emulation_error;
5190
5191                 if (vcpu->arch.halt_request) {
5192                         vcpu->arch.halt_request = 0;
5193                         ret = kvm_vcpu_halt(vcpu);
5194                         goto out;
5195                 }
5196
5197                 if (signal_pending(current))
5198                         goto out;
5199                 if (need_resched())
5200                         schedule();
5201         }
5202
5203 out:
5204         return ret;
5205
5206 emulation_error:
5207         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5208         vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
5209         vcpu->run->internal.ndata = 0;
5210         return 0;
5211 }
5212
5213 static void grow_ple_window(struct kvm_vcpu *vcpu)
5214 {
5215         struct vcpu_vmx *vmx = to_vmx(vcpu);
5216         int old = vmx->ple_window;
5217
5218         vmx->ple_window = __grow_ple_window(old, ple_window,
5219                                             ple_window_grow,
5220                                             ple_window_max);
5221
5222         if (vmx->ple_window != old)
5223                 vmx->ple_window_dirty = true;
5224
5225         trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);
5226 }
5227
5228 static void shrink_ple_window(struct kvm_vcpu *vcpu)
5229 {
5230         struct vcpu_vmx *vmx = to_vmx(vcpu);
5231         int old = vmx->ple_window;
5232
5233         vmx->ple_window = __shrink_ple_window(old, ple_window,
5234                                               ple_window_shrink,
5235                                               ple_window);
5236
5237         if (vmx->ple_window != old)
5238                 vmx->ple_window_dirty = true;
5239
5240         trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);
5241 }
5242
5243 /*
5244  * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
5245  */
5246 static void wakeup_handler(void)
5247 {
5248         struct kvm_vcpu *vcpu;
5249         int cpu = smp_processor_id();
5250
5251         spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
5252         list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
5253                         blocked_vcpu_list) {
5254                 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
5255
5256                 if (pi_test_on(pi_desc) == 1)
5257                         kvm_vcpu_kick(vcpu);
5258         }
5259         spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
5260 }
5261
5262 static void vmx_enable_tdp(void)
5263 {
5264         kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK,
5265                 enable_ept_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull,
5266                 enable_ept_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull,
5267                 0ull, VMX_EPT_EXECUTABLE_MASK,
5268                 cpu_has_vmx_ept_execute_only() ? 0ull : VMX_EPT_READABLE_MASK,
5269                 VMX_EPT_RWX_MASK, 0ull);
5270
5271         ept_set_mmio_spte_mask();
5272         kvm_enable_tdp();
5273 }
5274
5275 /*
5276  * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
5277  * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
5278  */
5279 static int handle_pause(struct kvm_vcpu *vcpu)
5280 {
5281         if (!kvm_pause_in_guest(vcpu->kvm))
5282                 grow_ple_window(vcpu);
5283
5284         /*
5285          * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting"
5286          * VM-execution control is ignored if CPL > 0. OTOH, KVM
5287          * never set PAUSE_EXITING and just set PLE if supported,
5288          * so the vcpu must be CPL=0 if it gets a PAUSE exit.
5289          */
5290         kvm_vcpu_on_spin(vcpu, true);
5291         return kvm_skip_emulated_instruction(vcpu);
5292 }
5293
5294 static int handle_nop(struct kvm_vcpu *vcpu)
5295 {
5296         return kvm_skip_emulated_instruction(vcpu);
5297 }
5298
5299 static int handle_mwait(struct kvm_vcpu *vcpu)
5300 {
5301         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
5302         return handle_nop(vcpu);
5303 }
5304
5305 static int handle_invalid_op(struct kvm_vcpu *vcpu)
5306 {
5307         kvm_queue_exception(vcpu, UD_VECTOR);
5308         return 1;
5309 }
5310
5311 static int handle_monitor_trap(struct kvm_vcpu *vcpu)
5312 {
5313         return 1;
5314 }
5315
5316 static int handle_monitor(struct kvm_vcpu *vcpu)
5317 {
5318         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
5319         return handle_nop(vcpu);
5320 }
5321
5322 static int handle_invpcid(struct kvm_vcpu *vcpu)
5323 {
5324         u32 vmx_instruction_info;
5325         unsigned long type;
5326         bool pcid_enabled;
5327         gva_t gva;
5328         struct x86_exception e;
5329         unsigned i;
5330         unsigned long roots_to_free = 0;
5331         struct {
5332                 u64 pcid;
5333                 u64 gla;
5334         } operand;
5335
5336         if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
5337                 kvm_queue_exception(vcpu, UD_VECTOR);
5338                 return 1;
5339         }
5340
5341         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5342         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
5343
5344         if (type > 3) {
5345                 kvm_inject_gp(vcpu, 0);
5346                 return 1;
5347         }
5348
5349         /* According to the Intel instruction reference, the memory operand
5350          * is read even if it isn't needed (e.g., for type==all)
5351          */
5352         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
5353                                 vmx_instruction_info, false, &gva))
5354                 return 1;
5355
5356         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
5357                 kvm_inject_page_fault(vcpu, &e);
5358                 return 1;
5359         }
5360
5361         if (operand.pcid >> 12 != 0) {
5362                 kvm_inject_gp(vcpu, 0);
5363                 return 1;
5364         }
5365
5366         pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
5367
5368         switch (type) {
5369         case INVPCID_TYPE_INDIV_ADDR:
5370                 if ((!pcid_enabled && (operand.pcid != 0)) ||
5371                     is_noncanonical_address(operand.gla, vcpu)) {
5372                         kvm_inject_gp(vcpu, 0);
5373                         return 1;
5374                 }
5375                 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
5376                 return kvm_skip_emulated_instruction(vcpu);
5377
5378         case INVPCID_TYPE_SINGLE_CTXT:
5379                 if (!pcid_enabled && (operand.pcid != 0)) {
5380                         kvm_inject_gp(vcpu, 0);
5381                         return 1;
5382                 }
5383
5384                 if (kvm_get_active_pcid(vcpu) == operand.pcid) {
5385                         kvm_mmu_sync_roots(vcpu);
5386                         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
5387                 }
5388
5389                 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5390                         if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].cr3)
5391                             == operand.pcid)
5392                                 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
5393
5394                 kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free);
5395                 /*
5396                  * If neither the current cr3 nor any of the prev_roots use the
5397                  * given PCID, then nothing needs to be done here because a
5398                  * resync will happen anyway before switching to any other CR3.
5399                  */
5400
5401                 return kvm_skip_emulated_instruction(vcpu);
5402
5403         case INVPCID_TYPE_ALL_NON_GLOBAL:
5404                 /*
5405                  * Currently, KVM doesn't mark global entries in the shadow
5406                  * page tables, so a non-global flush just degenerates to a
5407                  * global flush. If needed, we could optimize this later by
5408                  * keeping track of global entries in shadow page tables.
5409                  */
5410
5411                 /* fall-through */
5412         case INVPCID_TYPE_ALL_INCL_GLOBAL:
5413                 kvm_mmu_unload(vcpu);
5414                 return kvm_skip_emulated_instruction(vcpu);
5415
5416         default:
5417                 BUG(); /* We have already checked above that type <= 3 */
5418         }
5419 }
5420
5421 static int handle_pml_full(struct kvm_vcpu *vcpu)
5422 {
5423         unsigned long exit_qualification;
5424
5425         trace_kvm_pml_full(vcpu->vcpu_id);
5426
5427         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5428
5429         /*
5430          * PML buffer FULL happened while executing iret from NMI,
5431          * "blocked by NMI" bit has to be set before next VM entry.
5432          */
5433         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
5434                         enable_vnmi &&
5435                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
5436                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
5437                                 GUEST_INTR_STATE_NMI);
5438
5439         /*
5440          * PML buffer already flushed at beginning of VMEXIT. Nothing to do
5441          * here.., and there's no userspace involvement needed for PML.
5442          */
5443         return 1;
5444 }
5445
5446 static int handle_preemption_timer(struct kvm_vcpu *vcpu)
5447 {
5448         if (!to_vmx(vcpu)->req_immediate_exit)
5449                 kvm_lapic_expired_hv_timer(vcpu);
5450         return 1;
5451 }
5452
5453 /*
5454  * When nested=0, all VMX instruction VM Exits filter here.  The handlers
5455  * are overwritten by nested_vmx_setup() when nested=1.
5456  */
5457 static int handle_vmx_instruction(struct kvm_vcpu *vcpu)
5458 {
5459         kvm_queue_exception(vcpu, UD_VECTOR);
5460         return 1;
5461 }
5462
5463 static int handle_encls(struct kvm_vcpu *vcpu)
5464 {
5465         /*
5466          * SGX virtualization is not yet supported.  There is no software
5467          * enable bit for SGX, so we have to trap ENCLS and inject a #UD
5468          * to prevent the guest from executing ENCLS.
5469          */
5470         kvm_queue_exception(vcpu, UD_VECTOR);
5471         return 1;
5472 }
5473
5474 /*
5475  * The exit handlers return 1 if the exit was handled fully and guest execution
5476  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
5477  * to be done to userspace and return 0.
5478  */
5479 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
5480         [EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
5481         [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
5482         [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
5483         [EXIT_REASON_NMI_WINDOW]              = handle_nmi_window,
5484         [EXIT_REASON_IO_INSTRUCTION]          = handle_io,
5485         [EXIT_REASON_CR_ACCESS]               = handle_cr,
5486         [EXIT_REASON_DR_ACCESS]               = handle_dr,
5487         [EXIT_REASON_CPUID]                   = handle_cpuid,
5488         [EXIT_REASON_MSR_READ]                = handle_rdmsr,
5489         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
5490         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
5491         [EXIT_REASON_HLT]                     = handle_halt,
5492         [EXIT_REASON_INVD]                    = handle_invd,
5493         [EXIT_REASON_INVLPG]                  = handle_invlpg,
5494         [EXIT_REASON_RDPMC]                   = handle_rdpmc,
5495         [EXIT_REASON_VMCALL]                  = handle_vmcall,
5496         [EXIT_REASON_VMCLEAR]                 = handle_vmx_instruction,
5497         [EXIT_REASON_VMLAUNCH]                = handle_vmx_instruction,
5498         [EXIT_REASON_VMPTRLD]                 = handle_vmx_instruction,
5499         [EXIT_REASON_VMPTRST]                 = handle_vmx_instruction,
5500         [EXIT_REASON_VMREAD]                  = handle_vmx_instruction,
5501         [EXIT_REASON_VMRESUME]                = handle_vmx_instruction,
5502         [EXIT_REASON_VMWRITE]                 = handle_vmx_instruction,
5503         [EXIT_REASON_VMOFF]                   = handle_vmx_instruction,
5504         [EXIT_REASON_VMON]                    = handle_vmx_instruction,
5505         [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
5506         [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
5507         [EXIT_REASON_APIC_WRITE]              = handle_apic_write,
5508         [EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,
5509         [EXIT_REASON_WBINVD]                  = handle_wbinvd,
5510         [EXIT_REASON_XSETBV]                  = handle_xsetbv,
5511         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
5512         [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
5513         [EXIT_REASON_GDTR_IDTR]               = handle_desc,
5514         [EXIT_REASON_LDTR_TR]                 = handle_desc,
5515         [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
5516         [EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
5517         [EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
5518         [EXIT_REASON_MWAIT_INSTRUCTION]       = handle_mwait,
5519         [EXIT_REASON_MONITOR_TRAP_FLAG]       = handle_monitor_trap,
5520         [EXIT_REASON_MONITOR_INSTRUCTION]     = handle_monitor,
5521         [EXIT_REASON_INVEPT]                  = handle_vmx_instruction,
5522         [EXIT_REASON_INVVPID]                 = handle_vmx_instruction,
5523         [EXIT_REASON_RDRAND]                  = handle_invalid_op,
5524         [EXIT_REASON_RDSEED]                  = handle_invalid_op,
5525         [EXIT_REASON_XSAVES]                  = handle_xsaves,
5526         [EXIT_REASON_XRSTORS]                 = handle_xrstors,
5527         [EXIT_REASON_PML_FULL]                = handle_pml_full,
5528         [EXIT_REASON_INVPCID]                 = handle_invpcid,
5529         [EXIT_REASON_VMFUNC]                  = handle_vmx_instruction,
5530         [EXIT_REASON_PREEMPTION_TIMER]        = handle_preemption_timer,
5531         [EXIT_REASON_ENCLS]                   = handle_encls,
5532 };
5533
5534 static const int kvm_vmx_max_exit_handlers =
5535         ARRAY_SIZE(kvm_vmx_exit_handlers);
5536
5537 static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
5538 {
5539         *info1 = vmcs_readl(EXIT_QUALIFICATION);
5540         *info2 = vmcs_read32(VM_EXIT_INTR_INFO);
5541 }
5542
5543 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
5544 {
5545         if (vmx->pml_pg) {
5546                 __free_page(vmx->pml_pg);
5547                 vmx->pml_pg = NULL;
5548         }
5549 }
5550
5551 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
5552 {
5553         struct vcpu_vmx *vmx = to_vmx(vcpu);
5554         u64 *pml_buf;
5555         u16 pml_idx;
5556
5557         pml_idx = vmcs_read16(GUEST_PML_INDEX);
5558
5559         /* Do nothing if PML buffer is empty */
5560         if (pml_idx == (PML_ENTITY_NUM - 1))
5561                 return;
5562
5563         /* PML index always points to next available PML buffer entity */
5564         if (pml_idx >= PML_ENTITY_NUM)
5565                 pml_idx = 0;
5566         else
5567                 pml_idx++;
5568
5569         pml_buf = page_address(vmx->pml_pg);
5570         for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
5571                 u64 gpa;
5572
5573                 gpa = pml_buf[pml_idx];
5574                 WARN_ON(gpa & (PAGE_SIZE - 1));
5575                 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
5576         }
5577
5578         /* reset PML index */
5579         vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
5580 }
5581
5582 /*
5583  * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
5584  * Called before reporting dirty_bitmap to userspace.
5585  */
5586 static void kvm_flush_pml_buffers(struct kvm *kvm)
5587 {
5588         int i;
5589         struct kvm_vcpu *vcpu;
5590         /*
5591          * We only need to kick vcpu out of guest mode here, as PML buffer
5592          * is flushed at beginning of all VMEXITs, and it's obvious that only
5593          * vcpus running in guest are possible to have unflushed GPAs in PML
5594          * buffer.
5595          */
5596         kvm_for_each_vcpu(i, vcpu, kvm)
5597                 kvm_vcpu_kick(vcpu);
5598 }
5599
5600 static void vmx_dump_sel(char *name, uint32_t sel)
5601 {
5602         pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
5603                name, vmcs_read16(sel),
5604                vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
5605                vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
5606                vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
5607 }
5608
5609 static void vmx_dump_dtsel(char *name, uint32_t limit)
5610 {
5611         pr_err("%s                           limit=0x%08x, base=0x%016lx\n",
5612                name, vmcs_read32(limit),
5613                vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
5614 }
5615
5616 static void dump_vmcs(void)
5617 {
5618         u32 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
5619         u32 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
5620         u32 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
5621         u32 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
5622         u32 secondary_exec_control = 0;
5623         unsigned long cr4 = vmcs_readl(GUEST_CR4);
5624         u64 efer = vmcs_read64(GUEST_IA32_EFER);
5625         int i, n;
5626
5627         if (cpu_has_secondary_exec_ctrls())
5628                 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
5629
5630         pr_err("*** Guest State ***\n");
5631         pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
5632                vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
5633                vmcs_readl(CR0_GUEST_HOST_MASK));
5634         pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
5635                cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
5636         pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
5637         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
5638             (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA))
5639         {
5640                 pr_err("PDPTR0 = 0x%016llx  PDPTR1 = 0x%016llx\n",
5641                        vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
5642                 pr_err("PDPTR2 = 0x%016llx  PDPTR3 = 0x%016llx\n",
5643                        vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
5644         }
5645         pr_err("RSP = 0x%016lx  RIP = 0x%016lx\n",
5646                vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
5647         pr_err("RFLAGS=0x%08lx         DR7 = 0x%016lx\n",
5648                vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
5649         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
5650                vmcs_readl(GUEST_SYSENTER_ESP),
5651                vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
5652         vmx_dump_sel("CS:  ", GUEST_CS_SELECTOR);
5653         vmx_dump_sel("DS:  ", GUEST_DS_SELECTOR);
5654         vmx_dump_sel("SS:  ", GUEST_SS_SELECTOR);
5655         vmx_dump_sel("ES:  ", GUEST_ES_SELECTOR);
5656         vmx_dump_sel("FS:  ", GUEST_FS_SELECTOR);
5657         vmx_dump_sel("GS:  ", GUEST_GS_SELECTOR);
5658         vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
5659         vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
5660         vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
5661         vmx_dump_sel("TR:  ", GUEST_TR_SELECTOR);
5662         if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
5663             (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
5664                 pr_err("EFER =     0x%016llx  PAT = 0x%016llx\n",
5665                        efer, vmcs_read64(GUEST_IA32_PAT));
5666         pr_err("DebugCtl = 0x%016llx  DebugExceptions = 0x%016lx\n",
5667                vmcs_read64(GUEST_IA32_DEBUGCTL),
5668                vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
5669         if (cpu_has_load_perf_global_ctrl() &&
5670             vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
5671                 pr_err("PerfGlobCtl = 0x%016llx\n",
5672                        vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
5673         if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
5674                 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
5675         pr_err("Interruptibility = %08x  ActivityState = %08x\n",
5676                vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
5677                vmcs_read32(GUEST_ACTIVITY_STATE));
5678         if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
5679                 pr_err("InterruptStatus = %04x\n",
5680                        vmcs_read16(GUEST_INTR_STATUS));
5681
5682         pr_err("*** Host State ***\n");
5683         pr_err("RIP = 0x%016lx  RSP = 0x%016lx\n",
5684                vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
5685         pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
5686                vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
5687                vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
5688                vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
5689                vmcs_read16(HOST_TR_SELECTOR));
5690         pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
5691                vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
5692                vmcs_readl(HOST_TR_BASE));
5693         pr_err("GDTBase=%016lx IDTBase=%016lx\n",
5694                vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
5695         pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
5696                vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
5697                vmcs_readl(HOST_CR4));
5698         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
5699                vmcs_readl(HOST_IA32_SYSENTER_ESP),
5700                vmcs_read32(HOST_IA32_SYSENTER_CS),
5701                vmcs_readl(HOST_IA32_SYSENTER_EIP));
5702         if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
5703                 pr_err("EFER = 0x%016llx  PAT = 0x%016llx\n",
5704                        vmcs_read64(HOST_IA32_EFER),
5705                        vmcs_read64(HOST_IA32_PAT));
5706         if (cpu_has_load_perf_global_ctrl() &&
5707             vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
5708                 pr_err("PerfGlobCtl = 0x%016llx\n",
5709                        vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
5710
5711         pr_err("*** Control State ***\n");
5712         pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
5713                pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
5714         pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
5715         pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
5716                vmcs_read32(EXCEPTION_BITMAP),
5717                vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
5718                vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
5719         pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
5720                vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
5721                vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
5722                vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
5723         pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
5724                vmcs_read32(VM_EXIT_INTR_INFO),
5725                vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
5726                vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
5727         pr_err("        reason=%08x qualification=%016lx\n",
5728                vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
5729         pr_err("IDTVectoring: info=%08x errcode=%08x\n",
5730                vmcs_read32(IDT_VECTORING_INFO_FIELD),
5731                vmcs_read32(IDT_VECTORING_ERROR_CODE));
5732         pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
5733         if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
5734                 pr_err("TSC Multiplier = 0x%016llx\n",
5735                        vmcs_read64(TSC_MULTIPLIER));
5736         if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW)
5737                 pr_err("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
5738         if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
5739                 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
5740         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
5741                 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
5742         n = vmcs_read32(CR3_TARGET_COUNT);
5743         for (i = 0; i + 1 < n; i += 4)
5744                 pr_err("CR3 target%u=%016lx target%u=%016lx\n",
5745                        i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2),
5746                        i + 1, vmcs_readl(CR3_TARGET_VALUE0 + i * 2 + 2));
5747         if (i < n)
5748                 pr_err("CR3 target%u=%016lx\n",
5749                        i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2));
5750         if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
5751                 pr_err("PLE Gap=%08x Window=%08x\n",
5752                        vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
5753         if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
5754                 pr_err("Virtual processor ID = 0x%04x\n",
5755                        vmcs_read16(VIRTUAL_PROCESSOR_ID));
5756 }
5757
5758 /*
5759  * The guest has exited.  See if we can fix it or if we need userspace
5760  * assistance.
5761  */
5762 static int vmx_handle_exit(struct kvm_vcpu *vcpu)
5763 {
5764         struct vcpu_vmx *vmx = to_vmx(vcpu);
5765         u32 exit_reason = vmx->exit_reason;
5766         u32 vectoring_info = vmx->idt_vectoring_info;
5767
5768         trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
5769
5770         /*
5771          * Flush logged GPAs PML buffer, this will make dirty_bitmap more
5772          * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
5773          * querying dirty_bitmap, we only need to kick all vcpus out of guest
5774          * mode as if vcpus is in root mode, the PML buffer must has been
5775          * flushed already.
5776          */
5777         if (enable_pml)
5778                 vmx_flush_pml_buffer(vcpu);
5779
5780         /* If guest state is invalid, start emulating */
5781         if (vmx->emulation_required)
5782                 return handle_invalid_guest_state(vcpu);
5783
5784         if (is_guest_mode(vcpu) && nested_vmx_exit_reflected(vcpu, exit_reason))
5785                 return nested_vmx_reflect_vmexit(vcpu, exit_reason);
5786
5787         if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
5788                 dump_vmcs();
5789                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
5790                 vcpu->run->fail_entry.hardware_entry_failure_reason
5791                         = exit_reason;
5792                 return 0;
5793         }
5794
5795         if (unlikely(vmx->fail)) {
5796                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
5797                 vcpu->run->fail_entry.hardware_entry_failure_reason
5798                         = vmcs_read32(VM_INSTRUCTION_ERROR);
5799                 return 0;
5800         }
5801
5802         /*
5803          * Note:
5804          * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
5805          * delivery event since it indicates guest is accessing MMIO.
5806          * The vm-exit can be triggered again after return to guest that
5807          * will cause infinite loop.
5808          */
5809         if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
5810                         (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
5811                         exit_reason != EXIT_REASON_EPT_VIOLATION &&
5812                         exit_reason != EXIT_REASON_PML_FULL &&
5813                         exit_reason != EXIT_REASON_TASK_SWITCH)) {
5814                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5815                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
5816                 vcpu->run->internal.ndata = 3;
5817                 vcpu->run->internal.data[0] = vectoring_info;
5818                 vcpu->run->internal.data[1] = exit_reason;
5819                 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
5820                 if (exit_reason == EXIT_REASON_EPT_MISCONFIG) {
5821                         vcpu->run->internal.ndata++;
5822                         vcpu->run->internal.data[3] =
5823                                 vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5824                 }
5825                 return 0;
5826         }
5827
5828         if (unlikely(!enable_vnmi &&
5829                      vmx->loaded_vmcs->soft_vnmi_blocked)) {
5830                 if (vmx_interrupt_allowed(vcpu)) {
5831                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
5832                 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL &&
5833                            vcpu->arch.nmi_pending) {
5834                         /*
5835                          * This CPU don't support us in finding the end of an
5836                          * NMI-blocked window if the guest runs with IRQs
5837                          * disabled. So we pull the trigger after 1 s of
5838                          * futile waiting, but inform the user about this.
5839                          */
5840                         printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
5841                                "state on VCPU %d after 1 s timeout\n",
5842                                __func__, vcpu->vcpu_id);
5843                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
5844                 }
5845         }
5846
5847         if (exit_reason < kvm_vmx_max_exit_handlers
5848             && kvm_vmx_exit_handlers[exit_reason])
5849                 return kvm_vmx_exit_handlers[exit_reason](vcpu);
5850         else {
5851                 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
5852                                 exit_reason);
5853                 kvm_queue_exception(vcpu, UD_VECTOR);
5854                 return 1;
5855         }
5856 }
5857
5858 /*
5859  * Software based L1D cache flush which is used when microcode providing
5860  * the cache control MSR is not loaded.
5861  *
5862  * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to
5863  * flush it is required to read in 64 KiB because the replacement algorithm
5864  * is not exactly LRU. This could be sized at runtime via topology
5865  * information but as all relevant affected CPUs have 32KiB L1D cache size
5866  * there is no point in doing so.
5867  */
5868 static void vmx_l1d_flush(struct kvm_vcpu *vcpu)
5869 {
5870         int size = PAGE_SIZE << L1D_CACHE_ORDER;
5871
5872         /*
5873          * This code is only executed when the the flush mode is 'cond' or
5874          * 'always'
5875          */
5876         if (static_branch_likely(&vmx_l1d_flush_cond)) {
5877                 bool flush_l1d;
5878
5879                 /*
5880                  * Clear the per-vcpu flush bit, it gets set again
5881                  * either from vcpu_run() or from one of the unsafe
5882                  * VMEXIT handlers.
5883                  */
5884                 flush_l1d = vcpu->arch.l1tf_flush_l1d;
5885                 vcpu->arch.l1tf_flush_l1d = false;
5886
5887                 /*
5888                  * Clear the per-cpu flush bit, it gets set again from
5889                  * the interrupt handlers.
5890                  */
5891                 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d();
5892                 kvm_clear_cpu_l1tf_flush_l1d();
5893
5894                 if (!flush_l1d)
5895                         return;
5896         }
5897
5898         vcpu->stat.l1d_flush++;
5899
5900         if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
5901                 wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
5902                 return;
5903         }
5904
5905         asm volatile(
5906                 /* First ensure the pages are in the TLB */
5907                 "xorl   %%eax, %%eax\n"
5908                 ".Lpopulate_tlb:\n\t"
5909                 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
5910                 "addl   $4096, %%eax\n\t"
5911                 "cmpl   %%eax, %[size]\n\t"
5912                 "jne    .Lpopulate_tlb\n\t"
5913                 "xorl   %%eax, %%eax\n\t"
5914                 "cpuid\n\t"
5915                 /* Now fill the cache */
5916                 "xorl   %%eax, %%eax\n"
5917                 ".Lfill_cache:\n"
5918                 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
5919                 "addl   $64, %%eax\n\t"
5920                 "cmpl   %%eax, %[size]\n\t"
5921                 "jne    .Lfill_cache\n\t"
5922                 "lfence\n"
5923                 :: [flush_pages] "r" (vmx_l1d_flush_pages),
5924                     [size] "r" (size)
5925                 : "eax", "ebx", "ecx", "edx");
5926 }
5927
5928 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
5929 {
5930         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5931
5932         if (is_guest_mode(vcpu) &&
5933                 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
5934                 return;
5935
5936         if (irr == -1 || tpr < irr) {
5937                 vmcs_write32(TPR_THRESHOLD, 0);
5938                 return;
5939         }
5940
5941         vmcs_write32(TPR_THRESHOLD, irr);
5942 }
5943
5944 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
5945 {
5946         u32 sec_exec_control;
5947
5948         if (!lapic_in_kernel(vcpu))
5949                 return;
5950
5951         if (!flexpriority_enabled &&
5952             !cpu_has_vmx_virtualize_x2apic_mode())
5953                 return;
5954
5955         /* Postpone execution until vmcs01 is the current VMCS. */
5956         if (is_guest_mode(vcpu)) {
5957                 to_vmx(vcpu)->nested.change_vmcs01_virtual_apic_mode = true;
5958                 return;
5959         }
5960
5961         sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
5962         sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
5963                               SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
5964
5965         switch (kvm_get_apic_mode(vcpu)) {
5966         case LAPIC_MODE_INVALID:
5967                 WARN_ONCE(true, "Invalid local APIC state");
5968         case LAPIC_MODE_DISABLED:
5969                 break;
5970         case LAPIC_MODE_XAPIC:
5971                 if (flexpriority_enabled) {
5972                         sec_exec_control |=
5973                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
5974                         vmx_flush_tlb(vcpu, true);
5975                 }
5976                 break;
5977         case LAPIC_MODE_X2APIC:
5978                 if (cpu_has_vmx_virtualize_x2apic_mode())
5979                         sec_exec_control |=
5980                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
5981                 break;
5982         }
5983         vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
5984
5985         vmx_update_msr_bitmap(vcpu);
5986 }
5987
5988 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
5989 {
5990         if (!is_guest_mode(vcpu)) {
5991                 vmcs_write64(APIC_ACCESS_ADDR, hpa);
5992                 vmx_flush_tlb(vcpu, true);
5993         }
5994 }
5995
5996 static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
5997 {
5998         u16 status;
5999         u8 old;
6000
6001         if (max_isr == -1)
6002                 max_isr = 0;
6003
6004         status = vmcs_read16(GUEST_INTR_STATUS);
6005         old = status >> 8;
6006         if (max_isr != old) {
6007                 status &= 0xff;
6008                 status |= max_isr << 8;
6009                 vmcs_write16(GUEST_INTR_STATUS, status);
6010         }
6011 }
6012
6013 static void vmx_set_rvi(int vector)
6014 {
6015         u16 status;
6016         u8 old;
6017
6018         if (vector == -1)
6019                 vector = 0;
6020
6021         status = vmcs_read16(GUEST_INTR_STATUS);
6022         old = (u8)status & 0xff;
6023         if ((u8)vector != old) {
6024                 status &= ~0xff;
6025                 status |= (u8)vector;
6026                 vmcs_write16(GUEST_INTR_STATUS, status);
6027         }
6028 }
6029
6030 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
6031 {
6032         /*
6033          * When running L2, updating RVI is only relevant when
6034          * vmcs12 virtual-interrupt-delivery enabled.
6035          * However, it can be enabled only when L1 also
6036          * intercepts external-interrupts and in that case
6037          * we should not update vmcs02 RVI but instead intercept
6038          * interrupt. Therefore, do nothing when running L2.
6039          */
6040         if (!is_guest_mode(vcpu))
6041                 vmx_set_rvi(max_irr);
6042 }
6043
6044 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
6045 {
6046         struct vcpu_vmx *vmx = to_vmx(vcpu);
6047         int max_irr;
6048         bool max_irr_updated;
6049
6050         WARN_ON(!vcpu->arch.apicv_active);
6051         if (pi_test_on(&vmx->pi_desc)) {
6052                 pi_clear_on(&vmx->pi_desc);
6053                 /*
6054                  * IOMMU can write to PIR.ON, so the barrier matters even on UP.
6055                  * But on x86 this is just a compiler barrier anyway.
6056                  */
6057                 smp_mb__after_atomic();
6058                 max_irr_updated =
6059                         kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr);
6060
6061                 /*
6062                  * If we are running L2 and L1 has a new pending interrupt
6063                  * which can be injected, we should re-evaluate
6064                  * what should be done with this new L1 interrupt.
6065                  * If L1 intercepts external-interrupts, we should
6066                  * exit from L2 to L1. Otherwise, interrupt should be
6067                  * delivered directly to L2.
6068                  */
6069                 if (is_guest_mode(vcpu) && max_irr_updated) {
6070                         if (nested_exit_on_intr(vcpu))
6071                                 kvm_vcpu_exiting_guest_mode(vcpu);
6072                         else
6073                                 kvm_make_request(KVM_REQ_EVENT, vcpu);
6074                 }
6075         } else {
6076                 max_irr = kvm_lapic_find_highest_irr(vcpu);
6077         }
6078         vmx_hwapic_irr_update(vcpu, max_irr);
6079         return max_irr;
6080 }
6081
6082 static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
6083 {
6084         if (!kvm_vcpu_apicv_active(vcpu))
6085                 return;
6086
6087         vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
6088         vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
6089         vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
6090         vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
6091 }
6092
6093 static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu)
6094 {
6095         struct vcpu_vmx *vmx = to_vmx(vcpu);
6096
6097         pi_clear_on(&vmx->pi_desc);
6098         memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir));
6099 }
6100
6101 static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
6102 {
6103         u32 exit_intr_info = 0;
6104         u16 basic_exit_reason = (u16)vmx->exit_reason;
6105
6106         if (!(basic_exit_reason == EXIT_REASON_MCE_DURING_VMENTRY
6107               || basic_exit_reason == EXIT_REASON_EXCEPTION_NMI))
6108                 return;
6109
6110         if (!(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
6111                 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
6112         vmx->exit_intr_info = exit_intr_info;
6113
6114         /* if exit due to PF check for async PF */
6115         if (is_page_fault(exit_intr_info))
6116                 vmx->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
6117
6118         /* Handle machine checks before interrupts are enabled */
6119         if (basic_exit_reason == EXIT_REASON_MCE_DURING_VMENTRY ||
6120             is_machine_check(exit_intr_info))
6121                 kvm_machine_check();
6122
6123         /* We need to handle NMIs before interrupts are enabled */
6124         if (is_nmi(exit_intr_info)) {
6125                 kvm_before_interrupt(&vmx->vcpu);
6126                 asm("int $2");
6127                 kvm_after_interrupt(&vmx->vcpu);
6128         }
6129 }
6130
6131 static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
6132 {
6133         u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
6134
6135         if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK))
6136                         == (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) {
6137                 unsigned int vector;
6138                 unsigned long entry;
6139                 gate_desc *desc;
6140                 struct vcpu_vmx *vmx = to_vmx(vcpu);
6141 #ifdef CONFIG_X86_64
6142                 unsigned long tmp;
6143 #endif
6144
6145                 vector =  exit_intr_info & INTR_INFO_VECTOR_MASK;
6146                 desc = (gate_desc *)vmx->host_idt_base + vector;
6147                 entry = gate_offset(desc);
6148                 asm volatile(
6149 #ifdef CONFIG_X86_64
6150                         "mov %%" _ASM_SP ", %[sp]\n\t"
6151                         "and $0xfffffffffffffff0, %%" _ASM_SP "\n\t"
6152                         "push $%c[ss]\n\t"
6153                         "push %[sp]\n\t"
6154 #endif
6155                         "pushf\n\t"
6156                         __ASM_SIZE(push) " $%c[cs]\n\t"
6157                         CALL_NOSPEC
6158                         :
6159 #ifdef CONFIG_X86_64
6160                         [sp]"=&r"(tmp),
6161 #endif
6162                         ASM_CALL_CONSTRAINT
6163                         :
6164                         THUNK_TARGET(entry),
6165                         [ss]"i"(__KERNEL_DS),
6166                         [cs]"i"(__KERNEL_CS)
6167                         );
6168         }
6169 }
6170 STACK_FRAME_NON_STANDARD(vmx_handle_external_intr);
6171
6172 static bool vmx_has_emulated_msr(int index)
6173 {
6174         switch (index) {
6175         case MSR_IA32_SMBASE:
6176                 /*
6177                  * We cannot do SMM unless we can run the guest in big
6178                  * real mode.
6179                  */
6180                 return enable_unrestricted_guest || emulate_invalid_guest_state;
6181         case MSR_AMD64_VIRT_SPEC_CTRL:
6182                 /* This is AMD only.  */
6183                 return false;
6184         default:
6185                 return true;
6186         }
6187 }
6188
6189 static bool vmx_pt_supported(void)
6190 {
6191         return pt_mode == PT_MODE_HOST_GUEST;
6192 }
6193
6194 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
6195 {
6196         u32 exit_intr_info;
6197         bool unblock_nmi;
6198         u8 vector;
6199         bool idtv_info_valid;
6200
6201         idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
6202
6203         if (enable_vnmi) {
6204                 if (vmx->loaded_vmcs->nmi_known_unmasked)
6205                         return;
6206                 /*
6207                  * Can't use vmx->exit_intr_info since we're not sure what
6208                  * the exit reason is.
6209                  */
6210                 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
6211                 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
6212                 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
6213                 /*
6214                  * SDM 3: 27.7.1.2 (September 2008)
6215                  * Re-set bit "block by NMI" before VM entry if vmexit caused by
6216                  * a guest IRET fault.
6217                  * SDM 3: 23.2.2 (September 2008)
6218                  * Bit 12 is undefined in any of the following cases:
6219                  *  If the VM exit sets the valid bit in the IDT-vectoring
6220                  *   information field.
6221                  *  If the VM exit is due to a double fault.
6222                  */
6223                 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
6224                     vector != DF_VECTOR && !idtv_info_valid)
6225                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
6226                                       GUEST_INTR_STATE_NMI);
6227                 else
6228                         vmx->loaded_vmcs->nmi_known_unmasked =
6229                                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
6230                                   & GUEST_INTR_STATE_NMI);
6231         } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked))
6232                 vmx->loaded_vmcs->vnmi_blocked_time +=
6233                         ktime_to_ns(ktime_sub(ktime_get(),
6234                                               vmx->loaded_vmcs->entry_time));
6235 }
6236
6237 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
6238                                       u32 idt_vectoring_info,
6239                                       int instr_len_field,
6240                                       int error_code_field)
6241 {
6242         u8 vector;
6243         int type;
6244         bool idtv_info_valid;
6245
6246         idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
6247
6248         vcpu->arch.nmi_injected = false;
6249         kvm_clear_exception_queue(vcpu);
6250         kvm_clear_interrupt_queue(vcpu);
6251
6252         if (!idtv_info_valid)
6253                 return;
6254
6255         kvm_make_request(KVM_REQ_EVENT, vcpu);
6256
6257         vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
6258         type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
6259
6260         switch (type) {
6261         case INTR_TYPE_NMI_INTR:
6262                 vcpu->arch.nmi_injected = true;
6263                 /*
6264                  * SDM 3: 27.7.1.2 (September 2008)
6265                  * Clear bit "block by NMI" before VM entry if a NMI
6266                  * delivery faulted.
6267                  */
6268                 vmx_set_nmi_mask(vcpu, false);
6269                 break;
6270         case INTR_TYPE_SOFT_EXCEPTION:
6271                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
6272                 /* fall through */
6273         case INTR_TYPE_HARD_EXCEPTION:
6274                 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
6275                         u32 err = vmcs_read32(error_code_field);
6276                         kvm_requeue_exception_e(vcpu, vector, err);
6277                 } else
6278                         kvm_requeue_exception(vcpu, vector);
6279                 break;
6280         case INTR_TYPE_SOFT_INTR:
6281                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
6282                 /* fall through */
6283         case INTR_TYPE_EXT_INTR:
6284                 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
6285                 break;
6286         default:
6287                 break;
6288         }
6289 }
6290
6291 static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
6292 {
6293         __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
6294                                   VM_EXIT_INSTRUCTION_LEN,
6295                                   IDT_VECTORING_ERROR_CODE);
6296 }
6297
6298 static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
6299 {
6300         __vmx_complete_interrupts(vcpu,
6301                                   vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
6302                                   VM_ENTRY_INSTRUCTION_LEN,
6303                                   VM_ENTRY_EXCEPTION_ERROR_CODE);
6304
6305         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
6306 }
6307
6308 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
6309 {
6310         int i, nr_msrs;
6311         struct perf_guest_switch_msr *msrs;
6312
6313         msrs = perf_guest_get_msrs(&nr_msrs);
6314
6315         if (!msrs)
6316                 return;
6317
6318         for (i = 0; i < nr_msrs; i++)
6319                 if (msrs[i].host == msrs[i].guest)
6320                         clear_atomic_switch_msr(vmx, msrs[i].msr);
6321                 else
6322                         add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
6323                                         msrs[i].host, false);
6324 }
6325
6326 static void vmx_arm_hv_timer(struct vcpu_vmx *vmx, u32 val)
6327 {
6328         vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, val);
6329         if (!vmx->loaded_vmcs->hv_timer_armed)
6330                 vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
6331                               PIN_BASED_VMX_PREEMPTION_TIMER);
6332         vmx->loaded_vmcs->hv_timer_armed = true;
6333 }
6334
6335 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu)
6336 {
6337         struct vcpu_vmx *vmx = to_vmx(vcpu);
6338         u64 tscl;
6339         u32 delta_tsc;
6340
6341         if (vmx->req_immediate_exit) {
6342                 vmx_arm_hv_timer(vmx, 0);
6343                 return;
6344         }
6345
6346         if (vmx->hv_deadline_tsc != -1) {
6347                 tscl = rdtsc();
6348                 if (vmx->hv_deadline_tsc > tscl)
6349                         /* set_hv_timer ensures the delta fits in 32-bits */
6350                         delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
6351                                 cpu_preemption_timer_multi);
6352                 else
6353                         delta_tsc = 0;
6354
6355                 vmx_arm_hv_timer(vmx, delta_tsc);
6356                 return;
6357         }
6358
6359         if (vmx->loaded_vmcs->hv_timer_armed)
6360                 vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
6361                                 PIN_BASED_VMX_PREEMPTION_TIMER);
6362         vmx->loaded_vmcs->hv_timer_armed = false;
6363 }
6364
6365 static void __vmx_vcpu_run(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx)
6366 {
6367         unsigned long evmcs_rsp;
6368
6369         vmx->__launched = vmx->loaded_vmcs->launched;
6370
6371         evmcs_rsp = static_branch_unlikely(&enable_evmcs) ?
6372                 (unsigned long)&current_evmcs->host_rsp : 0;
6373
6374         if (static_branch_unlikely(&vmx_l1d_should_flush))
6375                 vmx_l1d_flush(vcpu);
6376
6377         asm(
6378                 /* Store host registers */
6379                 "push %%" _ASM_DX "; push %%" _ASM_BP ";"
6380                 "push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
6381                 "push %%" _ASM_CX " \n\t"
6382                 "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */
6383                 "cmp %%" _ASM_SP ", %c[host_rsp](%%" _ASM_CX ") \n\t"
6384                 "je 1f \n\t"
6385                 "mov %%" _ASM_SP ", %c[host_rsp](%%" _ASM_CX ") \n\t"
6386                 /* Avoid VMWRITE when Enlightened VMCS is in use */
6387                 "test %%" _ASM_SI ", %%" _ASM_SI " \n\t"
6388                 "jz 2f \n\t"
6389                 "mov %%" _ASM_SP ", (%%" _ASM_SI ") \n\t"
6390                 "jmp 1f \n\t"
6391                 "2: \n\t"
6392                 __ex("vmwrite %%" _ASM_SP ", %%" _ASM_DX) "\n\t"
6393                 "1: \n\t"
6394                 "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */
6395
6396                 /* Reload cr2 if changed */
6397                 "mov %c[cr2](%%" _ASM_CX "), %%" _ASM_AX " \n\t"
6398                 "mov %%cr2, %%" _ASM_DX " \n\t"
6399                 "cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
6400                 "je 3f \n\t"
6401                 "mov %%" _ASM_AX", %%cr2 \n\t"
6402                 "3: \n\t"
6403                 /* Check if vmlaunch or vmresume is needed */
6404                 "cmpl $0, %c[launched](%%" _ASM_CX ") \n\t"
6405                 /* Load guest registers.  Don't clobber flags. */
6406                 "mov %c[rax](%%" _ASM_CX "), %%" _ASM_AX " \n\t"
6407                 "mov %c[rbx](%%" _ASM_CX "), %%" _ASM_BX " \n\t"
6408                 "mov %c[rdx](%%" _ASM_CX "), %%" _ASM_DX " \n\t"
6409                 "mov %c[rsi](%%" _ASM_CX "), %%" _ASM_SI " \n\t"
6410                 "mov %c[rdi](%%" _ASM_CX "), %%" _ASM_DI " \n\t"
6411                 "mov %c[rbp](%%" _ASM_CX "), %%" _ASM_BP " \n\t"
6412 #ifdef CONFIG_X86_64
6413                 "mov %c[r8](%%" _ASM_CX "),  %%r8  \n\t"
6414                 "mov %c[r9](%%" _ASM_CX "),  %%r9  \n\t"
6415                 "mov %c[r10](%%" _ASM_CX "), %%r10 \n\t"
6416                 "mov %c[r11](%%" _ASM_CX "), %%r11 \n\t"
6417                 "mov %c[r12](%%" _ASM_CX "), %%r12 \n\t"
6418                 "mov %c[r13](%%" _ASM_CX "), %%r13 \n\t"
6419                 "mov %c[r14](%%" _ASM_CX "), %%r14 \n\t"
6420                 "mov %c[r15](%%" _ASM_CX "), %%r15 \n\t"
6421 #endif
6422                 /* Load guest RCX.  This kills the vmx_vcpu pointer! */
6423                 "mov %c[rcx](%%" _ASM_CX "), %%" _ASM_CX " \n\t"
6424
6425                 /* Enter guest mode */
6426                 "call vmx_vmenter\n\t"
6427
6428                 /* Save guest's RCX to the stack placeholder (see above) */
6429                 "mov %%" _ASM_CX ", %c[wordsize](%%" _ASM_SP ") \n\t"
6430
6431                 /* Load host's RCX, i.e. the vmx_vcpu pointer */
6432                 "pop %%" _ASM_CX " \n\t"
6433
6434                 /* Set vmx->fail based on EFLAGS.{CF,ZF} */
6435                 "setbe %c[fail](%%" _ASM_CX ")\n\t"
6436
6437                 /* Save all guest registers, including RCX from the stack */
6438                 "mov %%" _ASM_AX ", %c[rax](%%" _ASM_CX ") \n\t"
6439                 "mov %%" _ASM_BX ", %c[rbx](%%" _ASM_CX ") \n\t"
6440                 __ASM_SIZE(pop) " %c[rcx](%%" _ASM_CX ") \n\t"
6441                 "mov %%" _ASM_DX ", %c[rdx](%%" _ASM_CX ") \n\t"
6442                 "mov %%" _ASM_SI ", %c[rsi](%%" _ASM_CX ") \n\t"
6443                 "mov %%" _ASM_DI ", %c[rdi](%%" _ASM_CX ") \n\t"
6444                 "mov %%" _ASM_BP ", %c[rbp](%%" _ASM_CX ") \n\t"
6445 #ifdef CONFIG_X86_64
6446                 "mov %%r8,  %c[r8](%%" _ASM_CX ") \n\t"
6447                 "mov %%r9,  %c[r9](%%" _ASM_CX ") \n\t"
6448                 "mov %%r10, %c[r10](%%" _ASM_CX ") \n\t"
6449                 "mov %%r11, %c[r11](%%" _ASM_CX ") \n\t"
6450                 "mov %%r12, %c[r12](%%" _ASM_CX ") \n\t"
6451                 "mov %%r13, %c[r13](%%" _ASM_CX ") \n\t"
6452                 "mov %%r14, %c[r14](%%" _ASM_CX ") \n\t"
6453                 "mov %%r15, %c[r15](%%" _ASM_CX ") \n\t"
6454                 /*
6455                 * Clear host registers marked as clobbered to prevent
6456                 * speculative use.
6457                 */
6458                 "xor %%r8d,  %%r8d \n\t"
6459                 "xor %%r9d,  %%r9d \n\t"
6460                 "xor %%r10d, %%r10d \n\t"
6461                 "xor %%r11d, %%r11d \n\t"
6462                 "xor %%r12d, %%r12d \n\t"
6463                 "xor %%r13d, %%r13d \n\t"
6464                 "xor %%r14d, %%r14d \n\t"
6465                 "xor %%r15d, %%r15d \n\t"
6466 #endif
6467                 "mov %%cr2, %%" _ASM_AX "   \n\t"
6468                 "mov %%" _ASM_AX ", %c[cr2](%%" _ASM_CX ") \n\t"
6469
6470                 "xor %%eax, %%eax \n\t"
6471                 "xor %%ebx, %%ebx \n\t"
6472                 "xor %%esi, %%esi \n\t"
6473                 "xor %%edi, %%edi \n\t"
6474                 "pop  %%" _ASM_BP "; pop  %%" _ASM_DX " \n\t"
6475               : ASM_CALL_CONSTRAINT
6476               : "c"(vmx), "d"((unsigned long)HOST_RSP), "S"(evmcs_rsp),
6477                 [launched]"i"(offsetof(struct vcpu_vmx, __launched)),
6478                 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
6479                 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
6480                 [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
6481                 [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
6482                 [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
6483                 [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
6484                 [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
6485                 [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
6486                 [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
6487 #ifdef CONFIG_X86_64
6488                 [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
6489                 [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
6490                 [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
6491                 [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
6492                 [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
6493                 [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
6494                 [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
6495                 [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
6496 #endif
6497                 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
6498                 [wordsize]"i"(sizeof(ulong))
6499               : "cc", "memory"
6500 #ifdef CONFIG_X86_64
6501                 , "rax", "rbx", "rdi"
6502                 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
6503 #else
6504                 , "eax", "ebx", "edi"
6505 #endif
6506               );
6507 }
6508 STACK_FRAME_NON_STANDARD(__vmx_vcpu_run);
6509
6510 static void vmx_vcpu_run(struct kvm_vcpu *vcpu)
6511 {
6512         struct vcpu_vmx *vmx = to_vmx(vcpu);
6513         unsigned long cr3, cr4;
6514
6515         /* Record the guest's net vcpu time for enforced NMI injections. */
6516         if (unlikely(!enable_vnmi &&
6517                      vmx->loaded_vmcs->soft_vnmi_blocked))
6518                 vmx->loaded_vmcs->entry_time = ktime_get();
6519
6520         /* Don't enter VMX if guest state is invalid, let the exit handler
6521            start emulation until we arrive back to a valid state */
6522         if (vmx->emulation_required)
6523                 return;
6524
6525         if (vmx->ple_window_dirty) {
6526                 vmx->ple_window_dirty = false;
6527                 vmcs_write32(PLE_WINDOW, vmx->ple_window);
6528         }
6529
6530         if (vmx->nested.need_vmcs12_sync)
6531                 nested_sync_from_vmcs12(vcpu);
6532
6533         if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
6534                 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
6535         if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
6536                 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
6537
6538         cr3 = __get_current_cr3_fast();
6539         if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
6540                 vmcs_writel(HOST_CR3, cr3);
6541                 vmx->loaded_vmcs->host_state.cr3 = cr3;
6542         }
6543
6544         cr4 = cr4_read_shadow();
6545         if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
6546                 vmcs_writel(HOST_CR4, cr4);
6547                 vmx->loaded_vmcs->host_state.cr4 = cr4;
6548         }
6549
6550         /* When single-stepping over STI and MOV SS, we must clear the
6551          * corresponding interruptibility bits in the guest state. Otherwise
6552          * vmentry fails as it then expects bit 14 (BS) in pending debug
6553          * exceptions being set, but that's not correct for the guest debugging
6554          * case. */
6555         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
6556                 vmx_set_interrupt_shadow(vcpu, 0);
6557
6558         if (static_cpu_has(X86_FEATURE_PKU) &&
6559             kvm_read_cr4_bits(vcpu, X86_CR4_PKE) &&
6560             vcpu->arch.pkru != vmx->host_pkru)
6561                 __write_pkru(vcpu->arch.pkru);
6562
6563         pt_guest_enter(vmx);
6564
6565         atomic_switch_perf_msrs(vmx);
6566
6567         vmx_update_hv_timer(vcpu);
6568
6569         /*
6570          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
6571          * it's non-zero. Since vmentry is serialising on affected CPUs, there
6572          * is no need to worry about the conditional branch over the wrmsr
6573          * being speculatively taken.
6574          */
6575         x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0);
6576
6577         __vmx_vcpu_run(vcpu, vmx);
6578
6579         /*
6580          * We do not use IBRS in the kernel. If this vCPU has used the
6581          * SPEC_CTRL MSR it may have left it on; save the value and
6582          * turn it off. This is much more efficient than blindly adding
6583          * it to the atomic save/restore list. Especially as the former
6584          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
6585          *
6586          * For non-nested case:
6587          * If the L01 MSR bitmap does not intercept the MSR, then we need to
6588          * save it.
6589          *
6590          * For nested case:
6591          * If the L02 MSR bitmap does not intercept the MSR, then we need to
6592          * save it.
6593          */
6594         if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
6595                 vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
6596
6597         x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
6598
6599         /* Eliminate branch target predictions from guest mode */
6600         vmexit_fill_RSB();
6601
6602         /* All fields are clean at this point */
6603         if (static_branch_unlikely(&enable_evmcs))
6604                 current_evmcs->hv_clean_fields |=
6605                         HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
6606
6607         /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
6608         if (vmx->host_debugctlmsr)
6609                 update_debugctlmsr(vmx->host_debugctlmsr);
6610
6611 #ifndef CONFIG_X86_64
6612         /*
6613          * The sysexit path does not restore ds/es, so we must set them to
6614          * a reasonable value ourselves.
6615          *
6616          * We can't defer this to vmx_prepare_switch_to_host() since that
6617          * function may be executed in interrupt context, which saves and
6618          * restore segments around it, nullifying its effect.
6619          */
6620         loadsegment(ds, __USER_DS);
6621         loadsegment(es, __USER_DS);
6622 #endif
6623
6624         vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
6625                                   | (1 << VCPU_EXREG_RFLAGS)
6626                                   | (1 << VCPU_EXREG_PDPTR)
6627                                   | (1 << VCPU_EXREG_SEGMENTS)
6628                                   | (1 << VCPU_EXREG_CR3));
6629         vcpu->arch.regs_dirty = 0;
6630
6631         pt_guest_exit(vmx);
6632
6633         /*
6634          * eager fpu is enabled if PKEY is supported and CR4 is switched
6635          * back on host, so it is safe to read guest PKRU from current
6636          * XSAVE.
6637          */
6638         if (static_cpu_has(X86_FEATURE_PKU) &&
6639             kvm_read_cr4_bits(vcpu, X86_CR4_PKE)) {
6640                 vcpu->arch.pkru = __read_pkru();
6641                 if (vcpu->arch.pkru != vmx->host_pkru)
6642                         __write_pkru(vmx->host_pkru);
6643         }
6644
6645         vmx->nested.nested_run_pending = 0;
6646         vmx->idt_vectoring_info = 0;
6647
6648         vmx->exit_reason = vmx->fail ? 0xdead : vmcs_read32(VM_EXIT_REASON);
6649         if (vmx->fail || (vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
6650                 return;
6651
6652         vmx->loaded_vmcs->launched = 1;
6653         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
6654
6655         vmx_complete_atomic_exit(vmx);
6656         vmx_recover_nmi_blocking(vmx);
6657         vmx_complete_interrupts(vmx);
6658 }
6659
6660 static struct kvm *vmx_vm_alloc(void)
6661 {
6662         struct kvm_vmx *kvm_vmx = vzalloc(sizeof(struct kvm_vmx));
6663         return &kvm_vmx->kvm;
6664 }
6665
6666 static void vmx_vm_free(struct kvm *kvm)
6667 {
6668         vfree(to_kvm_vmx(kvm));
6669 }
6670
6671 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
6672 {
6673         struct vcpu_vmx *vmx = to_vmx(vcpu);
6674
6675         if (enable_pml)
6676                 vmx_destroy_pml_buffer(vmx);
6677         free_vpid(vmx->vpid);
6678         leave_guest_mode(vcpu);
6679         nested_vmx_free_vcpu(vcpu);
6680         free_loaded_vmcs(vmx->loaded_vmcs);
6681         kfree(vmx->guest_msrs);
6682         kvm_vcpu_uninit(vcpu);
6683         kmem_cache_free(x86_fpu_cache, vmx->vcpu.arch.guest_fpu);
6684         kmem_cache_free(kvm_vcpu_cache, vmx);
6685 }
6686
6687 static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
6688 {
6689         int err;
6690         struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
6691         unsigned long *msr_bitmap;
6692         int cpu;
6693
6694         if (!vmx)
6695                 return ERR_PTR(-ENOMEM);
6696
6697         vmx->vcpu.arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache, GFP_KERNEL);
6698         if (!vmx->vcpu.arch.guest_fpu) {
6699                 printk(KERN_ERR "kvm: failed to allocate vcpu's fpu\n");
6700                 err = -ENOMEM;
6701                 goto free_partial_vcpu;
6702         }
6703
6704         vmx->vpid = allocate_vpid();
6705
6706         err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
6707         if (err)
6708                 goto free_vcpu;
6709
6710         err = -ENOMEM;
6711
6712         /*
6713          * If PML is turned on, failure on enabling PML just results in failure
6714          * of creating the vcpu, therefore we can simplify PML logic (by
6715          * avoiding dealing with cases, such as enabling PML partially on vcpus
6716          * for the guest, etc.
6717          */
6718         if (enable_pml) {
6719                 vmx->pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO);
6720                 if (!vmx->pml_pg)
6721                         goto uninit_vcpu;
6722         }
6723
6724         vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
6725         BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0])
6726                      > PAGE_SIZE);
6727
6728         if (!vmx->guest_msrs)
6729                 goto free_pml;
6730
6731         err = alloc_loaded_vmcs(&vmx->vmcs01);
6732         if (err < 0)
6733                 goto free_msrs;
6734
6735         msr_bitmap = vmx->vmcs01.msr_bitmap;
6736         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_TSC, MSR_TYPE_R);
6737         vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE, MSR_TYPE_RW);
6738         vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE, MSR_TYPE_RW);
6739         vmx_disable_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
6740         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
6741         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW);
6742         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW);
6743         vmx->msr_bitmap_mode = 0;
6744
6745         vmx->loaded_vmcs = &vmx->vmcs01;
6746         cpu = get_cpu();
6747         vmx_vcpu_load(&vmx->vcpu, cpu);
6748         vmx->vcpu.cpu = cpu;
6749         vmx_vcpu_setup(vmx);
6750         vmx_vcpu_put(&vmx->vcpu);
6751         put_cpu();
6752         if (cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
6753                 err = alloc_apic_access_page(kvm);
6754                 if (err)
6755                         goto free_vmcs;
6756         }
6757
6758         if (enable_ept && !enable_unrestricted_guest) {
6759                 err = init_rmode_identity_map(kvm);
6760                 if (err)
6761                         goto free_vmcs;
6762         }
6763
6764         if (nested)
6765                 nested_vmx_setup_ctls_msrs(&vmx->nested.msrs,
6766                                            vmx_capability.ept,
6767                                            kvm_vcpu_apicv_active(&vmx->vcpu));
6768         else
6769                 memset(&vmx->nested.msrs, 0, sizeof(vmx->nested.msrs));
6770
6771         vmx->nested.posted_intr_nv = -1;
6772         vmx->nested.current_vmptr = -1ull;
6773
6774         vmx->msr_ia32_feature_control_valid_bits = FEATURE_CONTROL_LOCKED;
6775
6776         /*
6777          * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
6778          * or POSTED_INTR_WAKEUP_VECTOR.
6779          */
6780         vmx->pi_desc.nv = POSTED_INTR_VECTOR;
6781         vmx->pi_desc.sn = 1;
6782
6783         vmx->ept_pointer = INVALID_PAGE;
6784
6785         return &vmx->vcpu;
6786
6787 free_vmcs:
6788         free_loaded_vmcs(vmx->loaded_vmcs);
6789 free_msrs:
6790         kfree(vmx->guest_msrs);
6791 free_pml:
6792         vmx_destroy_pml_buffer(vmx);
6793 uninit_vcpu:
6794         kvm_vcpu_uninit(&vmx->vcpu);
6795 free_vcpu:
6796         free_vpid(vmx->vpid);
6797         kmem_cache_free(x86_fpu_cache, vmx->vcpu.arch.guest_fpu);
6798 free_partial_vcpu:
6799         kmem_cache_free(kvm_vcpu_cache, vmx);
6800         return ERR_PTR(err);
6801 }
6802
6803 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html for details.\n"
6804 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html for details.\n"
6805
6806 static int vmx_vm_init(struct kvm *kvm)
6807 {
6808         spin_lock_init(&to_kvm_vmx(kvm)->ept_pointer_lock);
6809
6810         if (!ple_gap)
6811                 kvm->arch.pause_in_guest = true;
6812
6813         if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) {
6814                 switch (l1tf_mitigation) {
6815                 case L1TF_MITIGATION_OFF:
6816                 case L1TF_MITIGATION_FLUSH_NOWARN:
6817                         /* 'I explicitly don't care' is set */
6818                         break;
6819                 case L1TF_MITIGATION_FLUSH:
6820                 case L1TF_MITIGATION_FLUSH_NOSMT:
6821                 case L1TF_MITIGATION_FULL:
6822                         /*
6823                          * Warn upon starting the first VM in a potentially
6824                          * insecure environment.
6825                          */
6826                         if (cpu_smt_control == CPU_SMT_ENABLED)
6827                                 pr_warn_once(L1TF_MSG_SMT);
6828                         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER)
6829                                 pr_warn_once(L1TF_MSG_L1D);
6830                         break;
6831                 case L1TF_MITIGATION_FULL_FORCE:
6832                         /* Flush is enforced */
6833                         break;
6834                 }
6835         }
6836         return 0;
6837 }
6838
6839 static void __init vmx_check_processor_compat(void *rtn)
6840 {
6841         struct vmcs_config vmcs_conf;
6842         struct vmx_capability vmx_cap;
6843
6844         *(int *)rtn = 0;
6845         if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0)
6846                 *(int *)rtn = -EIO;
6847         if (nested)
6848                 nested_vmx_setup_ctls_msrs(&vmcs_conf.nested, vmx_cap.ept,
6849                                            enable_apicv);
6850         if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
6851                 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
6852                                 smp_processor_id());
6853                 *(int *)rtn = -EIO;
6854         }
6855 }
6856
6857 static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
6858 {
6859         u8 cache;
6860         u64 ipat = 0;
6861
6862         /* For VT-d and EPT combination
6863          * 1. MMIO: always map as UC
6864          * 2. EPT with VT-d:
6865          *   a. VT-d without snooping control feature: can't guarantee the
6866          *      result, try to trust guest.
6867          *   b. VT-d with snooping control feature: snooping control feature of
6868          *      VT-d engine can guarantee the cache correctness. Just set it
6869          *      to WB to keep consistent with host. So the same as item 3.
6870          * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
6871          *    consistent with host MTRR
6872          */
6873         if (is_mmio) {
6874                 cache = MTRR_TYPE_UNCACHABLE;
6875                 goto exit;
6876         }
6877
6878         if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
6879                 ipat = VMX_EPT_IPAT_BIT;
6880                 cache = MTRR_TYPE_WRBACK;
6881                 goto exit;
6882         }
6883
6884         if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
6885                 ipat = VMX_EPT_IPAT_BIT;
6886                 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
6887                         cache = MTRR_TYPE_WRBACK;
6888                 else
6889                         cache = MTRR_TYPE_UNCACHABLE;
6890                 goto exit;
6891         }
6892
6893         cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
6894
6895 exit:
6896         return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat;
6897 }
6898
6899 static int vmx_get_lpage_level(void)
6900 {
6901         if (enable_ept && !cpu_has_vmx_ept_1g_page())
6902                 return PT_DIRECTORY_LEVEL;
6903         else
6904                 /* For shadow and EPT supported 1GB page */
6905                 return PT_PDPE_LEVEL;
6906 }
6907
6908 static void vmcs_set_secondary_exec_control(u32 new_ctl)
6909 {
6910         /*
6911          * These bits in the secondary execution controls field
6912          * are dynamic, the others are mostly based on the hypervisor
6913          * architecture and the guest's CPUID.  Do not touch the
6914          * dynamic bits.
6915          */
6916         u32 mask =
6917                 SECONDARY_EXEC_SHADOW_VMCS |
6918                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
6919                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
6920                 SECONDARY_EXEC_DESC;
6921
6922         u32 cur_ctl = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
6923
6924         vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
6925                      (new_ctl & ~mask) | (cur_ctl & mask));
6926 }
6927
6928 /*
6929  * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
6930  * (indicating "allowed-1") if they are supported in the guest's CPUID.
6931  */
6932 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
6933 {
6934         struct vcpu_vmx *vmx = to_vmx(vcpu);
6935         struct kvm_cpuid_entry2 *entry;
6936
6937         vmx->nested.msrs.cr0_fixed1 = 0xffffffff;
6938         vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE;
6939
6940 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do {            \
6941         if (entry && (entry->_reg & (_cpuid_mask)))                     \
6942                 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask);     \
6943 } while (0)
6944
6945         entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
6946         cr4_fixed1_update(X86_CR4_VME,        edx, bit(X86_FEATURE_VME));
6947         cr4_fixed1_update(X86_CR4_PVI,        edx, bit(X86_FEATURE_VME));
6948         cr4_fixed1_update(X86_CR4_TSD,        edx, bit(X86_FEATURE_TSC));
6949         cr4_fixed1_update(X86_CR4_DE,         edx, bit(X86_FEATURE_DE));
6950         cr4_fixed1_update(X86_CR4_PSE,        edx, bit(X86_FEATURE_PSE));
6951         cr4_fixed1_update(X86_CR4_PAE,        edx, bit(X86_FEATURE_PAE));
6952         cr4_fixed1_update(X86_CR4_MCE,        edx, bit(X86_FEATURE_MCE));
6953         cr4_fixed1_update(X86_CR4_PGE,        edx, bit(X86_FEATURE_PGE));
6954         cr4_fixed1_update(X86_CR4_OSFXSR,     edx, bit(X86_FEATURE_FXSR));
6955         cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, bit(X86_FEATURE_XMM));
6956         cr4_fixed1_update(X86_CR4_VMXE,       ecx, bit(X86_FEATURE_VMX));
6957         cr4_fixed1_update(X86_CR4_SMXE,       ecx, bit(X86_FEATURE_SMX));
6958         cr4_fixed1_update(X86_CR4_PCIDE,      ecx, bit(X86_FEATURE_PCID));
6959         cr4_fixed1_update(X86_CR4_OSXSAVE,    ecx, bit(X86_FEATURE_XSAVE));
6960
6961         entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
6962         cr4_fixed1_update(X86_CR4_FSGSBASE,   ebx, bit(X86_FEATURE_FSGSBASE));
6963         cr4_fixed1_update(X86_CR4_SMEP,       ebx, bit(X86_FEATURE_SMEP));
6964         cr4_fixed1_update(X86_CR4_SMAP,       ebx, bit(X86_FEATURE_SMAP));
6965         cr4_fixed1_update(X86_CR4_PKE,        ecx, bit(X86_FEATURE_PKU));
6966         cr4_fixed1_update(X86_CR4_UMIP,       ecx, bit(X86_FEATURE_UMIP));
6967
6968 #undef cr4_fixed1_update
6969 }
6970
6971 static void nested_vmx_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
6972 {
6973         struct vcpu_vmx *vmx = to_vmx(vcpu);
6974
6975         if (kvm_mpx_supported()) {
6976                 bool mpx_enabled = guest_cpuid_has(vcpu, X86_FEATURE_MPX);
6977
6978                 if (mpx_enabled) {
6979                         vmx->nested.msrs.entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
6980                         vmx->nested.msrs.exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
6981                 } else {
6982                         vmx->nested.msrs.entry_ctls_high &= ~VM_ENTRY_LOAD_BNDCFGS;
6983                         vmx->nested.msrs.exit_ctls_high &= ~VM_EXIT_CLEAR_BNDCFGS;
6984                 }
6985         }
6986 }
6987
6988 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu)
6989 {
6990         struct vcpu_vmx *vmx = to_vmx(vcpu);
6991         struct kvm_cpuid_entry2 *best = NULL;
6992         int i;
6993
6994         for (i = 0; i < PT_CPUID_LEAVES; i++) {
6995                 best = kvm_find_cpuid_entry(vcpu, 0x14, i);
6996                 if (!best)
6997                         return;
6998                 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax;
6999                 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx;
7000                 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx;
7001                 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx;
7002         }
7003
7004         /* Get the number of configurable Address Ranges for filtering */
7005         vmx->pt_desc.addr_range = intel_pt_validate_cap(vmx->pt_desc.caps,
7006                                                 PT_CAP_num_address_ranges);
7007
7008         /* Initialize and clear the no dependency bits */
7009         vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS |
7010                         RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC);
7011
7012         /*
7013          * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise
7014          * will inject an #GP
7015          */
7016         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering))
7017                 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN;
7018
7019         /*
7020          * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and
7021          * PSBFreq can be set
7022          */
7023         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc))
7024                 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC |
7025                                 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ);
7026
7027         /*
7028          * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn BranchEn and
7029          * MTCFreq can be set
7030          */
7031         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc))
7032                 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN |
7033                                 RTIT_CTL_BRANCH_EN | RTIT_CTL_MTC_RANGE);
7034
7035         /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */
7036         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite))
7037                 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW |
7038                                                         RTIT_CTL_PTW_EN);
7039
7040         /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */
7041         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace))
7042                 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN;
7043
7044         /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */
7045         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output))
7046                 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA;
7047
7048         /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabircEn can be set */
7049         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys))
7050                 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN;
7051
7052         /* unmask address range configure area */
7053         for (i = 0; i < vmx->pt_desc.addr_range; i++)
7054                 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4));
7055 }
7056
7057 static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
7058 {
7059         struct vcpu_vmx *vmx = to_vmx(vcpu);
7060
7061         if (cpu_has_secondary_exec_ctrls()) {
7062                 vmx_compute_secondary_exec_control(vmx);
7063                 vmcs_set_secondary_exec_control(vmx->secondary_exec_control);
7064         }
7065
7066         if (nested_vmx_allowed(vcpu))
7067                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
7068                         FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
7069         else
7070                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
7071                         ~FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
7072
7073         if (nested_vmx_allowed(vcpu)) {
7074                 nested_vmx_cr_fixed1_bits_update(vcpu);
7075                 nested_vmx_entry_exit_ctls_update(vcpu);
7076         }
7077
7078         if (boot_cpu_has(X86_FEATURE_INTEL_PT) &&
7079                         guest_cpuid_has(vcpu, X86_FEATURE_INTEL_PT))
7080                 update_intel_pt_cfg(vcpu);
7081 }
7082
7083 static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
7084 {
7085         if (func == 1 && nested)
7086                 entry->ecx |= bit(X86_FEATURE_VMX);
7087 }
7088
7089 static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu)
7090 {
7091         to_vmx(vcpu)->req_immediate_exit = true;
7092 }
7093
7094 static int vmx_check_intercept(struct kvm_vcpu *vcpu,
7095                                struct x86_instruction_info *info,
7096                                enum x86_intercept_stage stage)
7097 {
7098         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
7099         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
7100
7101         /*
7102          * RDPID causes #UD if disabled through secondary execution controls.
7103          * Because it is marked as EmulateOnUD, we need to intercept it here.
7104          */
7105         if (info->intercept == x86_intercept_rdtscp &&
7106             !nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDTSCP)) {
7107                 ctxt->exception.vector = UD_VECTOR;
7108                 ctxt->exception.error_code_valid = false;
7109                 return X86EMUL_PROPAGATE_FAULT;
7110         }
7111
7112         /* TODO: check more intercepts... */
7113         return X86EMUL_CONTINUE;
7114 }
7115
7116 #ifdef CONFIG_X86_64
7117 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */
7118 static inline int u64_shl_div_u64(u64 a, unsigned int shift,
7119                                   u64 divisor, u64 *result)
7120 {
7121         u64 low = a << shift, high = a >> (64 - shift);
7122
7123         /* To avoid the overflow on divq */
7124         if (high >= divisor)
7125                 return 1;
7126
7127         /* Low hold the result, high hold rem which is discarded */
7128         asm("divq %2\n\t" : "=a" (low), "=d" (high) :
7129             "rm" (divisor), "0" (low), "1" (high));
7130         *result = low;
7131
7132         return 0;
7133 }
7134
7135 static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc)
7136 {
7137         struct vcpu_vmx *vmx;
7138         u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles;
7139
7140         if (kvm_mwait_in_guest(vcpu->kvm))
7141                 return -EOPNOTSUPP;
7142
7143         vmx = to_vmx(vcpu);
7144         tscl = rdtsc();
7145         guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
7146         delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
7147         lapic_timer_advance_cycles = nsec_to_cycles(vcpu, lapic_timer_advance_ns);
7148
7149         if (delta_tsc > lapic_timer_advance_cycles)
7150                 delta_tsc -= lapic_timer_advance_cycles;
7151         else
7152                 delta_tsc = 0;
7153
7154         /* Convert to host delta tsc if tsc scaling is enabled */
7155         if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
7156                         u64_shl_div_u64(delta_tsc,
7157                                 kvm_tsc_scaling_ratio_frac_bits,
7158                                 vcpu->arch.tsc_scaling_ratio,
7159                                 &delta_tsc))
7160                 return -ERANGE;
7161
7162         /*
7163          * If the delta tsc can't fit in the 32 bit after the multi shift,
7164          * we can't use the preemption timer.
7165          * It's possible that it fits on later vmentries, but checking
7166          * on every vmentry is costly so we just use an hrtimer.
7167          */
7168         if (delta_tsc >> (cpu_preemption_timer_multi + 32))
7169                 return -ERANGE;
7170
7171         vmx->hv_deadline_tsc = tscl + delta_tsc;
7172         return delta_tsc == 0;
7173 }
7174
7175 static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
7176 {
7177         to_vmx(vcpu)->hv_deadline_tsc = -1;
7178 }
7179 #endif
7180
7181 static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
7182 {
7183         if (!kvm_pause_in_guest(vcpu->kvm))
7184                 shrink_ple_window(vcpu);
7185 }
7186
7187 static void vmx_slot_enable_log_dirty(struct kvm *kvm,
7188                                      struct kvm_memory_slot *slot)
7189 {
7190         kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
7191         kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
7192 }
7193
7194 static void vmx_slot_disable_log_dirty(struct kvm *kvm,
7195                                        struct kvm_memory_slot *slot)
7196 {
7197         kvm_mmu_slot_set_dirty(kvm, slot);
7198 }
7199
7200 static void vmx_flush_log_dirty(struct kvm *kvm)
7201 {
7202         kvm_flush_pml_buffers(kvm);
7203 }
7204
7205 static int vmx_write_pml_buffer(struct kvm_vcpu *vcpu)
7206 {
7207         struct vmcs12 *vmcs12;
7208         struct vcpu_vmx *vmx = to_vmx(vcpu);
7209         gpa_t gpa;
7210         struct page *page = NULL;
7211         u64 *pml_address;
7212
7213         if (is_guest_mode(vcpu)) {
7214                 WARN_ON_ONCE(vmx->nested.pml_full);
7215
7216                 /*
7217                  * Check if PML is enabled for the nested guest.
7218                  * Whether eptp bit 6 is set is already checked
7219                  * as part of A/D emulation.
7220                  */
7221                 vmcs12 = get_vmcs12(vcpu);
7222                 if (!nested_cpu_has_pml(vmcs12))
7223                         return 0;
7224
7225                 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) {
7226                         vmx->nested.pml_full = true;
7227                         return 1;
7228                 }
7229
7230                 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS) & ~0xFFFull;
7231
7232                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->pml_address);
7233                 if (is_error_page(page))
7234                         return 0;
7235
7236                 pml_address = kmap(page);
7237                 pml_address[vmcs12->guest_pml_index--] = gpa;
7238                 kunmap(page);
7239                 kvm_release_page_clean(page);
7240         }
7241
7242         return 0;
7243 }
7244
7245 static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
7246                                            struct kvm_memory_slot *memslot,
7247                                            gfn_t offset, unsigned long mask)
7248 {
7249         kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
7250 }
7251
7252 static void __pi_post_block(struct kvm_vcpu *vcpu)
7253 {
7254         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
7255         struct pi_desc old, new;
7256         unsigned int dest;
7257
7258         do {
7259                 old.control = new.control = pi_desc->control;
7260                 WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR,
7261                      "Wakeup handler not enabled while the VCPU is blocked\n");
7262
7263                 dest = cpu_physical_id(vcpu->cpu);
7264
7265                 if (x2apic_enabled())
7266                         new.ndst = dest;
7267                 else
7268                         new.ndst = (dest << 8) & 0xFF00;
7269
7270                 /* set 'NV' to 'notification vector' */
7271                 new.nv = POSTED_INTR_VECTOR;
7272         } while (cmpxchg64(&pi_desc->control, old.control,
7273                            new.control) != old.control);
7274
7275         if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) {
7276                 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
7277                 list_del(&vcpu->blocked_vcpu_list);
7278                 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
7279                 vcpu->pre_pcpu = -1;
7280         }
7281 }
7282
7283 /*
7284  * This routine does the following things for vCPU which is going
7285  * to be blocked if VT-d PI is enabled.
7286  * - Store the vCPU to the wakeup list, so when interrupts happen
7287  *   we can find the right vCPU to wake up.
7288  * - Change the Posted-interrupt descriptor as below:
7289  *      'NDST' <-- vcpu->pre_pcpu
7290  *      'NV' <-- POSTED_INTR_WAKEUP_VECTOR
7291  * - If 'ON' is set during this process, which means at least one
7292  *   interrupt is posted for this vCPU, we cannot block it, in
7293  *   this case, return 1, otherwise, return 0.
7294  *
7295  */
7296 static int pi_pre_block(struct kvm_vcpu *vcpu)
7297 {
7298         unsigned int dest;
7299         struct pi_desc old, new;
7300         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
7301
7302         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
7303                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
7304                 !kvm_vcpu_apicv_active(vcpu))
7305                 return 0;
7306
7307         WARN_ON(irqs_disabled());
7308         local_irq_disable();
7309         if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) {
7310                 vcpu->pre_pcpu = vcpu->cpu;
7311                 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
7312                 list_add_tail(&vcpu->blocked_vcpu_list,
7313                               &per_cpu(blocked_vcpu_on_cpu,
7314                                        vcpu->pre_pcpu));
7315                 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
7316         }
7317
7318         do {
7319                 old.control = new.control = pi_desc->control;
7320
7321                 WARN((pi_desc->sn == 1),
7322                      "Warning: SN field of posted-interrupts "
7323                      "is set before blocking\n");
7324
7325                 /*
7326                  * Since vCPU can be preempted during this process,
7327                  * vcpu->cpu could be different with pre_pcpu, we
7328                  * need to set pre_pcpu as the destination of wakeup
7329                  * notification event, then we can find the right vCPU
7330                  * to wakeup in wakeup handler if interrupts happen
7331                  * when the vCPU is in blocked state.
7332                  */
7333                 dest = cpu_physical_id(vcpu->pre_pcpu);
7334
7335                 if (x2apic_enabled())
7336                         new.ndst = dest;
7337                 else
7338                         new.ndst = (dest << 8) & 0xFF00;
7339
7340                 /* set 'NV' to 'wakeup vector' */
7341                 new.nv = POSTED_INTR_WAKEUP_VECTOR;
7342         } while (cmpxchg64(&pi_desc->control, old.control,
7343                            new.control) != old.control);
7344
7345         /* We should not block the vCPU if an interrupt is posted for it.  */
7346         if (pi_test_on(pi_desc) == 1)
7347                 __pi_post_block(vcpu);
7348
7349         local_irq_enable();
7350         return (vcpu->pre_pcpu == -1);
7351 }
7352
7353 static int vmx_pre_block(struct kvm_vcpu *vcpu)
7354 {
7355         if (pi_pre_block(vcpu))
7356                 return 1;
7357
7358         if (kvm_lapic_hv_timer_in_use(vcpu))
7359                 kvm_lapic_switch_to_sw_timer(vcpu);
7360
7361         return 0;
7362 }
7363
7364 static void pi_post_block(struct kvm_vcpu *vcpu)
7365 {
7366         if (vcpu->pre_pcpu == -1)
7367                 return;
7368
7369         WARN_ON(irqs_disabled());
7370         local_irq_disable();
7371         __pi_post_block(vcpu);
7372         local_irq_enable();
7373 }
7374
7375 static void vmx_post_block(struct kvm_vcpu *vcpu)
7376 {
7377         if (kvm_x86_ops->set_hv_timer)
7378                 kvm_lapic_switch_to_hv_timer(vcpu);
7379
7380         pi_post_block(vcpu);
7381 }
7382
7383 /*
7384  * vmx_update_pi_irte - set IRTE for Posted-Interrupts
7385  *
7386  * @kvm: kvm
7387  * @host_irq: host irq of the interrupt
7388  * @guest_irq: gsi of the interrupt
7389  * @set: set or unset PI
7390  * returns 0 on success, < 0 on failure
7391  */
7392 static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
7393                               uint32_t guest_irq, bool set)
7394 {
7395         struct kvm_kernel_irq_routing_entry *e;
7396         struct kvm_irq_routing_table *irq_rt;
7397         struct kvm_lapic_irq irq;
7398         struct kvm_vcpu *vcpu;
7399         struct vcpu_data vcpu_info;
7400         int idx, ret = 0;
7401
7402         if (!kvm_arch_has_assigned_device(kvm) ||
7403                 !irq_remapping_cap(IRQ_POSTING_CAP) ||
7404                 !kvm_vcpu_apicv_active(kvm->vcpus[0]))
7405                 return 0;
7406
7407         idx = srcu_read_lock(&kvm->irq_srcu);
7408         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
7409         if (guest_irq >= irq_rt->nr_rt_entries ||
7410             hlist_empty(&irq_rt->map[guest_irq])) {
7411                 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
7412                              guest_irq, irq_rt->nr_rt_entries);
7413                 goto out;
7414         }
7415
7416         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
7417                 if (e->type != KVM_IRQ_ROUTING_MSI)
7418                         continue;
7419                 /*
7420                  * VT-d PI cannot support posting multicast/broadcast
7421                  * interrupts to a vCPU, we still use interrupt remapping
7422                  * for these kind of interrupts.
7423                  *
7424                  * For lowest-priority interrupts, we only support
7425                  * those with single CPU as the destination, e.g. user
7426                  * configures the interrupts via /proc/irq or uses
7427                  * irqbalance to make the interrupts single-CPU.
7428                  *
7429                  * We will support full lowest-priority interrupt later.
7430                  */
7431
7432                 kvm_set_msi_irq(kvm, e, &irq);
7433                 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
7434                         /*
7435                          * Make sure the IRTE is in remapped mode if
7436                          * we don't handle it in posted mode.
7437                          */
7438                         ret = irq_set_vcpu_affinity(host_irq, NULL);
7439                         if (ret < 0) {
7440                                 printk(KERN_INFO
7441                                    "failed to back to remapped mode, irq: %u\n",
7442                                    host_irq);
7443                                 goto out;
7444                         }
7445
7446                         continue;
7447                 }
7448
7449                 vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
7450                 vcpu_info.vector = irq.vector;
7451
7452                 trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,
7453                                 vcpu_info.vector, vcpu_info.pi_desc_addr, set);
7454
7455                 if (set)
7456                         ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
7457                 else
7458                         ret = irq_set_vcpu_affinity(host_irq, NULL);
7459
7460                 if (ret < 0) {
7461                         printk(KERN_INFO "%s: failed to update PI IRTE\n",
7462                                         __func__);
7463                         goto out;
7464                 }
7465         }
7466
7467         ret = 0;
7468 out:
7469         srcu_read_unlock(&kvm->irq_srcu, idx);
7470         return ret;
7471 }
7472
7473 static void vmx_setup_mce(struct kvm_vcpu *vcpu)
7474 {
7475         if (vcpu->arch.mcg_cap & MCG_LMCE_P)
7476                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
7477                         FEATURE_CONTROL_LMCE;
7478         else
7479                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
7480                         ~FEATURE_CONTROL_LMCE;
7481 }
7482
7483 static int vmx_smi_allowed(struct kvm_vcpu *vcpu)
7484 {
7485         /* we need a nested vmexit to enter SMM, postpone if run is pending */
7486         if (to_vmx(vcpu)->nested.nested_run_pending)
7487                 return 0;
7488         return 1;
7489 }
7490
7491 static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
7492 {
7493         struct vcpu_vmx *vmx = to_vmx(vcpu);
7494
7495         vmx->nested.smm.guest_mode = is_guest_mode(vcpu);
7496         if (vmx->nested.smm.guest_mode)
7497                 nested_vmx_vmexit(vcpu, -1, 0, 0);
7498
7499         vmx->nested.smm.vmxon = vmx->nested.vmxon;
7500         vmx->nested.vmxon = false;
7501         vmx_clear_hlt(vcpu);
7502         return 0;
7503 }
7504
7505 static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, u64 smbase)
7506 {
7507         struct vcpu_vmx *vmx = to_vmx(vcpu);
7508         int ret;
7509
7510         if (vmx->nested.smm.vmxon) {
7511                 vmx->nested.vmxon = true;
7512                 vmx->nested.smm.vmxon = false;
7513         }
7514
7515         if (vmx->nested.smm.guest_mode) {
7516                 vcpu->arch.hflags &= ~HF_SMM_MASK;
7517                 ret = nested_vmx_enter_non_root_mode(vcpu, false);
7518                 vcpu->arch.hflags |= HF_SMM_MASK;
7519                 if (ret)
7520                         return ret;
7521
7522                 vmx->nested.smm.guest_mode = false;
7523         }
7524         return 0;
7525 }
7526
7527 static int enable_smi_window(struct kvm_vcpu *vcpu)
7528 {
7529         return 0;
7530 }
7531
7532 static __init int hardware_setup(void)
7533 {
7534         unsigned long host_bndcfgs;
7535         int r, i;
7536
7537         rdmsrl_safe(MSR_EFER, &host_efer);
7538
7539         for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
7540                 kvm_define_shared_msr(i, vmx_msr_index[i]);
7541
7542         if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0)
7543                 return -EIO;
7544
7545         if (boot_cpu_has(X86_FEATURE_NX))
7546                 kvm_enable_efer_bits(EFER_NX);
7547
7548         if (boot_cpu_has(X86_FEATURE_MPX)) {
7549                 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs);
7550                 WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost");
7551         }
7552
7553         if (boot_cpu_has(X86_FEATURE_XSAVES))
7554                 rdmsrl(MSR_IA32_XSS, host_xss);
7555
7556         if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() ||
7557             !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global()))
7558                 enable_vpid = 0;
7559
7560         if (!cpu_has_vmx_ept() ||
7561             !cpu_has_vmx_ept_4levels() ||
7562             !cpu_has_vmx_ept_mt_wb() ||
7563             !cpu_has_vmx_invept_global())
7564                 enable_ept = 0;
7565
7566         if (!cpu_has_vmx_ept_ad_bits() || !enable_ept)
7567                 enable_ept_ad_bits = 0;
7568
7569         if (!cpu_has_vmx_unrestricted_guest() || !enable_ept)
7570                 enable_unrestricted_guest = 0;
7571
7572         if (!cpu_has_vmx_flexpriority())
7573                 flexpriority_enabled = 0;
7574
7575         if (!cpu_has_virtual_nmis())
7576                 enable_vnmi = 0;
7577
7578         /*
7579          * set_apic_access_page_addr() is used to reload apic access
7580          * page upon invalidation.  No need to do anything if not
7581          * using the APIC_ACCESS_ADDR VMCS field.
7582          */
7583         if (!flexpriority_enabled)
7584                 kvm_x86_ops->set_apic_access_page_addr = NULL;
7585
7586         if (!cpu_has_vmx_tpr_shadow())
7587                 kvm_x86_ops->update_cr8_intercept = NULL;
7588
7589         if (enable_ept && !cpu_has_vmx_ept_2m_page())
7590                 kvm_disable_largepages();
7591
7592 #if IS_ENABLED(CONFIG_HYPERV)
7593         if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH
7594             && enable_ept) {
7595                 kvm_x86_ops->tlb_remote_flush = hv_remote_flush_tlb;
7596                 kvm_x86_ops->tlb_remote_flush_with_range =
7597                                 hv_remote_flush_tlb_with_range;
7598         }
7599 #endif
7600
7601         if (!cpu_has_vmx_ple()) {
7602                 ple_gap = 0;
7603                 ple_window = 0;
7604                 ple_window_grow = 0;
7605                 ple_window_max = 0;
7606                 ple_window_shrink = 0;
7607         }
7608
7609         if (!cpu_has_vmx_apicv()) {
7610                 enable_apicv = 0;
7611                 kvm_x86_ops->sync_pir_to_irr = NULL;
7612         }
7613
7614         if (cpu_has_vmx_tsc_scaling()) {
7615                 kvm_has_tsc_control = true;
7616                 kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
7617                 kvm_tsc_scaling_ratio_frac_bits = 48;
7618         }
7619
7620         set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
7621
7622         if (enable_ept)
7623                 vmx_enable_tdp();
7624         else
7625                 kvm_disable_tdp();
7626
7627         /*
7628          * Only enable PML when hardware supports PML feature, and both EPT
7629          * and EPT A/D bit features are enabled -- PML depends on them to work.
7630          */
7631         if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
7632                 enable_pml = 0;
7633
7634         if (!enable_pml) {
7635                 kvm_x86_ops->slot_enable_log_dirty = NULL;
7636                 kvm_x86_ops->slot_disable_log_dirty = NULL;
7637                 kvm_x86_ops->flush_log_dirty = NULL;
7638                 kvm_x86_ops->enable_log_dirty_pt_masked = NULL;
7639         }
7640
7641         if (!cpu_has_vmx_preemption_timer())
7642                 kvm_x86_ops->request_immediate_exit = __kvm_request_immediate_exit;
7643
7644         if (cpu_has_vmx_preemption_timer() && enable_preemption_timer) {
7645                 u64 vmx_msr;
7646
7647                 rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
7648                 cpu_preemption_timer_multi =
7649                         vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
7650         } else {
7651                 kvm_x86_ops->set_hv_timer = NULL;
7652                 kvm_x86_ops->cancel_hv_timer = NULL;
7653         }
7654
7655         kvm_set_posted_intr_wakeup_handler(wakeup_handler);
7656
7657         kvm_mce_cap_supported |= MCG_LMCE_P;
7658
7659         if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST)
7660                 return -EINVAL;
7661         if (!enable_ept || !cpu_has_vmx_intel_pt())
7662                 pt_mode = PT_MODE_SYSTEM;
7663
7664         if (nested) {
7665                 nested_vmx_setup_ctls_msrs(&vmcs_config.nested,
7666                                            vmx_capability.ept, enable_apicv);
7667
7668                 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers);
7669                 if (r)
7670                         return r;
7671         }
7672
7673         r = alloc_kvm_area();
7674         if (r)
7675                 nested_vmx_hardware_unsetup();
7676         return r;
7677 }
7678
7679 static __exit void hardware_unsetup(void)
7680 {
7681         if (nested)
7682                 nested_vmx_hardware_unsetup();
7683
7684         free_kvm_area();
7685 }
7686
7687 static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
7688         .cpu_has_kvm_support = cpu_has_kvm_support,
7689         .disabled_by_bios = vmx_disabled_by_bios,
7690         .hardware_setup = hardware_setup,
7691         .hardware_unsetup = hardware_unsetup,
7692         .check_processor_compatibility = vmx_check_processor_compat,
7693         .hardware_enable = hardware_enable,
7694         .hardware_disable = hardware_disable,
7695         .cpu_has_accelerated_tpr = report_flexpriority,
7696         .has_emulated_msr = vmx_has_emulated_msr,
7697
7698         .vm_init = vmx_vm_init,
7699         .vm_alloc = vmx_vm_alloc,
7700         .vm_free = vmx_vm_free,
7701
7702         .vcpu_create = vmx_create_vcpu,
7703         .vcpu_free = vmx_free_vcpu,
7704         .vcpu_reset = vmx_vcpu_reset,
7705
7706         .prepare_guest_switch = vmx_prepare_switch_to_guest,
7707         .vcpu_load = vmx_vcpu_load,
7708         .vcpu_put = vmx_vcpu_put,
7709
7710         .update_bp_intercept = update_exception_bitmap,
7711         .get_msr_feature = vmx_get_msr_feature,
7712         .get_msr = vmx_get_msr,
7713         .set_msr = vmx_set_msr,
7714         .get_segment_base = vmx_get_segment_base,
7715         .get_segment = vmx_get_segment,
7716         .set_segment = vmx_set_segment,
7717         .get_cpl = vmx_get_cpl,
7718         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
7719         .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
7720         .decache_cr3 = vmx_decache_cr3,
7721         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
7722         .set_cr0 = vmx_set_cr0,
7723         .set_cr3 = vmx_set_cr3,
7724         .set_cr4 = vmx_set_cr4,
7725         .set_efer = vmx_set_efer,
7726         .get_idt = vmx_get_idt,
7727         .set_idt = vmx_set_idt,
7728         .get_gdt = vmx_get_gdt,
7729         .set_gdt = vmx_set_gdt,
7730         .get_dr6 = vmx_get_dr6,
7731         .set_dr6 = vmx_set_dr6,
7732         .set_dr7 = vmx_set_dr7,
7733         .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
7734         .cache_reg = vmx_cache_reg,
7735         .get_rflags = vmx_get_rflags,
7736         .set_rflags = vmx_set_rflags,
7737
7738         .tlb_flush = vmx_flush_tlb,
7739         .tlb_flush_gva = vmx_flush_tlb_gva,
7740
7741         .run = vmx_vcpu_run,
7742         .handle_exit = vmx_handle_exit,
7743         .skip_emulated_instruction = skip_emulated_instruction,
7744         .set_interrupt_shadow = vmx_set_interrupt_shadow,
7745         .get_interrupt_shadow = vmx_get_interrupt_shadow,
7746         .patch_hypercall = vmx_patch_hypercall,
7747         .set_irq = vmx_inject_irq,
7748         .set_nmi = vmx_inject_nmi,
7749         .queue_exception = vmx_queue_exception,
7750         .cancel_injection = vmx_cancel_injection,
7751         .interrupt_allowed = vmx_interrupt_allowed,
7752         .nmi_allowed = vmx_nmi_allowed,
7753         .get_nmi_mask = vmx_get_nmi_mask,
7754         .set_nmi_mask = vmx_set_nmi_mask,
7755         .enable_nmi_window = enable_nmi_window,
7756         .enable_irq_window = enable_irq_window,
7757         .update_cr8_intercept = update_cr8_intercept,
7758         .set_virtual_apic_mode = vmx_set_virtual_apic_mode,
7759         .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
7760         .get_enable_apicv = vmx_get_enable_apicv,
7761         .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
7762         .load_eoi_exitmap = vmx_load_eoi_exitmap,
7763         .apicv_post_state_restore = vmx_apicv_post_state_restore,
7764         .hwapic_irr_update = vmx_hwapic_irr_update,
7765         .hwapic_isr_update = vmx_hwapic_isr_update,
7766         .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt,
7767         .sync_pir_to_irr = vmx_sync_pir_to_irr,
7768         .deliver_posted_interrupt = vmx_deliver_posted_interrupt,
7769
7770         .set_tss_addr = vmx_set_tss_addr,
7771         .set_identity_map_addr = vmx_set_identity_map_addr,
7772         .get_tdp_level = get_ept_level,
7773         .get_mt_mask = vmx_get_mt_mask,
7774
7775         .get_exit_info = vmx_get_exit_info,
7776
7777         .get_lpage_level = vmx_get_lpage_level,
7778
7779         .cpuid_update = vmx_cpuid_update,
7780
7781         .rdtscp_supported = vmx_rdtscp_supported,
7782         .invpcid_supported = vmx_invpcid_supported,
7783
7784         .set_supported_cpuid = vmx_set_supported_cpuid,
7785
7786         .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
7787
7788         .read_l1_tsc_offset = vmx_read_l1_tsc_offset,
7789         .write_l1_tsc_offset = vmx_write_l1_tsc_offset,
7790
7791         .set_tdp_cr3 = vmx_set_cr3,
7792
7793         .check_intercept = vmx_check_intercept,
7794         .handle_external_intr = vmx_handle_external_intr,
7795         .mpx_supported = vmx_mpx_supported,
7796         .xsaves_supported = vmx_xsaves_supported,
7797         .umip_emulated = vmx_umip_emulated,
7798         .pt_supported = vmx_pt_supported,
7799
7800         .request_immediate_exit = vmx_request_immediate_exit,
7801
7802         .sched_in = vmx_sched_in,
7803
7804         .slot_enable_log_dirty = vmx_slot_enable_log_dirty,
7805         .slot_disable_log_dirty = vmx_slot_disable_log_dirty,
7806         .flush_log_dirty = vmx_flush_log_dirty,
7807         .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
7808         .write_log_dirty = vmx_write_pml_buffer,
7809
7810         .pre_block = vmx_pre_block,
7811         .post_block = vmx_post_block,
7812
7813         .pmu_ops = &intel_pmu_ops,
7814
7815         .update_pi_irte = vmx_update_pi_irte,
7816
7817 #ifdef CONFIG_X86_64
7818         .set_hv_timer = vmx_set_hv_timer,
7819         .cancel_hv_timer = vmx_cancel_hv_timer,
7820 #endif
7821
7822         .setup_mce = vmx_setup_mce,
7823
7824         .smi_allowed = vmx_smi_allowed,
7825         .pre_enter_smm = vmx_pre_enter_smm,
7826         .pre_leave_smm = vmx_pre_leave_smm,
7827         .enable_smi_window = enable_smi_window,
7828
7829         .check_nested_events = NULL,
7830         .get_nested_state = NULL,
7831         .set_nested_state = NULL,
7832         .get_vmcs12_pages = NULL,
7833         .nested_enable_evmcs = NULL,
7834 };
7835
7836 static void vmx_cleanup_l1d_flush(void)
7837 {
7838         if (vmx_l1d_flush_pages) {
7839                 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER);
7840                 vmx_l1d_flush_pages = NULL;
7841         }
7842         /* Restore state so sysfs ignores VMX */
7843         l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
7844 }
7845
7846 static void vmx_exit(void)
7847 {
7848 #ifdef CONFIG_KEXEC_CORE
7849         RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
7850         synchronize_rcu();
7851 #endif
7852
7853         kvm_exit();
7854
7855 #if IS_ENABLED(CONFIG_HYPERV)
7856         if (static_branch_unlikely(&enable_evmcs)) {
7857                 int cpu;
7858                 struct hv_vp_assist_page *vp_ap;
7859                 /*
7860                  * Reset everything to support using non-enlightened VMCS
7861                  * access later (e.g. when we reload the module with
7862                  * enlightened_vmcs=0)
7863                  */
7864                 for_each_online_cpu(cpu) {
7865                         vp_ap = hv_get_vp_assist_page(cpu);
7866
7867                         if (!vp_ap)
7868                                 continue;
7869
7870                         vp_ap->current_nested_vmcs = 0;
7871                         vp_ap->enlighten_vmentry = 0;
7872                 }
7873
7874                 static_branch_disable(&enable_evmcs);
7875         }
7876 #endif
7877         vmx_cleanup_l1d_flush();
7878 }
7879 module_exit(vmx_exit);
7880
7881 static int __init vmx_init(void)
7882 {
7883         int r;
7884
7885 #if IS_ENABLED(CONFIG_HYPERV)
7886         /*
7887          * Enlightened VMCS usage should be recommended and the host needs
7888          * to support eVMCS v1 or above. We can also disable eVMCS support
7889          * with module parameter.
7890          */
7891         if (enlightened_vmcs &&
7892             ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED &&
7893             (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >=
7894             KVM_EVMCS_VERSION) {
7895                 int cpu;
7896
7897                 /* Check that we have assist pages on all online CPUs */
7898                 for_each_online_cpu(cpu) {
7899                         if (!hv_get_vp_assist_page(cpu)) {
7900                                 enlightened_vmcs = false;
7901                                 break;
7902                         }
7903                 }
7904
7905                 if (enlightened_vmcs) {
7906                         pr_info("KVM: vmx: using Hyper-V Enlightened VMCS\n");
7907                         static_branch_enable(&enable_evmcs);
7908                 }
7909         } else {
7910                 enlightened_vmcs = false;
7911         }
7912 #endif
7913
7914         r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
7915                      __alignof__(struct vcpu_vmx), THIS_MODULE);
7916         if (r)
7917                 return r;
7918
7919         /*
7920          * Must be called after kvm_init() so enable_ept is properly set
7921          * up. Hand the parameter mitigation value in which was stored in
7922          * the pre module init parser. If no parameter was given, it will
7923          * contain 'auto' which will be turned into the default 'cond'
7924          * mitigation mode.
7925          */
7926         if (boot_cpu_has(X86_BUG_L1TF)) {
7927                 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param);
7928                 if (r) {
7929                         vmx_exit();
7930                         return r;
7931                 }
7932         }
7933
7934 #ifdef CONFIG_KEXEC_CORE
7935         rcu_assign_pointer(crash_vmclear_loaded_vmcss,
7936                            crash_vmclear_local_loaded_vmcss);
7937 #endif
7938         vmx_check_vmcs12_offsets();
7939
7940         return 0;
7941 }
7942 module_init(vmx_init);