x86/kvmclock: Switch kvmclock data to a PER_CPU variable

[sfrench/cifs-2.6.git] / arch / x86 / kernel / kvmclock.c

diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index 7d690d2238f81c6eae4b86ae312645d6d2dd6f81..91b94c0ae4e337598dddc02f6a7630187642dd1e 100644 (file)
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -23,6 +23,7 @@
 #include <asm/apic.h>
 #include <linux/percpu.h>
 #include <linux/hardirq.h>
+#include <linux/cpuhotplug.h>
 #include <linux/sched.h>
 #include <linux/sched/clock.h>
 #include <linux/mm.h>
@@ -55,12 +56,23 @@ early_param("no-kvmclock-vsyscall", parse_no_kvmclock_vsyscall);
 
 /* Aligned to page sizes to match whats mapped via vsyscalls to userspace */
 #define HV_CLOCK_SIZE  (sizeof(struct pvclock_vsyscall_time_info) * NR_CPUS)
+#define HVC_BOOT_ARRAY_SIZE \
+       (PAGE_SIZE / sizeof(struct pvclock_vsyscall_time_info))
 
-static u8 hv_clock_mem[PAGE_ALIGN(HV_CLOCK_SIZE)] __aligned(PAGE_SIZE);
-
-/* The hypervisor will put information about time periodically here */
-static struct pvclock_vsyscall_time_info *hv_clock __ro_after_init;
+static struct pvclock_vsyscall_time_info
+                       hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __aligned(PAGE_SIZE);
 static struct pvclock_wall_clock wall_clock;
+static DEFINE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu);
+
+static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void)
+{
+       return &this_cpu_read(hv_clock_per_cpu)->pvti;
+}
+
+static inline struct pvclock_vsyscall_time_info *this_cpu_hvclock(void)
+{
+       return this_cpu_read(hv_clock_per_cpu);
+}
 
 /*
  * The wallclock is the time of day when we booted. Since then, some time may
@@ -69,17 +81,10 @@ static struct pvclock_wall_clock wall_clock;
  */
 static void kvm_get_wallclock(struct timespec64 *now)
 {
-       struct pvclock_vcpu_time_info *vcpu_time;
-       int cpu;
-
        wrmsrl(msr_kvm_wall_clock, slow_virt_to_phys(&wall_clock));
-
-       cpu = get_cpu();
-
-       vcpu_time = &hv_clock[cpu].pvti;
-       pvclock_read_wallclock(&wall_clock, vcpu_time, now);
-
-       put_cpu();
+       preempt_disable();
+       pvclock_read_wallclock(&wall_clock, this_cpu_pvti(), now);
+       preempt_enable();
 }
 
 static int kvm_set_wallclock(const struct timespec64 *now)
@@ -89,14 +94,10 @@ static int kvm_set_wallclock(const struct timespec64 *now)
 
 static u64 kvm_clock_read(void)
 {
-       struct pvclock_vcpu_time_info *src;
        u64 ret;
-       int cpu;
 
        preempt_disable_notrace();
-       cpu = smp_processor_id();
-       src = &hv_clock[cpu].pvti;
-       ret = pvclock_clocksource_read(src);
+       ret = pvclock_clocksource_read(this_cpu_pvti());
        preempt_enable_notrace();
        return ret;
 }
@@ -141,7 +142,7 @@ static inline void kvm_sched_clock_init(bool stable)
 static unsigned long kvm_get_tsc_khz(void)
 {
        setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
-       return pvclock_tsc_khz(&hv_clock[0].pvti);
+       return pvclock_tsc_khz(this_cpu_pvti());
 }
 
 static void kvm_get_preset_lpj(void)
@@ -158,15 +159,14 @@ static void kvm_get_preset_lpj(void)
 
 bool kvm_check_and_clear_guest_paused(void)
 {
-       struct pvclock_vcpu_time_info *src;
+       struct pvclock_vsyscall_time_info *src = this_cpu_hvclock();
        bool ret = false;
 
-       if (!hv_clock)
+       if (!src)
                return ret;
 
-       src = &hv_clock[smp_processor_id()].pvti;
-       if ((src->flags & PVCLOCK_GUEST_STOPPED) != 0) {
-               src->flags &= ~PVCLOCK_GUEST_STOPPED;
+       if ((src->pvti.flags & PVCLOCK_GUEST_STOPPED) != 0) {
+               src->pvti.flags &= ~PVCLOCK_GUEST_STOPPED;
                pvclock_touch_watchdogs();
                ret = true;
        }
@@ -184,17 +184,15 @@ EXPORT_SYMBOL_GPL(kvm_clock);
 
 static void kvm_register_clock(char *txt)
 {
-       struct pvclock_vcpu_time_info *src;
-       int cpu = smp_processor_id();
+       struct pvclock_vsyscall_time_info *src = this_cpu_hvclock();
        u64 pa;
 
-       if (!hv_clock)
+       if (!src)
                return;
 
-       src = &hv_clock[cpu].pvti;
-       pa = slow_virt_to_phys(src) | 0x01ULL;
+       pa = slow_virt_to_phys(&src->pvti) | 0x01ULL;
        wrmsrl(msr_kvm_system_time, pa);
-       pr_info("kvm-clock: cpu %d, msr %llx, %s", cpu, pa, txt);
+       pr_info("kvm-clock: cpu %d, msr %llx, %s", smp_processor_id(), pa, txt);
 }
 
 static void kvm_save_sched_clock_state(void)
@@ -242,12 +240,12 @@ static int __init kvm_setup_vsyscall_timeinfo(void)
 #ifdef CONFIG_X86_64
        u8 flags;
 
-       if (!hv_clock || !kvmclock_vsyscall)
+       if (!per_cpu(hv_clock_per_cpu, 0) || !kvmclock_vsyscall)
                return 0;
 
-       flags = pvclock_read_flags(&hv_clock[0].pvti);
+       flags = pvclock_read_flags(&hv_clock_boot[0].pvti);
        if (!(flags & PVCLOCK_TSC_STABLE_BIT))
-               return 1;
+               return 0;
 
        kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK;
 #endif
@@ -255,6 +253,28 @@ static int __init kvm_setup_vsyscall_timeinfo(void)
 }
 early_initcall(kvm_setup_vsyscall_timeinfo);
 
+static int kvmclock_setup_percpu(unsigned int cpu)
+{
+       struct pvclock_vsyscall_time_info *p = per_cpu(hv_clock_per_cpu, cpu);
+
+       /*
+        * The per cpu area setup replicates CPU0 data to all cpu
+        * pointers. So carefully check. CPU0 has been set up in init
+        * already.
+        */
+       if (!cpu || (p && p != per_cpu(hv_clock_per_cpu, 0)))
+               return 0;
+
+       /* Use the static page for the first CPUs, allocate otherwise */
+       if (cpu < HVC_BOOT_ARRAY_SIZE)
+               p = &hv_clock_boot[cpu];
+       else
+               p = kzalloc(sizeof(*p), GFP_KERNEL);
+
+       per_cpu(hv_clock_per_cpu, cpu) = p;
+       return p ? 0 : -ENOMEM;
+}
+
 void __init kvmclock_init(void)
 {
        u8 flags;
@@ -269,17 +289,22 @@ void __init kvmclock_init(void)
                return;
        }
 
+       if (cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "kvmclock:setup_percpu",
+                             kvmclock_setup_percpu, NULL) < 0) {
+               return;
+       }
+
        pr_info("kvm-clock: Using msrs %x and %x",
                msr_kvm_system_time, msr_kvm_wall_clock);
 
-       hv_clock = (struct pvclock_vsyscall_time_info *)hv_clock_mem;
+       this_cpu_write(hv_clock_per_cpu, &hv_clock_boot[0]);
        kvm_register_clock("primary cpu clock");
-       pvclock_set_pvti_cpu0_va(hv_clock);
+       pvclock_set_pvti_cpu0_va(hv_clock_boot);
 
        if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE_STABLE_BIT))
                pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
 
-       flags = pvclock_read_flags(&hv_clock[0].pvti);
+       flags = pvclock_read_flags(&hv_clock_boot[0].pvti);
        kvm_sched_clock_init(flags & PVCLOCK_TSC_STABLE_BIT);
 
        x86_platform.calibrate_tsc = kvm_get_tsc_khz;