drivers/xen/time.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Xen stolen ticks accounting.
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/kernel_stat.h>
   7 #include <linux/math64.h>
   8 #include <linux/gfp.h>
   9
  10 #include <asm/paravirt.h>
  11 #include <asm/xen/hypervisor.h>
  12 #include <asm/xen/hypercall.h>
  13
  14 #include <xen/events.h>
  15 #include <xen/features.h>
  16 #include <xen/interface/xen.h>
  17 #include <xen/interface/vcpu.h>
  18 #include <xen/xen-ops.h>
  19
  20 /* runstate info updated by Xen */
  21 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
  22
  23 /* return an consistent snapshot of 64-bit time/counter value */
  24 static u64 get64(const u64 *p)
  25 {
  26         u64 ret;
  27
  28         if (BITS_PER_LONG < 64) {
  29                 u32 *p32 = (u32 *)p;
  30                 u32 h, l, h2;
  31
  32                 /*
  33                  * Read high then low, and then make sure high is
  34                  * still the same; this will only loop if low wraps
  35                  * and carries into high.
  36                  * XXX some clean way to make this endian-proof?
  37                  */
  38                 do {
  39                         h = READ_ONCE(p32[1]);
  40                         l = READ_ONCE(p32[0]);
  41                         h2 = READ_ONCE(p32[1]);
  42                 } while(h2 != h);
  43
  44                 ret = (((u64)h) << 32) | l;
  45         } else
  46                 ret = READ_ONCE(*p);
  47
  48         return ret;
  49 }
  50
  51 static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res,
  52                                           unsigned int cpu)
  53 {
  54         u64 state_time;
  55         struct vcpu_runstate_info *state;
  56
  57         BUG_ON(preemptible());
  58
  59         state = per_cpu_ptr(&xen_runstate, cpu);
  60
  61         do {
  62                 state_time = get64(&state->state_entry_time);
  63                 rmb();  /* Hypervisor might update data. */
  64                 *res = READ_ONCE(*state);
  65                 rmb();  /* Hypervisor might update data. */
  66         } while (get64(&state->state_entry_time) != state_time ||
  67                  (state_time & XEN_RUNSTATE_UPDATE));
  68 }
  69
  70 /*
  71  * Runstate accounting
  72  */
  73 void xen_get_runstate_snapshot(struct vcpu_runstate_info *res)
  74 {
  75         xen_get_runstate_snapshot_cpu(res, smp_processor_id());
  76 }
  77
  78 /* return true when a vcpu could run but has no real cpu to run on */
  79 bool xen_vcpu_stolen(int vcpu)
  80 {
  81         return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
  82 }
  83
  84 u64 xen_steal_clock(int cpu)
  85 {
  86         struct vcpu_runstate_info state;
  87
  88         xen_get_runstate_snapshot_cpu(&state, cpu);
  89         return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline];
  90 }
  91
  92 void xen_setup_runstate_info(int cpu)
  93 {
  94         struct vcpu_register_runstate_memory_area area;
  95
  96         area.addr.v = &per_cpu(xen_runstate, cpu);
  97
  98         if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
  99                                xen_vcpu_nr(cpu), &area))
 100                 BUG();
 101 }
 102
 103 void __init xen_time_setup_guest(void)
 104 {
 105         bool xen_runstate_remote;
 106
 107         xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable,
 108                                         VMASST_TYPE_runstate_update_flag);
 109
 110         pv_time_ops.steal_clock = xen_steal_clock;
 111
 112         static_key_slow_inc(&paravirt_steal_enabled);
 113         if (xen_runstate_remote)
 114                 static_key_slow_inc(&paravirt_steal_rq_enabled);
 115 }