arch/x86/include/asm/mshyperv.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_MSHYPER_H
   3 #define _ASM_X86_MSHYPER_H
   4
   5 #include <linux/types.h>
   6 #include <linux/atomic.h>
   7 #include <linux/nmi.h>
   8 #include <asm/io.h>
   9 #include <asm/hyperv-tlfs.h>
  10 #include <asm/nospec-branch.h>
  11
  12 #define VP_INVAL        U32_MAX
  13
  14 struct ms_hyperv_info {
  15         u32 features;
  16         u32 misc_features;
  17         u32 hints;
  18         u32 nested_features;
  19         u32 max_vp_index;
  20         u32 max_lp_index;
  21 };
  22
  23 extern struct ms_hyperv_info ms_hyperv;
  24
  25 /*
  26  * Generate the guest ID.
  27  */
  28
  29 static inline  __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version,
  30                                        __u64 d_info2)
  31 {
  32         __u64 guest_id = 0;
  33
  34         guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48);
  35         guest_id |= (d_info1 << 48);
  36         guest_id |= (kernel_version << 16);
  37         guest_id |= d_info2;
  38
  39         return guest_id;
  40 }
  41
  42
  43 /* Free the message slot and signal end-of-message if required */
  44 static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type)
  45 {
  46         /*
  47          * On crash we're reading some other CPU's message page and we need
  48          * to be careful: this other CPU may already had cleared the header
  49          * and the host may already had delivered some other message there.
  50          * In case we blindly write msg->header.message_type we're going
  51          * to lose it. We can still lose a message of the same type but
  52          * we count on the fact that there can only be one
  53          * CHANNELMSG_UNLOAD_RESPONSE and we don't care about other messages
  54          * on crash.
  55          */
  56         if (cmpxchg(&msg->header.message_type, old_msg_type,
  57                     HVMSG_NONE) != old_msg_type)
  58                 return;
  59
  60         /*
  61          * Make sure the write to MessageType (ie set to
  62          * HVMSG_NONE) happens before we read the
  63          * MessagePending and EOMing. Otherwise, the EOMing
  64          * will not deliver any more messages since there is
  65          * no empty slot
  66          */
  67         mb();
  68
  69         if (msg->header.message_flags.msg_pending) {
  70                 /*
  71                  * This will cause message queue rescan to
  72                  * possibly deliver another msg from the
  73                  * hypervisor
  74                  */
  75                 wrmsrl(HV_X64_MSR_EOM, 0);
  76         }
  77 }
  78
  79 #define hv_init_timer(timer, tick) \
  80         wrmsrl(HV_X64_MSR_STIMER0_COUNT + (2*timer), tick)
  81 #define hv_init_timer_config(timer, val) \
  82         wrmsrl(HV_X64_MSR_STIMER0_CONFIG + (2*timer), val)
  83
  84 #define hv_get_simp(val) rdmsrl(HV_X64_MSR_SIMP, val)
  85 #define hv_set_simp(val) wrmsrl(HV_X64_MSR_SIMP, val)
  86
  87 #define hv_get_siefp(val) rdmsrl(HV_X64_MSR_SIEFP, val)
  88 #define hv_set_siefp(val) wrmsrl(HV_X64_MSR_SIEFP, val)
  89
  90 #define hv_get_synic_state(val) rdmsrl(HV_X64_MSR_SCONTROL, val)
  91 #define hv_set_synic_state(val) wrmsrl(HV_X64_MSR_SCONTROL, val)
  92
  93 #define hv_get_vp_index(index) rdmsrl(HV_X64_MSR_VP_INDEX, index)
  94
  95 #define hv_get_synint_state(int_num, val) \
  96         rdmsrl(HV_X64_MSR_SINT0 + int_num, val)
  97 #define hv_set_synint_state(int_num, val) \
  98         wrmsrl(HV_X64_MSR_SINT0 + int_num, val)
  99
 100 #define hv_get_crash_ctl(val) \
 101         rdmsrl(HV_X64_MSR_CRASH_CTL, val)
 102
 103 void hyperv_callback_vector(void);
 104 void hyperv_reenlightenment_vector(void);
 105 #ifdef CONFIG_TRACING
 106 #define trace_hyperv_callback_vector hyperv_callback_vector
 107 #endif
 108 void hyperv_vector_handler(struct pt_regs *regs);
 109 void hv_setup_vmbus_irq(void (*handler)(void));
 110 void hv_remove_vmbus_irq(void);
 111
 112 void hv_setup_kexec_handler(void (*handler)(void));
 113 void hv_remove_kexec_handler(void);
 114 void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs));
 115 void hv_remove_crash_handler(void);
 116
 117 /*
 118  * Routines for stimer0 Direct Mode handling.
 119  * On x86/x64, there are no percpu actions to take.
 120  */
 121 void hv_stimer0_vector_handler(struct pt_regs *regs);
 122 void hv_stimer0_callback_vector(void);
 123 int hv_setup_stimer0_irq(int *irq, int *vector, void (*handler)(void));
 124 void hv_remove_stimer0_irq(int irq);
 125
 126 static inline void hv_enable_stimer0_percpu_irq(int irq) {}
 127 static inline void hv_disable_stimer0_percpu_irq(int irq) {}
 128
 129
 130 #if IS_ENABLED(CONFIG_HYPERV)
 131 extern struct clocksource *hyperv_cs;
 132 extern void *hv_hypercall_pg;
 133 extern void  __percpu  **hyperv_pcpu_input_arg;
 134
 135 static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 136 {
 137         u64 input_address = input ? virt_to_phys(input) : 0;
 138         u64 output_address = output ? virt_to_phys(output) : 0;
 139         u64 hv_status;
 140
 141 #ifdef CONFIG_X86_64
 142         if (!hv_hypercall_pg)
 143                 return U64_MAX;
 144
 145         __asm__ __volatile__("mov %4, %%r8\n"
 146                              CALL_NOSPEC
 147                              : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 148                                "+c" (control), "+d" (input_address)
 149                              :  "r" (output_address),
 150                                 THUNK_TARGET(hv_hypercall_pg)
 151                              : "cc", "memory", "r8", "r9", "r10", "r11");
 152 #else
 153         u32 input_address_hi = upper_32_bits(input_address);
 154         u32 input_address_lo = lower_32_bits(input_address);
 155         u32 output_address_hi = upper_32_bits(output_address);
 156         u32 output_address_lo = lower_32_bits(output_address);
 157
 158         if (!hv_hypercall_pg)
 159                 return U64_MAX;
 160
 161         __asm__ __volatile__(CALL_NOSPEC
 162                              : "=A" (hv_status),
 163                                "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 164                              : "A" (control),
 165                                "b" (input_address_hi),
 166                                "D"(output_address_hi), "S"(output_address_lo),
 167                                THUNK_TARGET(hv_hypercall_pg)
 168                              : "cc", "memory");
 169 #endif /* !x86_64 */
 170         return hv_status;
 171 }
 172
 173 /* Fast hypercall with 8 bytes of input and no output */
 174 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 175 {
 176         u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
 177
 178 #ifdef CONFIG_X86_64
 179         {
 180                 __asm__ __volatile__(CALL_NOSPEC
 181                                      : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 182                                        "+c" (control), "+d" (input1)
 183                                      : THUNK_TARGET(hv_hypercall_pg)
 184                                      : "cc", "r8", "r9", "r10", "r11");
 185         }
 186 #else
 187         {
 188                 u32 input1_hi = upper_32_bits(input1);
 189                 u32 input1_lo = lower_32_bits(input1);
 190
 191                 __asm__ __volatile__ (CALL_NOSPEC
 192                                       : "=A"(hv_status),
 193                                         "+c"(input1_lo),
 194                                         ASM_CALL_CONSTRAINT
 195                                       : "A" (control),
 196                                         "b" (input1_hi),
 197                                         THUNK_TARGET(hv_hypercall_pg)
 198                                       : "cc", "edi", "esi");
 199         }
 200 #endif
 201                 return hv_status;
 202 }
 203
 204 /* Fast hypercall with 16 bytes of input */
 205 static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
 206 {
 207         u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
 208
 209 #ifdef CONFIG_X86_64
 210         {
 211                 __asm__ __volatile__("mov %4, %%r8\n"
 212                                      CALL_NOSPEC
 213                                      : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 214                                        "+c" (control), "+d" (input1)
 215                                      : "r" (input2),
 216                                        THUNK_TARGET(hv_hypercall_pg)
 217                                      : "cc", "r8", "r9", "r10", "r11");
 218         }
 219 #else
 220         {
 221                 u32 input1_hi = upper_32_bits(input1);
 222                 u32 input1_lo = lower_32_bits(input1);
 223                 u32 input2_hi = upper_32_bits(input2);
 224                 u32 input2_lo = lower_32_bits(input2);
 225
 226                 __asm__ __volatile__ (CALL_NOSPEC
 227                                       : "=A"(hv_status),
 228                                         "+c"(input1_lo), ASM_CALL_CONSTRAINT
 229                                       : "A" (control), "b" (input1_hi),
 230                                         "D"(input2_hi), "S"(input2_lo),
 231                                         THUNK_TARGET(hv_hypercall_pg)
 232                                       : "cc");
 233         }
 234 #endif
 235                 return hv_status;
 236 }
 237
 238 /*
 239  * Rep hypercalls. Callers of this functions are supposed to ensure that
 240  * rep_count and varhead_size comply with Hyper-V hypercall definition.
 241  */
 242 static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size,
 243                                       void *input, void *output)
 244 {
 245         u64 control = code;
 246         u64 status;
 247         u16 rep_comp;
 248
 249         control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET;
 250         control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET;
 251
 252         do {
 253                 status = hv_do_hypercall(control, input, output);
 254                 if ((status & HV_HYPERCALL_RESULT_MASK) != HV_STATUS_SUCCESS)
 255                         return status;
 256
 257                 /* Bits 32-43 of status have 'Reps completed' data. */
 258                 rep_comp = (status & HV_HYPERCALL_REP_COMP_MASK) >>
 259                         HV_HYPERCALL_REP_COMP_OFFSET;
 260
 261                 control &= ~HV_HYPERCALL_REP_START_MASK;
 262                 control |= (u64)rep_comp << HV_HYPERCALL_REP_START_OFFSET;
 263
 264                 touch_nmi_watchdog();
 265         } while (rep_comp < rep_count);
 266
 267         return status;
 268 }
 269
 270 /*
 271  * Hypervisor's notion of virtual processor ID is different from
 272  * Linux' notion of CPU ID. This information can only be retrieved
 273  * in the context of the calling CPU. Setup a map for easy access
 274  * to this information.
 275  */
 276 extern u32 *hv_vp_index;
 277 extern u32 hv_max_vp_index;
 278 extern struct hv_vp_assist_page **hv_vp_assist_page;
 279
 280 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)
 281 {
 282         if (!hv_vp_assist_page)
 283                 return NULL;
 284
 285         return hv_vp_assist_page[cpu];
 286 }
 287
 288 /**
 289  * hv_cpu_number_to_vp_number() - Map CPU to VP.
 290  * @cpu_number: CPU number in Linux terms
 291  *
 292  * This function returns the mapping between the Linux processor
 293  * number and the hypervisor's virtual processor number, useful
 294  * in making hypercalls and such that talk about specific
 295  * processors.
 296  *
 297  * Return: Virtual processor number in Hyper-V terms
 298  */
 299 static inline int hv_cpu_number_to_vp_number(int cpu_number)
 300 {
 301         return hv_vp_index[cpu_number];
 302 }
 303
 304 static inline int cpumask_to_vpset(struct hv_vpset *vpset,
 305                                     const struct cpumask *cpus)
 306 {
 307         int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1;
 308
 309         /* valid_bank_mask can represent up to 64 banks */
 310         if (hv_max_vp_index / 64 >= 64)
 311                 return 0;
 312
 313         /*
 314          * Clear all banks up to the maximum possible bank as hv_tlb_flush_ex
 315          * structs are not cleared between calls, we risk flushing unneeded
 316          * vCPUs otherwise.
 317          */
 318         for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++)
 319                 vpset->bank_contents[vcpu_bank] = 0;
 320
 321         /*
 322          * Some banks may end up being empty but this is acceptable.
 323          */
 324         for_each_cpu(cpu, cpus) {
 325                 vcpu = hv_cpu_number_to_vp_number(cpu);
 326                 if (vcpu == VP_INVAL)
 327                         return -1;
 328                 vcpu_bank = vcpu / 64;
 329                 vcpu_offset = vcpu % 64;
 330                 __set_bit(vcpu_offset, (unsigned long *)
 331                           &vpset->bank_contents[vcpu_bank]);
 332                 if (vcpu_bank >= nr_bank)
 333                         nr_bank = vcpu_bank + 1;
 334         }
 335         vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0);
 336         return nr_bank;
 337 }
 338
 339 void __init hyperv_init(void);
 340 void hyperv_setup_mmu_ops(void);
 341 void hyperv_report_panic(struct pt_regs *regs, long err);
 342 void hyperv_report_panic_msg(phys_addr_t pa, size_t size);
 343 bool hv_is_hyperv_initialized(void);
 344 void hyperv_cleanup(void);
 345
 346 void hyperv_reenlightenment_intr(struct pt_regs *regs);
 347 void set_hv_tscchange_cb(void (*cb)(void));
 348 void clear_hv_tscchange_cb(void);
 349 void hyperv_stop_tsc_emulation(void);
 350 int hyperv_flush_guest_mapping(u64 as);
 351
 352 #ifdef CONFIG_X86_64
 353 void hv_apic_init(void);
 354 void __init hv_init_spinlocks(void);
 355 bool hv_vcpu_is_preempted(int vcpu);
 356 #else
 357 static inline void hv_apic_init(void) {}
 358 #endif
 359
 360 #else /* CONFIG_HYPERV */
 361 static inline void hyperv_init(void) {}
 362 static inline bool hv_is_hyperv_initialized(void) { return false; }
 363 static inline void hyperv_cleanup(void) {}
 364 static inline void hyperv_setup_mmu_ops(void) {}
 365 static inline void set_hv_tscchange_cb(void (*cb)(void)) {}
 366 static inline void clear_hv_tscchange_cb(void) {}
 367 static inline void hyperv_stop_tsc_emulation(void) {};
 368 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)
 369 {
 370         return NULL;
 371 }
 372 static inline int hyperv_flush_guest_mapping(u64 as) { return -1; }
 373 #endif /* CONFIG_HYPERV */
 374
 375 #ifdef CONFIG_HYPERV_TSCPAGE
 376 struct ms_hyperv_tsc_page *hv_get_tsc_page(void);
 377 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg,
 378                                        u64 *cur_tsc)
 379 {
 380         u64 scale, offset;
 381         u32 sequence;
 382
 383         /*
 384          * The protocol for reading Hyper-V TSC page is specified in Hypervisor
 385          * Top-Level Functional Specification ver. 3.0 and above. To get the
 386          * reference time we must do the following:
 387          * - READ ReferenceTscSequence
 388          *   A special '0' value indicates the time source is unreliable and we
 389          *   need to use something else. The currently published specification
 390          *   versions (up to 4.0b) contain a mistake and wrongly claim '-1'
 391          *   instead of '0' as the special value, see commit c35b82ef0294.
 392          * - ReferenceTime =
 393          *        ((RDTSC() * ReferenceTscScale) >> 64) + ReferenceTscOffset
 394          * - READ ReferenceTscSequence again. In case its value has changed
 395          *   since our first reading we need to discard ReferenceTime and repeat
 396          *   the whole sequence as the hypervisor was updating the page in
 397          *   between.
 398          */
 399         do {
 400                 sequence = READ_ONCE(tsc_pg->tsc_sequence);
 401                 if (!sequence)
 402                         return U64_MAX;
 403                 /*
 404                  * Make sure we read sequence before we read other values from
 405                  * TSC page.
 406                  */
 407                 smp_rmb();
 408
 409                 scale = READ_ONCE(tsc_pg->tsc_scale);
 410                 offset = READ_ONCE(tsc_pg->tsc_offset);
 411                 *cur_tsc = rdtsc_ordered();
 412
 413                 /*
 414                  * Make sure we read sequence after we read all other values
 415                  * from TSC page.
 416                  */
 417                 smp_rmb();
 418
 419         } while (READ_ONCE(tsc_pg->tsc_sequence) != sequence);
 420
 421         return mul_u64_u64_shr(*cur_tsc, scale, 64) + offset;
 422 }
 423
 424 static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg)
 425 {
 426         u64 cur_tsc;
 427
 428         return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc);
 429 }
 430
 431 #else
 432 static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
 433 {
 434         return NULL;
 435 }
 436
 437 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg,
 438                                        u64 *cur_tsc)
 439 {
 440         BUG();
 441         return U64_MAX;
 442 }
 443 #endif
 444 #endif