arch/x86/include/asm/mshyperv.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_MSHYPER_H
   3 #define _ASM_X86_MSHYPER_H
   4
   5 #include <linux/types.h>
   6 #include <linux/atomic.h>
   7 #include <linux/nmi.h>
   8 #include <asm/io.h>
   9 #include <asm/hyperv-tlfs.h>
  10 #include <asm/nospec-branch.h>
  11
  12 #define VP_INVAL        U32_MAX
  13
  14 struct ms_hyperv_info {
  15         u32 features;
  16         u32 misc_features;
  17         u32 hints;
  18         u32 nested_features;
  19         u32 max_vp_index;
  20         u32 max_lp_index;
  21 };
  22
  23 extern struct ms_hyperv_info ms_hyperv;
  24
  25 /*
  26  * Generate the guest ID.
  27  */
  28
  29 static inline  __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version,
  30                                        __u64 d_info2)
  31 {
  32         __u64 guest_id = 0;
  33
  34         guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48);
  35         guest_id |= (d_info1 << 48);
  36         guest_id |= (kernel_version << 16);
  37         guest_id |= d_info2;
  38
  39         return guest_id;
  40 }
  41
  42
  43 /* Free the message slot and signal end-of-message if required */
  44 static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type)
  45 {
  46         /*
  47          * On crash we're reading some other CPU's message page and we need
  48          * to be careful: this other CPU may already had cleared the header
  49          * and the host may already had delivered some other message there.
  50          * In case we blindly write msg->header.message_type we're going
  51          * to lose it. We can still lose a message of the same type but
  52          * we count on the fact that there can only be one
  53          * CHANNELMSG_UNLOAD_RESPONSE and we don't care about other messages
  54          * on crash.
  55          */
  56         if (cmpxchg(&msg->header.message_type, old_msg_type,
  57                     HVMSG_NONE) != old_msg_type)
  58                 return;
  59
  60         /*
  61          * Make sure the write to MessageType (ie set to
  62          * HVMSG_NONE) happens before we read the
  63          * MessagePending and EOMing. Otherwise, the EOMing
  64          * will not deliver any more messages since there is
  65          * no empty slot
  66          */
  67         mb();
  68
  69         if (msg->header.message_flags.msg_pending) {
  70                 /*
  71                  * This will cause message queue rescan to
  72                  * possibly deliver another msg from the
  73                  * hypervisor
  74                  */
  75                 wrmsrl(HV_X64_MSR_EOM, 0);
  76         }
  77 }
  78
  79 #define hv_init_timer(timer, tick) wrmsrl(timer, tick)
  80 #define hv_init_timer_config(config, val) wrmsrl(config, val)
  81
  82 #define hv_get_simp(val) rdmsrl(HV_X64_MSR_SIMP, val)
  83 #define hv_set_simp(val) wrmsrl(HV_X64_MSR_SIMP, val)
  84
  85 #define hv_get_siefp(val) rdmsrl(HV_X64_MSR_SIEFP, val)
  86 #define hv_set_siefp(val) wrmsrl(HV_X64_MSR_SIEFP, val)
  87
  88 #define hv_get_synic_state(val) rdmsrl(HV_X64_MSR_SCONTROL, val)
  89 #define hv_set_synic_state(val) wrmsrl(HV_X64_MSR_SCONTROL, val)
  90
  91 #define hv_get_vp_index(index) rdmsrl(HV_X64_MSR_VP_INDEX, index)
  92
  93 #define hv_get_synint_state(int_num, val) rdmsrl(int_num, val)
  94 #define hv_set_synint_state(int_num, val) wrmsrl(int_num, val)
  95
  96 void hyperv_callback_vector(void);
  97 void hyperv_reenlightenment_vector(void);
  98 #ifdef CONFIG_TRACING
  99 #define trace_hyperv_callback_vector hyperv_callback_vector
 100 #endif
 101 void hyperv_vector_handler(struct pt_regs *regs);
 102 void hv_setup_vmbus_irq(void (*handler)(void));
 103 void hv_remove_vmbus_irq(void);
 104
 105 void hv_setup_kexec_handler(void (*handler)(void));
 106 void hv_remove_kexec_handler(void);
 107 void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs));
 108 void hv_remove_crash_handler(void);
 109
 110 /*
 111  * Routines for stimer0 Direct Mode handling.
 112  * On x86/x64, there are no percpu actions to take.
 113  */
 114 void hv_stimer0_vector_handler(struct pt_regs *regs);
 115 void hv_stimer0_callback_vector(void);
 116 int hv_setup_stimer0_irq(int *irq, int *vector, void (*handler)(void));
 117 void hv_remove_stimer0_irq(int irq);
 118
 119 static inline void hv_enable_stimer0_percpu_irq(int irq) {}
 120 static inline void hv_disable_stimer0_percpu_irq(int irq) {}
 121
 122
 123 #if IS_ENABLED(CONFIG_HYPERV)
 124 extern struct clocksource *hyperv_cs;
 125 extern void *hv_hypercall_pg;
 126 extern void  __percpu  **hyperv_pcpu_input_arg;
 127
 128 static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 129 {
 130         u64 input_address = input ? virt_to_phys(input) : 0;
 131         u64 output_address = output ? virt_to_phys(output) : 0;
 132         u64 hv_status;
 133
 134 #ifdef CONFIG_X86_64
 135         if (!hv_hypercall_pg)
 136                 return U64_MAX;
 137
 138         __asm__ __volatile__("mov %4, %%r8\n"
 139                              CALL_NOSPEC
 140                              : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 141                                "+c" (control), "+d" (input_address)
 142                              :  "r" (output_address),
 143                                 THUNK_TARGET(hv_hypercall_pg)
 144                              : "cc", "memory", "r8", "r9", "r10", "r11");
 145 #else
 146         u32 input_address_hi = upper_32_bits(input_address);
 147         u32 input_address_lo = lower_32_bits(input_address);
 148         u32 output_address_hi = upper_32_bits(output_address);
 149         u32 output_address_lo = lower_32_bits(output_address);
 150
 151         if (!hv_hypercall_pg)
 152                 return U64_MAX;
 153
 154         __asm__ __volatile__(CALL_NOSPEC
 155                              : "=A" (hv_status),
 156                                "+c" (input_address_lo), ASM_CALL_CONSTRAINT
 157                              : "A" (control),
 158                                "b" (input_address_hi),
 159                                "D"(output_address_hi), "S"(output_address_lo),
 160                                THUNK_TARGET(hv_hypercall_pg)
 161                              : "cc", "memory");
 162 #endif /* !x86_64 */
 163         return hv_status;
 164 }
 165
 166 /* Fast hypercall with 8 bytes of input and no output */
 167 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 168 {
 169         u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
 170
 171 #ifdef CONFIG_X86_64
 172         {
 173                 __asm__ __volatile__(CALL_NOSPEC
 174                                      : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 175                                        "+c" (control), "+d" (input1)
 176                                      : THUNK_TARGET(hv_hypercall_pg)
 177                                      : "cc", "r8", "r9", "r10", "r11");
 178         }
 179 #else
 180         {
 181                 u32 input1_hi = upper_32_bits(input1);
 182                 u32 input1_lo = lower_32_bits(input1);
 183
 184                 __asm__ __volatile__ (CALL_NOSPEC
 185                                       : "=A"(hv_status),
 186                                         "+c"(input1_lo),
 187                                         ASM_CALL_CONSTRAINT
 188                                       : "A" (control),
 189                                         "b" (input1_hi),
 190                                         THUNK_TARGET(hv_hypercall_pg)
 191                                       : "cc", "edi", "esi");
 192         }
 193 #endif
 194                 return hv_status;
 195 }
 196
 197 /* Fast hypercall with 16 bytes of input */
 198 static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)
 199 {
 200         u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT;
 201
 202 #ifdef CONFIG_X86_64
 203         {
 204                 __asm__ __volatile__("mov %4, %%r8\n"
 205                                      CALL_NOSPEC
 206                                      : "=a" (hv_status), ASM_CALL_CONSTRAINT,
 207                                        "+c" (control), "+d" (input1)
 208                                      : "r" (input2),
 209                                        THUNK_TARGET(hv_hypercall_pg)
 210                                      : "cc", "r8", "r9", "r10", "r11");
 211         }
 212 #else
 213         {
 214                 u32 input1_hi = upper_32_bits(input1);
 215                 u32 input1_lo = lower_32_bits(input1);
 216                 u32 input2_hi = upper_32_bits(input2);
 217                 u32 input2_lo = lower_32_bits(input2);
 218
 219                 __asm__ __volatile__ (CALL_NOSPEC
 220                                       : "=A"(hv_status),
 221                                         "+c"(input1_lo), ASM_CALL_CONSTRAINT
 222                                       : "A" (control), "b" (input1_hi),
 223                                         "D"(input2_hi), "S"(input2_lo),
 224                                         THUNK_TARGET(hv_hypercall_pg)
 225                                       : "cc");
 226         }
 227 #endif
 228                 return hv_status;
 229 }
 230
 231 /*
 232  * Rep hypercalls. Callers of this functions are supposed to ensure that
 233  * rep_count and varhead_size comply with Hyper-V hypercall definition.
 234  */
 235 static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size,
 236                                       void *input, void *output)
 237 {
 238         u64 control = code;
 239         u64 status;
 240         u16 rep_comp;
 241
 242         control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET;
 243         control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET;
 244
 245         do {
 246                 status = hv_do_hypercall(control, input, output);
 247                 if ((status & HV_HYPERCALL_RESULT_MASK) != HV_STATUS_SUCCESS)
 248                         return status;
 249
 250                 /* Bits 32-43 of status have 'Reps completed' data. */
 251                 rep_comp = (status & HV_HYPERCALL_REP_COMP_MASK) >>
 252                         HV_HYPERCALL_REP_COMP_OFFSET;
 253
 254                 control &= ~HV_HYPERCALL_REP_START_MASK;
 255                 control |= (u64)rep_comp << HV_HYPERCALL_REP_START_OFFSET;
 256
 257                 touch_nmi_watchdog();
 258         } while (rep_comp < rep_count);
 259
 260         return status;
 261 }
 262
 263 /*
 264  * Hypervisor's notion of virtual processor ID is different from
 265  * Linux' notion of CPU ID. This information can only be retrieved
 266  * in the context of the calling CPU. Setup a map for easy access
 267  * to this information.
 268  */
 269 extern u32 *hv_vp_index;
 270 extern u32 hv_max_vp_index;
 271 extern struct hv_vp_assist_page **hv_vp_assist_page;
 272
 273 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)
 274 {
 275         if (!hv_vp_assist_page)
 276                 return NULL;
 277
 278         return hv_vp_assist_page[cpu];
 279 }
 280
 281 /**
 282  * hv_cpu_number_to_vp_number() - Map CPU to VP.
 283  * @cpu_number: CPU number in Linux terms
 284  *
 285  * This function returns the mapping between the Linux processor
 286  * number and the hypervisor's virtual processor number, useful
 287  * in making hypercalls and such that talk about specific
 288  * processors.
 289  *
 290  * Return: Virtual processor number in Hyper-V terms
 291  */
 292 static inline int hv_cpu_number_to_vp_number(int cpu_number)
 293 {
 294         return hv_vp_index[cpu_number];
 295 }
 296
 297 static inline int cpumask_to_vpset(struct hv_vpset *vpset,
 298                                     const struct cpumask *cpus)
 299 {
 300         int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1;
 301
 302         /* valid_bank_mask can represent up to 64 banks */
 303         if (hv_max_vp_index / 64 >= 64)
 304                 return 0;
 305
 306         /*
 307          * Clear all banks up to the maximum possible bank as hv_tlb_flush_ex
 308          * structs are not cleared between calls, we risk flushing unneeded
 309          * vCPUs otherwise.
 310          */
 311         for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++)
 312                 vpset->bank_contents[vcpu_bank] = 0;
 313
 314         /*
 315          * Some banks may end up being empty but this is acceptable.
 316          */
 317         for_each_cpu(cpu, cpus) {
 318                 vcpu = hv_cpu_number_to_vp_number(cpu);
 319                 if (vcpu == VP_INVAL)
 320                         return -1;
 321                 vcpu_bank = vcpu / 64;
 322                 vcpu_offset = vcpu % 64;
 323                 __set_bit(vcpu_offset, (unsigned long *)
 324                           &vpset->bank_contents[vcpu_bank]);
 325                 if (vcpu_bank >= nr_bank)
 326                         nr_bank = vcpu_bank + 1;
 327         }
 328         vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0);
 329         return nr_bank;
 330 }
 331
 332 void __init hyperv_init(void);
 333 void hyperv_setup_mmu_ops(void);
 334 void hyperv_report_panic(struct pt_regs *regs, long err);
 335 bool hv_is_hyperv_initialized(void);
 336 void hyperv_cleanup(void);
 337
 338 void hyperv_reenlightenment_intr(struct pt_regs *regs);
 339 void set_hv_tscchange_cb(void (*cb)(void));
 340 void clear_hv_tscchange_cb(void);
 341 void hyperv_stop_tsc_emulation(void);
 342
 343 #ifdef CONFIG_X86_64
 344 void hv_apic_init(void);
 345 #else
 346 static inline void hv_apic_init(void) {}
 347 #endif
 348
 349 #else /* CONFIG_HYPERV */
 350 static inline void hyperv_init(void) {}
 351 static inline bool hv_is_hyperv_initialized(void) { return false; }
 352 static inline void hyperv_cleanup(void) {}
 353 static inline void hyperv_setup_mmu_ops(void) {}
 354 static inline void set_hv_tscchange_cb(void (*cb)(void)) {}
 355 static inline void clear_hv_tscchange_cb(void) {}
 356 static inline void hyperv_stop_tsc_emulation(void) {};
 357 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)
 358 {
 359         return NULL;
 360 }
 361 #endif /* CONFIG_HYPERV */
 362
 363 #ifdef CONFIG_HYPERV_TSCPAGE
 364 struct ms_hyperv_tsc_page *hv_get_tsc_page(void);
 365 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg,
 366                                        u64 *cur_tsc)
 367 {
 368         u64 scale, offset;
 369         u32 sequence;
 370
 371         /*
 372          * The protocol for reading Hyper-V TSC page is specified in Hypervisor
 373          * Top-Level Functional Specification ver. 3.0 and above. To get the
 374          * reference time we must do the following:
 375          * - READ ReferenceTscSequence
 376          *   A special '0' value indicates the time source is unreliable and we
 377          *   need to use something else. The currently published specification
 378          *   versions (up to 4.0b) contain a mistake and wrongly claim '-1'
 379          *   instead of '0' as the special value, see commit c35b82ef0294.
 380          * - ReferenceTime =
 381          *        ((RDTSC() * ReferenceTscScale) >> 64) + ReferenceTscOffset
 382          * - READ ReferenceTscSequence again. In case its value has changed
 383          *   since our first reading we need to discard ReferenceTime and repeat
 384          *   the whole sequence as the hypervisor was updating the page in
 385          *   between.
 386          */
 387         do {
 388                 sequence = READ_ONCE(tsc_pg->tsc_sequence);
 389                 if (!sequence)
 390                         return U64_MAX;
 391                 /*
 392                  * Make sure we read sequence before we read other values from
 393                  * TSC page.
 394                  */
 395                 smp_rmb();
 396
 397                 scale = READ_ONCE(tsc_pg->tsc_scale);
 398                 offset = READ_ONCE(tsc_pg->tsc_offset);
 399                 *cur_tsc = rdtsc_ordered();
 400
 401                 /*
 402                  * Make sure we read sequence after we read all other values
 403                  * from TSC page.
 404                  */
 405                 smp_rmb();
 406
 407         } while (READ_ONCE(tsc_pg->tsc_sequence) != sequence);
 408
 409         return mul_u64_u64_shr(*cur_tsc, scale, 64) + offset;
 410 }
 411
 412 static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg)
 413 {
 414         u64 cur_tsc;
 415
 416         return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc);
 417 }
 418
 419 #else
 420 static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
 421 {
 422         return NULL;
 423 }
 424
 425 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg,
 426                                        u64 *cur_tsc)
 427 {
 428         BUG();
 429         return U64_MAX;
 430 }
 431 #endif
 432 #endif