1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_MSHYPER_H 3 #define _ASM_X86_MSHYPER_H 4 5 #include <linux/types.h> 6 #include <linux/atomic.h> 7 #include <linux/nmi.h> 8 #include <asm/io.h> 9 #include <asm/hyperv-tlfs.h> 10 #include <asm/nospec-branch.h> 11 12 struct ms_hyperv_info { 13 u32 features; 14 u32 misc_features; 15 u32 hints; 16 u32 nested_features; 17 u32 max_vp_index; 18 u32 max_lp_index; 19 }; 20 21 extern struct ms_hyperv_info ms_hyperv; 22 23 24 /* 25 * Generate the guest ID. 26 */ 27 28 static inline __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version, 29 __u64 d_info2) 30 { 31 __u64 guest_id = 0; 32 33 guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48); 34 guest_id |= (d_info1 << 48); 35 guest_id |= (kernel_version << 16); 36 guest_id |= d_info2; 37 38 return guest_id; 39 } 40 41 42 /* Free the message slot and signal end-of-message if required */ 43 static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type) 44 { 45 /* 46 * On crash we're reading some other CPU's message page and we need 47 * to be careful: this other CPU may already had cleared the header 48 * and the host may already had delivered some other message there. 49 * In case we blindly write msg->header.message_type we're going 50 * to lose it. We can still lose a message of the same type but 51 * we count on the fact that there can only be one 52 * CHANNELMSG_UNLOAD_RESPONSE and we don't care about other messages 53 * on crash. 54 */ 55 if (cmpxchg(&msg->header.message_type, old_msg_type, 56 HVMSG_NONE) != old_msg_type) 57 return; 58 59 /* 60 * Make sure the write to MessageType (ie set to 61 * HVMSG_NONE) happens before we read the 62 * MessagePending and EOMing. Otherwise, the EOMing 63 * will not deliver any more messages since there is 64 * no empty slot 65 */ 66 mb(); 67 68 if (msg->header.message_flags.msg_pending) { 69 /* 70 * This will cause message queue rescan to 71 * possibly deliver another msg from the 72 * hypervisor 73 */ 74 wrmsrl(HV_X64_MSR_EOM, 0); 75 } 76 } 77 78 #define hv_init_timer(timer, tick) \ 79 wrmsrl(HV_X64_MSR_STIMER0_COUNT + (2*timer), tick) 80 #define hv_init_timer_config(timer, val) \ 81 wrmsrl(HV_X64_MSR_STIMER0_CONFIG + (2*timer), val) 82 83 #define hv_get_simp(val) rdmsrl(HV_X64_MSR_SIMP, val) 84 #define hv_set_simp(val) wrmsrl(HV_X64_MSR_SIMP, val) 85 86 #define hv_get_siefp(val) rdmsrl(HV_X64_MSR_SIEFP, val) 87 #define hv_set_siefp(val) wrmsrl(HV_X64_MSR_SIEFP, val) 88 89 #define hv_get_synic_state(val) rdmsrl(HV_X64_MSR_SCONTROL, val) 90 #define hv_set_synic_state(val) wrmsrl(HV_X64_MSR_SCONTROL, val) 91 92 #define hv_get_vp_index(index) rdmsrl(HV_X64_MSR_VP_INDEX, index) 93 94 #define hv_get_synint_state(int_num, val) \ 95 rdmsrl(HV_X64_MSR_SINT0 + int_num, val) 96 #define hv_set_synint_state(int_num, val) \ 97 wrmsrl(HV_X64_MSR_SINT0 + int_num, val) 98 99 void hyperv_callback_vector(void); 100 void hyperv_reenlightenment_vector(void); 101 #ifdef CONFIG_TRACING 102 #define trace_hyperv_callback_vector hyperv_callback_vector 103 #endif 104 void hyperv_vector_handler(struct pt_regs *regs); 105 void hv_setup_vmbus_irq(void (*handler)(void)); 106 void hv_remove_vmbus_irq(void); 107 108 void hv_setup_kexec_handler(void (*handler)(void)); 109 void hv_remove_kexec_handler(void); 110 void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs)); 111 void hv_remove_crash_handler(void); 112 113 /* 114 * Routines for stimer0 Direct Mode handling. 115 * On x86/x64, there are no percpu actions to take. 116 */ 117 void hv_stimer0_vector_handler(struct pt_regs *regs); 118 void hv_stimer0_callback_vector(void); 119 int hv_setup_stimer0_irq(int *irq, int *vector, void (*handler)(void)); 120 void hv_remove_stimer0_irq(int irq); 121 122 static inline void hv_enable_stimer0_percpu_irq(int irq) {} 123 static inline void hv_disable_stimer0_percpu_irq(int irq) {} 124 125 126 #if IS_ENABLED(CONFIG_HYPERV) 127 extern struct clocksource *hyperv_cs; 128 extern void *hv_hypercall_pg; 129 extern void __percpu **hyperv_pcpu_input_arg; 130 131 static inline u64 hv_do_hypercall(u64 control, void *input, void *output) 132 { 133 u64 input_address = input ? virt_to_phys(input) : 0; 134 u64 output_address = output ? virt_to_phys(output) : 0; 135 u64 hv_status; 136 137 #ifdef CONFIG_X86_64 138 if (!hv_hypercall_pg) 139 return U64_MAX; 140 141 __asm__ __volatile__("mov %4, %%r8\n" 142 CALL_NOSPEC 143 : "=a" (hv_status), ASM_CALL_CONSTRAINT, 144 "+c" (control), "+d" (input_address) 145 : "r" (output_address), 146 THUNK_TARGET(hv_hypercall_pg) 147 : "cc", "memory", "r8", "r9", "r10", "r11"); 148 #else 149 u32 input_address_hi = upper_32_bits(input_address); 150 u32 input_address_lo = lower_32_bits(input_address); 151 u32 output_address_hi = upper_32_bits(output_address); 152 u32 output_address_lo = lower_32_bits(output_address); 153 154 if (!hv_hypercall_pg) 155 return U64_MAX; 156 157 __asm__ __volatile__(CALL_NOSPEC 158 : "=A" (hv_status), 159 "+c" (input_address_lo), ASM_CALL_CONSTRAINT 160 : "A" (control), 161 "b" (input_address_hi), 162 "D"(output_address_hi), "S"(output_address_lo), 163 THUNK_TARGET(hv_hypercall_pg) 164 : "cc", "memory"); 165 #endif /* !x86_64 */ 166 return hv_status; 167 } 168 169 /* Fast hypercall with 8 bytes of input and no output */ 170 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) 171 { 172 u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT; 173 174 #ifdef CONFIG_X86_64 175 { 176 __asm__ __volatile__(CALL_NOSPEC 177 : "=a" (hv_status), ASM_CALL_CONSTRAINT, 178 "+c" (control), "+d" (input1) 179 : THUNK_TARGET(hv_hypercall_pg) 180 : "cc", "r8", "r9", "r10", "r11"); 181 } 182 #else 183 { 184 u32 input1_hi = upper_32_bits(input1); 185 u32 input1_lo = lower_32_bits(input1); 186 187 __asm__ __volatile__ (CALL_NOSPEC 188 : "=A"(hv_status), 189 "+c"(input1_lo), 190 ASM_CALL_CONSTRAINT 191 : "A" (control), 192 "b" (input1_hi), 193 THUNK_TARGET(hv_hypercall_pg) 194 : "cc", "edi", "esi"); 195 } 196 #endif 197 return hv_status; 198 } 199 200 /* 201 * Rep hypercalls. Callers of this functions are supposed to ensure that 202 * rep_count and varhead_size comply with Hyper-V hypercall definition. 203 */ 204 static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, 205 void *input, void *output) 206 { 207 u64 control = code; 208 u64 status; 209 u16 rep_comp; 210 211 control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET; 212 control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET; 213 214 do { 215 status = hv_do_hypercall(control, input, output); 216 if ((status & HV_HYPERCALL_RESULT_MASK) != HV_STATUS_SUCCESS) 217 return status; 218 219 /* Bits 32-43 of status have 'Reps completed' data. */ 220 rep_comp = (status & HV_HYPERCALL_REP_COMP_MASK) >> 221 HV_HYPERCALL_REP_COMP_OFFSET; 222 223 control &= ~HV_HYPERCALL_REP_START_MASK; 224 control |= (u64)rep_comp << HV_HYPERCALL_REP_START_OFFSET; 225 226 touch_nmi_watchdog(); 227 } while (rep_comp < rep_count); 228 229 return status; 230 } 231 232 /* 233 * Hypervisor's notion of virtual processor ID is different from 234 * Linux' notion of CPU ID. This information can only be retrieved 235 * in the context of the calling CPU. Setup a map for easy access 236 * to this information. 237 */ 238 extern u32 *hv_vp_index; 239 extern u32 hv_max_vp_index; 240 extern struct hv_vp_assist_page **hv_vp_assist_page; 241 242 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) 243 { 244 if (!hv_vp_assist_page) 245 return NULL; 246 247 return hv_vp_assist_page[cpu]; 248 } 249 250 /** 251 * hv_cpu_number_to_vp_number() - Map CPU to VP. 252 * @cpu_number: CPU number in Linux terms 253 * 254 * This function returns the mapping between the Linux processor 255 * number and the hypervisor's virtual processor number, useful 256 * in making hypercalls and such that talk about specific 257 * processors. 258 * 259 * Return: Virtual processor number in Hyper-V terms 260 */ 261 static inline int hv_cpu_number_to_vp_number(int cpu_number) 262 { 263 return hv_vp_index[cpu_number]; 264 } 265 266 static inline int cpumask_to_vpset(struct hv_vpset *vpset, 267 const struct cpumask *cpus) 268 { 269 int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1; 270 271 /* valid_bank_mask can represent up to 64 banks */ 272 if (hv_max_vp_index / 64 >= 64) 273 return 0; 274 275 /* 276 * Clear all banks up to the maximum possible bank as hv_tlb_flush_ex 277 * structs are not cleared between calls, we risk flushing unneeded 278 * vCPUs otherwise. 279 */ 280 for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++) 281 vpset->bank_contents[vcpu_bank] = 0; 282 283 /* 284 * Some banks may end up being empty but this is acceptable. 285 */ 286 for_each_cpu(cpu, cpus) { 287 vcpu = hv_cpu_number_to_vp_number(cpu); 288 vcpu_bank = vcpu / 64; 289 vcpu_offset = vcpu % 64; 290 __set_bit(vcpu_offset, (unsigned long *) 291 &vpset->bank_contents[vcpu_bank]); 292 if (vcpu_bank >= nr_bank) 293 nr_bank = vcpu_bank + 1; 294 } 295 vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0); 296 return nr_bank; 297 } 298 299 void __init hyperv_init(void); 300 void hyperv_setup_mmu_ops(void); 301 void hyperv_report_panic(struct pt_regs *regs, long err); 302 bool hv_is_hyperv_initialized(void); 303 void hyperv_cleanup(void); 304 305 void hyperv_reenlightenment_intr(struct pt_regs *regs); 306 void set_hv_tscchange_cb(void (*cb)(void)); 307 void clear_hv_tscchange_cb(void); 308 void hyperv_stop_tsc_emulation(void); 309 310 #ifdef CONFIG_X86_64 311 void hv_apic_init(void); 312 #else 313 static inline void hv_apic_init(void) {} 314 #endif 315 316 #else /* CONFIG_HYPERV */ 317 static inline void hyperv_init(void) {} 318 static inline bool hv_is_hyperv_initialized(void) { return false; } 319 static inline void hyperv_cleanup(void) {} 320 static inline void hyperv_setup_mmu_ops(void) {} 321 static inline void set_hv_tscchange_cb(void (*cb)(void)) {} 322 static inline void clear_hv_tscchange_cb(void) {} 323 static inline void hyperv_stop_tsc_emulation(void) {}; 324 static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) 325 { 326 return NULL; 327 } 328 #endif /* CONFIG_HYPERV */ 329 330 #ifdef CONFIG_HYPERV_TSCPAGE 331 struct ms_hyperv_tsc_page *hv_get_tsc_page(void); 332 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, 333 u64 *cur_tsc) 334 { 335 u64 scale, offset; 336 u32 sequence; 337 338 /* 339 * The protocol for reading Hyper-V TSC page is specified in Hypervisor 340 * Top-Level Functional Specification ver. 3.0 and above. To get the 341 * reference time we must do the following: 342 * - READ ReferenceTscSequence 343 * A special '0' value indicates the time source is unreliable and we 344 * need to use something else. The currently published specification 345 * versions (up to 4.0b) contain a mistake and wrongly claim '-1' 346 * instead of '0' as the special value, see commit c35b82ef0294. 347 * - ReferenceTime = 348 * ((RDTSC() * ReferenceTscScale) >> 64) + ReferenceTscOffset 349 * - READ ReferenceTscSequence again. In case its value has changed 350 * since our first reading we need to discard ReferenceTime and repeat 351 * the whole sequence as the hypervisor was updating the page in 352 * between. 353 */ 354 do { 355 sequence = READ_ONCE(tsc_pg->tsc_sequence); 356 if (!sequence) 357 return U64_MAX; 358 /* 359 * Make sure we read sequence before we read other values from 360 * TSC page. 361 */ 362 smp_rmb(); 363 364 scale = READ_ONCE(tsc_pg->tsc_scale); 365 offset = READ_ONCE(tsc_pg->tsc_offset); 366 *cur_tsc = rdtsc_ordered(); 367 368 /* 369 * Make sure we read sequence after we read all other values 370 * from TSC page. 371 */ 372 smp_rmb(); 373 374 } while (READ_ONCE(tsc_pg->tsc_sequence) != sequence); 375 376 return mul_u64_u64_shr(*cur_tsc, scale, 64) + offset; 377 } 378 379 static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg) 380 { 381 u64 cur_tsc; 382 383 return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc); 384 } 385 386 #else 387 static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void) 388 { 389 return NULL; 390 } 391 392 static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, 393 u64 *cur_tsc) 394 { 395 BUG(); 396 return U64_MAX; 397 } 398 #endif 399 #endif 400