1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2023, Microsoft Corporation. 4 * 5 * Author: 6 * Roman Kisel <romank@linux.microsoft.com> 7 * Saurabh Sengar <ssengar@linux.microsoft.com> 8 * Naman Jain <namjain@linux.microsoft.com> 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/miscdevice.h> 14 #include <linux/anon_inodes.h> 15 #include <linux/cpuhotplug.h> 16 #include <linux/count_zeros.h> 17 #include <linux/entry-virt.h> 18 #include <linux/eventfd.h> 19 #include <linux/poll.h> 20 #include <linux/file.h> 21 #include <linux/vmalloc.h> 22 #include <asm/debugreg.h> 23 #include <asm/mshyperv.h> 24 #include <trace/events/ipi.h> 25 #include <uapi/asm/mtrr.h> 26 #include <uapi/linux/mshv.h> 27 #include <hyperv/hvhdk.h> 28 29 #include "../../kernel/fpu/legacy.h" 30 #include "mshv.h" 31 #include "mshv_vtl.h" 32 #include "hyperv_vmbus.h" 33 34 MODULE_AUTHOR("Microsoft"); 35 MODULE_LICENSE("GPL"); 36 MODULE_DESCRIPTION("Microsoft Hyper-V VTL Driver"); 37 38 #define MSHV_ENTRY_REASON_LOWER_VTL_CALL 0x1 39 #define MSHV_ENTRY_REASON_INTERRUPT 0x2 40 #define MSHV_ENTRY_REASON_INTERCEPT 0x3 41 42 #define MSHV_REAL_OFF_SHIFT 16 43 #define MSHV_PG_OFF_CPU_MASK (BIT_ULL(MSHV_REAL_OFF_SHIFT) - 1) 44 #define MSHV_RUN_PAGE_OFFSET 0 45 #define MSHV_REG_PAGE_OFFSET 1 46 #define VTL2_VMBUS_SINT_INDEX 7 47 48 static struct device *mem_dev; 49 50 static struct tasklet_struct msg_dpc; 51 static wait_queue_head_t fd_wait_queue; 52 static bool has_message; 53 static struct eventfd_ctx *flag_eventfds[HV_EVENT_FLAGS_COUNT]; 54 static DEFINE_MUTEX(flag_lock); 55 static bool __read_mostly mshv_has_reg_page; 56 57 /* hvcall code is of type u16, allocate a bitmap of size (1 << 16) to accommodate it */ 58 #define MAX_BITMAP_SIZE ((U16_MAX + 1) / 8) 59 60 struct mshv_vtl_hvcall_fd { 61 u8 allow_bitmap[MAX_BITMAP_SIZE]; 62 bool allow_map_initialized; 63 /* 64 * Used to protect hvcall setup in IOCTLs 65 */ 66 struct mutex init_mutex; 67 struct miscdevice *dev; 68 }; 69 70 struct mshv_vtl_poll_file { 71 struct file *file; 72 wait_queue_entry_t wait; 73 wait_queue_head_t *wqh; 74 poll_table pt; 75 int cpu; 76 }; 77 78 struct mshv_vtl { 79 struct device *module_dev; 80 u64 id; 81 }; 82 83 struct mshv_vtl_per_cpu { 84 struct mshv_vtl_run *run; 85 struct page *reg_page; 86 }; 87 88 /* SYNIC_OVERLAY_PAGE_MSR - internal, identical to hv_synic_simp */ 89 union hv_synic_overlay_page_msr { 90 u64 as_uint64; 91 struct { 92 u64 enabled: 1; 93 u64 reserved: 11; 94 u64 pfn: 52; 95 } __packed; 96 }; 97 98 static struct mutex mshv_vtl_poll_file_lock; 99 static union hv_register_vsm_page_offsets mshv_vsm_page_offsets; 100 static union hv_register_vsm_capabilities mshv_vsm_capabilities; 101 102 static DEFINE_PER_CPU(struct mshv_vtl_poll_file, mshv_vtl_poll_file); 103 static DEFINE_PER_CPU(unsigned long long, num_vtl0_transitions); 104 static DEFINE_PER_CPU(struct mshv_vtl_per_cpu, mshv_vtl_per_cpu); 105 106 static const union hv_input_vtl input_vtl_zero; 107 static const union hv_input_vtl input_vtl_normal = { 108 .use_target_vtl = 1, 109 }; 110 111 static const struct file_operations mshv_vtl_fops; 112 113 static long 114 mshv_ioctl_create_vtl(void __user *user_arg, struct device *module_dev) 115 { 116 struct mshv_vtl *vtl; 117 struct file *file; 118 int fd; 119 120 vtl = kzalloc_obj(*vtl); 121 if (!vtl) 122 return -ENOMEM; 123 124 fd = get_unused_fd_flags(O_CLOEXEC); 125 if (fd < 0) { 126 kfree(vtl); 127 return fd; 128 } 129 file = anon_inode_getfile("mshv_vtl", &mshv_vtl_fops, 130 vtl, O_RDWR); 131 if (IS_ERR(file)) { 132 put_unused_fd(fd); 133 kfree(vtl); 134 return PTR_ERR(file); 135 } 136 vtl->module_dev = module_dev; 137 fd_install(fd, file); 138 139 return fd; 140 } 141 142 static long 143 mshv_ioctl_check_extension(void __user *user_arg) 144 { 145 u32 arg; 146 147 if (copy_from_user(&arg, user_arg, sizeof(arg))) 148 return -EFAULT; 149 150 switch (arg) { 151 case MSHV_CAP_CORE_API_STABLE: 152 return 0; 153 case MSHV_CAP_REGISTER_PAGE: 154 return mshv_has_reg_page; 155 case MSHV_CAP_VTL_RETURN_ACTION: 156 return mshv_vsm_capabilities.return_action_available; 157 case MSHV_CAP_DR6_SHARED: 158 return mshv_vsm_capabilities.dr6_shared; 159 } 160 161 return -EOPNOTSUPP; 162 } 163 164 static long 165 mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) 166 { 167 struct miscdevice *misc = filp->private_data; 168 169 switch (ioctl) { 170 case MSHV_CHECK_EXTENSION: 171 return mshv_ioctl_check_extension((void __user *)arg); 172 case MSHV_CREATE_VTL: 173 return mshv_ioctl_create_vtl((void __user *)arg, misc->this_device); 174 } 175 176 return -ENOTTY; 177 } 178 179 static const struct file_operations mshv_dev_fops = { 180 .owner = THIS_MODULE, 181 .unlocked_ioctl = mshv_dev_ioctl, 182 .llseek = noop_llseek, 183 }; 184 185 static struct miscdevice mshv_dev = { 186 .minor = MISC_DYNAMIC_MINOR, 187 .name = "mshv", 188 .fops = &mshv_dev_fops, 189 .mode = 0600, 190 }; 191 192 static struct mshv_vtl_run *mshv_vtl_this_run(void) 193 { 194 return *this_cpu_ptr(&mshv_vtl_per_cpu.run); 195 } 196 197 static struct mshv_vtl_run *mshv_vtl_cpu_run(int cpu) 198 { 199 return *per_cpu_ptr(&mshv_vtl_per_cpu.run, cpu); 200 } 201 202 static struct page *mshv_vtl_cpu_reg_page(int cpu) 203 { 204 return *per_cpu_ptr(&mshv_vtl_per_cpu.reg_page, cpu); 205 } 206 207 static void mshv_vtl_configure_reg_page(struct mshv_vtl_per_cpu *per_cpu) 208 { 209 struct hv_register_assoc reg_assoc = {}; 210 union hv_synic_overlay_page_msr overlay = {}; 211 struct page *reg_page; 212 213 reg_page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL); 214 if (!reg_page) { 215 WARN(1, "failed to allocate register page\n"); 216 return; 217 } 218 219 overlay.enabled = 1; 220 overlay.pfn = page_to_hvpfn(reg_page); 221 reg_assoc.name = HV_X64_REGISTER_REG_PAGE; 222 reg_assoc.value.reg64 = overlay.as_uint64; 223 224 if (hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, 225 1, input_vtl_zero, ®_assoc)) { 226 WARN(1, "failed to setup register page\n"); 227 __free_page(reg_page); 228 return; 229 } 230 231 per_cpu->reg_page = reg_page; 232 mshv_has_reg_page = true; 233 } 234 235 static void mshv_vtl_synic_enable_regs(unsigned int cpu) 236 { 237 union hv_synic_sint sint; 238 239 sint.as_uint64 = 0; 240 sint.vector = HYPERVISOR_CALLBACK_VECTOR; 241 sint.masked = false; 242 sint.auto_eoi = hv_recommend_using_aeoi(); 243 244 /* Enable intercepts */ 245 if (!mshv_vsm_capabilities.intercept_page_available) 246 hv_set_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX, 247 sint.as_uint64); 248 249 /* VTL2 Host VSP SINT is (un)masked when the user mode requests that */ 250 } 251 252 static int mshv_vtl_get_vsm_regs(void) 253 { 254 struct hv_register_assoc registers[2]; 255 int ret, count = 2; 256 257 registers[0].name = HV_REGISTER_VSM_CODE_PAGE_OFFSETS; 258 registers[1].name = HV_REGISTER_VSM_CAPABILITIES; 259 260 ret = hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, 261 count, input_vtl_zero, registers); 262 if (ret) 263 return ret; 264 265 mshv_vsm_page_offsets.as_uint64 = registers[0].value.reg64; 266 mshv_vsm_capabilities.as_uint64 = registers[1].value.reg64; 267 268 return ret; 269 } 270 271 static int mshv_vtl_configure_vsm_partition(struct device *dev) 272 { 273 union hv_register_vsm_partition_config config; 274 struct hv_register_assoc reg_assoc; 275 276 config.as_uint64 = 0; 277 config.default_vtl_protection_mask = HV_MAP_GPA_PERMISSIONS_MASK; 278 config.enable_vtl_protection = 1; 279 config.zero_memory_on_reset = 1; 280 config.intercept_vp_startup = 1; 281 config.intercept_cpuid_unimplemented = 1; 282 283 if (mshv_vsm_capabilities.intercept_page_available) { 284 dev_dbg(dev, "using intercept page\n"); 285 config.intercept_page = 1; 286 } 287 288 reg_assoc.name = HV_REGISTER_VSM_PARTITION_CONFIG; 289 reg_assoc.value.reg64 = config.as_uint64; 290 291 return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, 292 1, input_vtl_zero, ®_assoc); 293 } 294 295 static void mshv_vtl_vmbus_isr(void) 296 { 297 struct hv_per_cpu_context *per_cpu; 298 struct hv_message *msg; 299 u32 message_type; 300 union hv_synic_event_flags *event_flags; 301 struct eventfd_ctx *eventfd; 302 u16 i; 303 304 per_cpu = this_cpu_ptr(hv_context.cpu_context); 305 if (smp_processor_id() == 0) { 306 msg = (struct hv_message *)per_cpu->hyp_synic_message_page + VTL2_VMBUS_SINT_INDEX; 307 message_type = READ_ONCE(msg->header.message_type); 308 if (message_type != HVMSG_NONE) 309 tasklet_schedule(&msg_dpc); 310 } 311 312 event_flags = (union hv_synic_event_flags *)per_cpu->hyp_synic_event_page + 313 VTL2_VMBUS_SINT_INDEX; 314 for_each_set_bit(i, event_flags->flags, HV_EVENT_FLAGS_COUNT) { 315 if (!sync_test_and_clear_bit(i, event_flags->flags)) 316 continue; 317 rcu_read_lock(); 318 eventfd = READ_ONCE(flag_eventfds[i]); 319 if (eventfd) 320 eventfd_signal(eventfd); 321 rcu_read_unlock(); 322 } 323 324 vmbus_isr(); 325 } 326 327 static int mshv_vtl_alloc_context(unsigned int cpu) 328 { 329 struct mshv_vtl_per_cpu *per_cpu = this_cpu_ptr(&mshv_vtl_per_cpu); 330 331 per_cpu->run = (struct mshv_vtl_run *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 332 if (!per_cpu->run) 333 return -ENOMEM; 334 335 if (mshv_vsm_capabilities.intercept_page_available) 336 mshv_vtl_configure_reg_page(per_cpu); 337 338 mshv_vtl_synic_enable_regs(cpu); 339 340 return 0; 341 } 342 343 static int mshv_vtl_cpuhp_online; 344 345 static int hv_vtl_setup_synic(void) 346 { 347 int ret; 348 349 /* Use our isr to first filter out packets destined for userspace */ 350 hv_setup_vmbus_handler(mshv_vtl_vmbus_isr); 351 352 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hyperv/vtl:online", 353 mshv_vtl_alloc_context, NULL); 354 if (ret < 0) { 355 hv_setup_vmbus_handler(vmbus_isr); 356 return ret; 357 } 358 359 mshv_vtl_cpuhp_online = ret; 360 361 return 0; 362 } 363 364 static void hv_vtl_remove_synic(void) 365 { 366 cpuhp_remove_state(mshv_vtl_cpuhp_online); 367 hv_setup_vmbus_handler(vmbus_isr); 368 } 369 370 static int vtl_get_vp_register(struct hv_register_assoc *reg) 371 { 372 return hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, 373 1, input_vtl_normal, reg); 374 } 375 376 static int vtl_set_vp_register(struct hv_register_assoc *reg) 377 { 378 return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, 379 1, input_vtl_normal, reg); 380 } 381 382 static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg) 383 { 384 struct mshv_vtl_ram_disposition vtl0_mem; 385 struct dev_pagemap *pgmap; 386 void *addr; 387 388 if (copy_from_user(&vtl0_mem, arg, sizeof(vtl0_mem))) 389 return -EFAULT; 390 if (vtl0_mem.last_pfn <= vtl0_mem.start_pfn) { 391 dev_err(vtl->module_dev, "range start pfn (%llx) > end pfn (%llx)\n", 392 vtl0_mem.start_pfn, vtl0_mem.last_pfn); 393 return -EFAULT; 394 } 395 396 pgmap = kzalloc_obj(*pgmap); 397 if (!pgmap) 398 return -ENOMEM; 399 400 /* 401 * vtl0_mem.last_pfn is excluded in the pagemap range for VTL0 as per design. 402 * last_pfn is not reserved or wasted, and reflects 'start_pfn + size' of pagemap range. 403 */ 404 pgmap->ranges[0].start = PFN_PHYS(vtl0_mem.start_pfn); 405 pgmap->ranges[0].end = PFN_PHYS(vtl0_mem.last_pfn) - 1; 406 pgmap->nr_range = 1; 407 pgmap->type = MEMORY_DEVICE_GENERIC; 408 409 /* 410 * Determine the highest page order that can be used for the given memory range. 411 * This works best when the range is aligned; i.e. both the start and the length. 412 * Clamp to MAX_FOLIO_ORDER to avoid a WARN in memremap_pages() when the range 413 * alignment exceeds the maximum supported folio order for this kernel config. 414 */ 415 pgmap->vmemmap_shift = min(count_trailing_zeros(vtl0_mem.start_pfn | vtl0_mem.last_pfn), 416 MAX_FOLIO_ORDER); 417 dev_dbg(vtl->module_dev, 418 "Add VTL0 memory: start: 0x%llx, end_pfn: 0x%llx, page order: %lu\n", 419 vtl0_mem.start_pfn, vtl0_mem.last_pfn, pgmap->vmemmap_shift); 420 421 addr = devm_memremap_pages(mem_dev, pgmap); 422 if (IS_ERR(addr)) { 423 dev_err(vtl->module_dev, "devm_memremap_pages error: %ld\n", PTR_ERR(addr)); 424 kfree(pgmap); 425 return PTR_ERR(addr); 426 } 427 428 /* Don't free pgmap, since it has to stick around until the memory 429 * is unmapped, which will never happen as there is no scenario 430 * where VTL0 can be released/shutdown without bringing down VTL2. 431 */ 432 return 0; 433 } 434 435 static void mshv_vtl_cancel(int cpu) 436 { 437 int here = get_cpu(); 438 439 if (here != cpu) { 440 if (!xchg_relaxed(&mshv_vtl_cpu_run(cpu)->cancel, 1)) 441 smp_send_reschedule(cpu); 442 } else { 443 WRITE_ONCE(mshv_vtl_this_run()->cancel, 1); 444 } 445 put_cpu(); 446 } 447 448 static int mshv_vtl_poll_file_wake(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key) 449 { 450 struct mshv_vtl_poll_file *poll_file = container_of(wait, struct mshv_vtl_poll_file, wait); 451 452 mshv_vtl_cancel(poll_file->cpu); 453 454 return 0; 455 } 456 457 static void mshv_vtl_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, poll_table *pt) 458 { 459 struct mshv_vtl_poll_file *poll_file = container_of(pt, struct mshv_vtl_poll_file, pt); 460 461 WARN_ON(poll_file->wqh); 462 poll_file->wqh = wqh; 463 add_wait_queue(wqh, &poll_file->wait); 464 } 465 466 static int mshv_vtl_ioctl_set_poll_file(struct mshv_vtl_set_poll_file __user *user_input) 467 { 468 struct file *file, *old_file; 469 struct mshv_vtl_poll_file *poll_file; 470 struct mshv_vtl_set_poll_file input; 471 472 if (copy_from_user(&input, user_input, sizeof(input))) 473 return -EFAULT; 474 475 if (input.cpu >= num_possible_cpus() || !cpu_online(input.cpu)) 476 return -EINVAL; 477 /* 478 * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists. 479 * CPU is expected to remain online after above cpu_online() check. 480 */ 481 482 file = NULL; 483 file = fget(input.fd); 484 if (!file) 485 return -EBADFD; 486 487 poll_file = per_cpu_ptr(&mshv_vtl_poll_file, READ_ONCE(input.cpu)); 488 if (!poll_file) 489 return -EINVAL; 490 491 mutex_lock(&mshv_vtl_poll_file_lock); 492 493 if (poll_file->wqh) 494 remove_wait_queue(poll_file->wqh, &poll_file->wait); 495 poll_file->wqh = NULL; 496 497 old_file = poll_file->file; 498 poll_file->file = file; 499 poll_file->cpu = input.cpu; 500 501 if (file) { 502 init_waitqueue_func_entry(&poll_file->wait, mshv_vtl_poll_file_wake); 503 init_poll_funcptr(&poll_file->pt, mshv_vtl_ptable_queue_proc); 504 vfs_poll(file, &poll_file->pt); 505 } 506 507 mutex_unlock(&mshv_vtl_poll_file_lock); 508 509 if (old_file) 510 fput(old_file); 511 512 return 0; 513 } 514 515 /* Static table mapping register names to their corresponding actions */ 516 static const struct { 517 enum hv_register_name reg_name; 518 int debug_reg_num; /* -1 if not a debug register */ 519 u32 msr_addr; /* 0 if not an MSR */ 520 } reg_table[] = { 521 /* Debug registers */ 522 {HV_X64_REGISTER_DR0, 0, 0}, 523 {HV_X64_REGISTER_DR1, 1, 0}, 524 {HV_X64_REGISTER_DR2, 2, 0}, 525 {HV_X64_REGISTER_DR3, 3, 0}, 526 {HV_X64_REGISTER_DR6, 6, 0}, 527 /* MTRR MSRs */ 528 {HV_X64_REGISTER_MSR_MTRR_CAP, -1, MSR_MTRRcap}, 529 {HV_X64_REGISTER_MSR_MTRR_DEF_TYPE, -1, MSR_MTRRdefType}, 530 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0, -1, MTRRphysBase_MSR(0)}, 531 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE1, -1, MTRRphysBase_MSR(1)}, 532 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE2, -1, MTRRphysBase_MSR(2)}, 533 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE3, -1, MTRRphysBase_MSR(3)}, 534 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE4, -1, MTRRphysBase_MSR(4)}, 535 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE5, -1, MTRRphysBase_MSR(5)}, 536 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE6, -1, MTRRphysBase_MSR(6)}, 537 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE7, -1, MTRRphysBase_MSR(7)}, 538 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE8, -1, MTRRphysBase_MSR(8)}, 539 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE9, -1, MTRRphysBase_MSR(9)}, 540 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEA, -1, MTRRphysBase_MSR(0xa)}, 541 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEB, -1, MTRRphysBase_MSR(0xb)}, 542 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEC, -1, MTRRphysBase_MSR(0xc)}, 543 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASED, -1, MTRRphysBase_MSR(0xd)}, 544 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEE, -1, MTRRphysBase_MSR(0xe)}, 545 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEF, -1, MTRRphysBase_MSR(0xf)}, 546 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0, -1, MTRRphysMask_MSR(0)}, 547 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK1, -1, MTRRphysMask_MSR(1)}, 548 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK2, -1, MTRRphysMask_MSR(2)}, 549 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK3, -1, MTRRphysMask_MSR(3)}, 550 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK4, -1, MTRRphysMask_MSR(4)}, 551 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK5, -1, MTRRphysMask_MSR(5)}, 552 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK6, -1, MTRRphysMask_MSR(6)}, 553 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK7, -1, MTRRphysMask_MSR(7)}, 554 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK8, -1, MTRRphysMask_MSR(8)}, 555 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK9, -1, MTRRphysMask_MSR(9)}, 556 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKA, -1, MTRRphysMask_MSR(0xa)}, 557 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKB, -1, MTRRphysMask_MSR(0xb)}, 558 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKC, -1, MTRRphysMask_MSR(0xc)}, 559 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKD, -1, MTRRphysMask_MSR(0xd)}, 560 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKE, -1, MTRRphysMask_MSR(0xe)}, 561 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKF, -1, MTRRphysMask_MSR(0xf)}, 562 {HV_X64_REGISTER_MSR_MTRR_FIX64K00000, -1, MSR_MTRRfix64K_00000}, 563 {HV_X64_REGISTER_MSR_MTRR_FIX16K80000, -1, MSR_MTRRfix16K_80000}, 564 {HV_X64_REGISTER_MSR_MTRR_FIX16KA0000, -1, MSR_MTRRfix16K_A0000}, 565 {HV_X64_REGISTER_MSR_MTRR_FIX4KC0000, -1, MSR_MTRRfix4K_C0000}, 566 {HV_X64_REGISTER_MSR_MTRR_FIX4KC8000, -1, MSR_MTRRfix4K_C8000}, 567 {HV_X64_REGISTER_MSR_MTRR_FIX4KD0000, -1, MSR_MTRRfix4K_D0000}, 568 {HV_X64_REGISTER_MSR_MTRR_FIX4KD8000, -1, MSR_MTRRfix4K_D8000}, 569 {HV_X64_REGISTER_MSR_MTRR_FIX4KE0000, -1, MSR_MTRRfix4K_E0000}, 570 {HV_X64_REGISTER_MSR_MTRR_FIX4KE8000, -1, MSR_MTRRfix4K_E8000}, 571 {HV_X64_REGISTER_MSR_MTRR_FIX4KF0000, -1, MSR_MTRRfix4K_F0000}, 572 {HV_X64_REGISTER_MSR_MTRR_FIX4KF8000, -1, MSR_MTRRfix4K_F8000}, 573 }; 574 575 static int mshv_vtl_get_set_reg(struct hv_register_assoc *regs, bool set) 576 { 577 u64 *reg64; 578 enum hv_register_name gpr_name; 579 int i; 580 581 gpr_name = regs->name; 582 reg64 = ®s->value.reg64; 583 584 /* Search for the register in the table */ 585 for (i = 0; i < ARRAY_SIZE(reg_table); i++) { 586 if (reg_table[i].reg_name != gpr_name) 587 continue; 588 if (reg_table[i].debug_reg_num != -1) { 589 /* Handle debug registers */ 590 if (gpr_name == HV_X64_REGISTER_DR6 && 591 !mshv_vsm_capabilities.dr6_shared) 592 goto hypercall; 593 if (set) 594 native_set_debugreg(reg_table[i].debug_reg_num, *reg64); 595 else 596 *reg64 = native_get_debugreg(reg_table[i].debug_reg_num); 597 } else { 598 /* Handle MSRs */ 599 if (set) 600 wrmsrq(reg_table[i].msr_addr, *reg64); 601 else 602 rdmsrq(reg_table[i].msr_addr, *reg64); 603 } 604 return 0; 605 } 606 607 hypercall: 608 return 1; 609 } 610 611 static void mshv_vtl_return(struct mshv_vtl_cpu_context *vtl0) 612 { 613 struct hv_vp_assist_page *hvp; 614 615 hvp = hv_vp_assist_page[smp_processor_id()]; 616 617 /* 618 * Process signal event direct set in the run page, if any. 619 */ 620 if (mshv_vsm_capabilities.return_action_available) { 621 u32 offset = READ_ONCE(mshv_vtl_this_run()->vtl_ret_action_size); 622 623 WRITE_ONCE(mshv_vtl_this_run()->vtl_ret_action_size, 0); 624 625 /* 626 * Hypervisor will take care of clearing out the actions 627 * set in the assist page. 628 */ 629 memcpy(hvp->vtl_ret_actions, 630 mshv_vtl_this_run()->vtl_ret_actions, 631 min_t(u32, offset, sizeof(hvp->vtl_ret_actions))); 632 } 633 634 mshv_vtl_return_call(vtl0); 635 } 636 637 static bool mshv_vtl_process_intercept(void) 638 { 639 struct hv_per_cpu_context *mshv_cpu; 640 void *synic_message_page; 641 struct hv_message *msg; 642 u32 message_type; 643 644 mshv_cpu = this_cpu_ptr(hv_context.cpu_context); 645 synic_message_page = mshv_cpu->hyp_synic_message_page; 646 if (unlikely(!synic_message_page)) 647 return true; 648 649 msg = (struct hv_message *)synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX; 650 message_type = READ_ONCE(msg->header.message_type); 651 if (message_type == HVMSG_NONE) 652 return true; 653 654 memcpy(mshv_vtl_this_run()->exit_message, msg, sizeof(*msg)); 655 vmbus_signal_eom(msg, message_type); 656 657 return false; 658 } 659 660 static int mshv_vtl_ioctl_return_to_lower_vtl(void) 661 { 662 preempt_disable(); 663 for (;;) { 664 unsigned long irq_flags; 665 struct hv_vp_assist_page *hvp; 666 int ret; 667 668 if (__xfer_to_guest_mode_work_pending()) { 669 preempt_enable(); 670 ret = xfer_to_guest_mode_handle_work(); 671 if (ret) 672 return ret; 673 preempt_disable(); 674 } 675 676 local_irq_save(irq_flags); 677 if (READ_ONCE(mshv_vtl_this_run()->cancel)) { 678 local_irq_restore(irq_flags); 679 preempt_enable(); 680 return -EINTR; 681 } 682 683 mshv_vtl_return(&mshv_vtl_this_run()->cpu_context); 684 local_irq_restore(irq_flags); 685 686 hvp = hv_vp_assist_page[smp_processor_id()]; 687 this_cpu_inc(num_vtl0_transitions); 688 switch (hvp->vtl_entry_reason) { 689 case MSHV_ENTRY_REASON_INTERRUPT: 690 if (!mshv_vsm_capabilities.intercept_page_available && 691 likely(!mshv_vtl_process_intercept())) 692 goto done; 693 break; 694 695 case MSHV_ENTRY_REASON_INTERCEPT: 696 WARN_ON(!mshv_vsm_capabilities.intercept_page_available); 697 memcpy(mshv_vtl_this_run()->exit_message, hvp->intercept_message, 698 sizeof(hvp->intercept_message)); 699 goto done; 700 701 default: 702 panic("unknown entry reason: %d", hvp->vtl_entry_reason); 703 } 704 } 705 706 done: 707 preempt_enable(); 708 709 return 0; 710 } 711 712 static long 713 mshv_vtl_ioctl_get_regs(void __user *user_args) 714 { 715 struct mshv_vp_registers args; 716 struct hv_register_assoc reg; 717 long ret; 718 719 if (copy_from_user(&args, user_args, sizeof(args))) 720 return -EFAULT; 721 722 /* This IOCTL supports processing only one register at a time. */ 723 if (args.count != 1) 724 return -EINVAL; 725 726 if (copy_from_user(®, (void __user *)args.regs_ptr, 727 sizeof(reg))) 728 return -EFAULT; 729 730 ret = mshv_vtl_get_set_reg(®, false); 731 if (!ret) 732 goto copy_args; /* No need of hypercall */ 733 ret = vtl_get_vp_register(®); 734 if (ret) 735 return ret; 736 737 copy_args: 738 if (copy_to_user((void __user *)args.regs_ptr, ®, sizeof(reg))) 739 ret = -EFAULT; 740 741 return ret; 742 } 743 744 static long 745 mshv_vtl_ioctl_set_regs(void __user *user_args) 746 { 747 struct mshv_vp_registers args; 748 struct hv_register_assoc reg; 749 long ret; 750 751 if (copy_from_user(&args, user_args, sizeof(args))) 752 return -EFAULT; 753 754 /* This IOCTL supports processing only one register at a time. */ 755 if (args.count != 1) 756 return -EINVAL; 757 758 if (copy_from_user(®, (void __user *)args.regs_ptr, sizeof(reg))) 759 return -EFAULT; 760 761 ret = mshv_vtl_get_set_reg(®, true); 762 if (!ret) 763 return ret; /* No need of hypercall */ 764 ret = vtl_set_vp_register(®); 765 766 return ret; 767 } 768 769 static long 770 mshv_vtl_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) 771 { 772 long ret; 773 struct mshv_vtl *vtl = filp->private_data; 774 775 switch (ioctl) { 776 case MSHV_SET_POLL_FILE: 777 ret = mshv_vtl_ioctl_set_poll_file((struct mshv_vtl_set_poll_file __user *)arg); 778 break; 779 case MSHV_GET_VP_REGISTERS: 780 ret = mshv_vtl_ioctl_get_regs((void __user *)arg); 781 break; 782 case MSHV_SET_VP_REGISTERS: 783 ret = mshv_vtl_ioctl_set_regs((void __user *)arg); 784 break; 785 case MSHV_RETURN_TO_LOWER_VTL: 786 ret = mshv_vtl_ioctl_return_to_lower_vtl(); 787 break; 788 case MSHV_ADD_VTL0_MEMORY: 789 ret = mshv_vtl_ioctl_add_vtl0_mem(vtl, (void __user *)arg); 790 break; 791 default: 792 dev_err(vtl->module_dev, "invalid vtl ioctl: %#x\n", ioctl); 793 ret = -ENOTTY; 794 } 795 796 return ret; 797 } 798 799 static vm_fault_t mshv_vtl_fault(struct vm_fault *vmf) 800 { 801 struct page *page; 802 int cpu = vmf->pgoff & MSHV_PG_OFF_CPU_MASK; 803 int real_off = vmf->pgoff >> MSHV_REAL_OFF_SHIFT; 804 805 if (cpu >= nr_cpu_ids || !cpu_online(cpu)) 806 return VM_FAULT_SIGBUS; 807 /* 808 * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists. 809 * CPU is expected to remain online after above cpu_online() check. 810 */ 811 812 if (real_off == MSHV_RUN_PAGE_OFFSET) { 813 page = virt_to_page(mshv_vtl_cpu_run(cpu)); 814 } else if (real_off == MSHV_REG_PAGE_OFFSET) { 815 if (!mshv_has_reg_page) 816 return VM_FAULT_SIGBUS; 817 page = mshv_vtl_cpu_reg_page(cpu); 818 } else { 819 return VM_FAULT_NOPAGE; 820 } 821 822 if (!page) 823 return VM_FAULT_SIGBUS; 824 825 get_page(page); 826 vmf->page = page; 827 828 return 0; 829 } 830 831 static const struct vm_operations_struct mshv_vtl_vm_ops = { 832 .fault = mshv_vtl_fault, 833 }; 834 835 static int mshv_vtl_mmap(struct file *filp, struct vm_area_struct *vma) 836 { 837 vma->vm_ops = &mshv_vtl_vm_ops; 838 839 return 0; 840 } 841 842 static int mshv_vtl_release(struct inode *inode, struct file *filp) 843 { 844 struct mshv_vtl *vtl = filp->private_data; 845 846 kfree(vtl); 847 848 return 0; 849 } 850 851 static const struct file_operations mshv_vtl_fops = { 852 .owner = THIS_MODULE, 853 .unlocked_ioctl = mshv_vtl_ioctl, 854 .release = mshv_vtl_release, 855 .mmap = mshv_vtl_mmap, 856 }; 857 858 static void mshv_vtl_synic_mask_vmbus_sint(void *info) 859 { 860 union hv_synic_sint sint; 861 const u8 *mask = info; 862 863 sint.as_uint64 = 0; 864 sint.vector = HYPERVISOR_CALLBACK_VECTOR; 865 sint.masked = (*mask != 0); 866 sint.auto_eoi = hv_recommend_using_aeoi(); 867 868 hv_set_msr(HV_MSR_SINT0 + VTL2_VMBUS_SINT_INDEX, 869 sint.as_uint64); 870 871 if (!sint.masked) 872 pr_debug("%s: Unmasking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id()); 873 else 874 pr_debug("%s: Masking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id()); 875 } 876 877 static void mshv_vtl_read_remote(void *buffer) 878 { 879 struct hv_per_cpu_context *mshv_cpu = this_cpu_ptr(hv_context.cpu_context); 880 struct hv_message *msg = (struct hv_message *)mshv_cpu->hyp_synic_message_page + 881 VTL2_VMBUS_SINT_INDEX; 882 u32 message_type = READ_ONCE(msg->header.message_type); 883 884 WRITE_ONCE(has_message, false); 885 if (message_type == HVMSG_NONE) 886 return; 887 888 memcpy(buffer, msg, sizeof(*msg)); 889 vmbus_signal_eom(msg, message_type); 890 } 891 892 static bool vtl_synic_mask_vmbus_sint_masked = true; 893 894 static ssize_t mshv_vtl_sint_read(struct file *filp, char __user *arg, size_t size, loff_t *offset) 895 { 896 struct hv_message msg = {}; 897 int ret; 898 899 if (size < sizeof(msg)) 900 return -EINVAL; 901 902 for (;;) { 903 smp_call_function_single(VMBUS_CONNECT_CPU, mshv_vtl_read_remote, &msg, true); 904 if (msg.header.message_type != HVMSG_NONE) 905 break; 906 907 if (READ_ONCE(vtl_synic_mask_vmbus_sint_masked)) 908 return 0; /* EOF */ 909 910 if (filp->f_flags & O_NONBLOCK) 911 return -EAGAIN; 912 913 ret = wait_event_interruptible(fd_wait_queue, 914 READ_ONCE(has_message) || 915 READ_ONCE(vtl_synic_mask_vmbus_sint_masked)); 916 if (ret) 917 return ret; 918 } 919 920 if (copy_to_user(arg, &msg, sizeof(msg))) 921 return -EFAULT; 922 923 return sizeof(msg); 924 } 925 926 static __poll_t mshv_vtl_sint_poll(struct file *filp, poll_table *wait) 927 { 928 __poll_t mask = 0; 929 930 poll_wait(filp, &fd_wait_queue, wait); 931 if (READ_ONCE(has_message) || READ_ONCE(vtl_synic_mask_vmbus_sint_masked)) 932 mask |= EPOLLIN | EPOLLRDNORM; 933 934 return mask; 935 } 936 937 static void mshv_vtl_sint_on_msg_dpc(unsigned long data) 938 { 939 WRITE_ONCE(has_message, true); 940 wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN); 941 } 942 943 static int mshv_vtl_sint_ioctl_post_msg(struct mshv_vtl_sint_post_msg __user *arg) 944 { 945 struct mshv_vtl_sint_post_msg message; 946 u8 payload[HV_MESSAGE_PAYLOAD_BYTE_COUNT]; 947 948 if (copy_from_user(&message, arg, sizeof(message))) 949 return -EFAULT; 950 if (message.payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) 951 return -EINVAL; 952 if (copy_from_user(payload, (void __user *)message.payload_ptr, 953 message.payload_size)) 954 return -EFAULT; 955 956 return hv_post_message((union hv_connection_id)message.connection_id, 957 message.message_type, (void *)payload, 958 message.payload_size); 959 } 960 961 static int mshv_vtl_sint_ioctl_signal_event(struct mshv_vtl_signal_event __user *arg) 962 { 963 u64 input, status; 964 struct mshv_vtl_signal_event signal_event; 965 966 if (copy_from_user(&signal_event, arg, sizeof(signal_event))) 967 return -EFAULT; 968 969 input = signal_event.connection_id | ((u64)signal_event.flag << 32); 970 971 status = hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, input); 972 973 return hv_result_to_errno(status); 974 } 975 976 static int mshv_vtl_sint_ioctl_set_eventfd(struct mshv_vtl_set_eventfd __user *arg) 977 { 978 struct mshv_vtl_set_eventfd set_eventfd; 979 struct eventfd_ctx *eventfd, *old_eventfd; 980 981 if (copy_from_user(&set_eventfd, arg, sizeof(set_eventfd))) 982 return -EFAULT; 983 if (set_eventfd.flag >= HV_EVENT_FLAGS_COUNT) 984 return -EINVAL; 985 986 eventfd = NULL; 987 if (set_eventfd.fd >= 0) { 988 eventfd = eventfd_ctx_fdget(set_eventfd.fd); 989 if (IS_ERR(eventfd)) 990 return PTR_ERR(eventfd); 991 } 992 993 guard(mutex)(&flag_lock); 994 old_eventfd = READ_ONCE(flag_eventfds[set_eventfd.flag]); 995 WRITE_ONCE(flag_eventfds[set_eventfd.flag], eventfd); 996 997 if (old_eventfd) { 998 synchronize_rcu(); 999 eventfd_ctx_put(old_eventfd); 1000 } 1001 1002 return 0; 1003 } 1004 1005 static int mshv_vtl_sint_ioctl_pause_msg_stream(struct mshv_sint_mask __user *arg) 1006 { 1007 static DEFINE_MUTEX(vtl2_vmbus_sint_mask_mutex); 1008 struct mshv_sint_mask mask; 1009 1010 if (copy_from_user(&mask, arg, sizeof(mask))) 1011 return -EFAULT; 1012 guard(mutex)(&vtl2_vmbus_sint_mask_mutex); 1013 on_each_cpu(mshv_vtl_synic_mask_vmbus_sint, &mask.mask, 1); 1014 WRITE_ONCE(vtl_synic_mask_vmbus_sint_masked, mask.mask != 0); 1015 if (mask.mask) 1016 wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN); 1017 1018 return 0; 1019 } 1020 1021 static long mshv_vtl_sint_ioctl(struct file *f, unsigned int cmd, unsigned long arg) 1022 { 1023 switch (cmd) { 1024 case MSHV_SINT_POST_MESSAGE: 1025 return mshv_vtl_sint_ioctl_post_msg((struct mshv_vtl_sint_post_msg __user *)arg); 1026 case MSHV_SINT_SIGNAL_EVENT: 1027 return mshv_vtl_sint_ioctl_signal_event((struct mshv_vtl_signal_event __user *)arg); 1028 case MSHV_SINT_SET_EVENTFD: 1029 return mshv_vtl_sint_ioctl_set_eventfd((struct mshv_vtl_set_eventfd __user *)arg); 1030 case MSHV_SINT_PAUSE_MESSAGE_STREAM: 1031 return mshv_vtl_sint_ioctl_pause_msg_stream((struct mshv_sint_mask __user *)arg); 1032 default: 1033 return -ENOIOCTLCMD; 1034 } 1035 } 1036 1037 static const struct file_operations mshv_vtl_sint_ops = { 1038 .owner = THIS_MODULE, 1039 .read = mshv_vtl_sint_read, 1040 .poll = mshv_vtl_sint_poll, 1041 .unlocked_ioctl = mshv_vtl_sint_ioctl, 1042 }; 1043 1044 static struct miscdevice mshv_vtl_sint_dev = { 1045 .name = "mshv_sint", 1046 .fops = &mshv_vtl_sint_ops, 1047 .mode = 0600, 1048 .minor = MISC_DYNAMIC_MINOR, 1049 }; 1050 1051 static int mshv_vtl_hvcall_dev_open(struct inode *node, struct file *f) 1052 { 1053 struct miscdevice *dev = f->private_data; 1054 struct mshv_vtl_hvcall_fd *fd; 1055 1056 if (!capable(CAP_SYS_ADMIN)) 1057 return -EPERM; 1058 1059 fd = vzalloc(sizeof(*fd)); 1060 if (!fd) 1061 return -ENOMEM; 1062 fd->dev = dev; 1063 f->private_data = fd; 1064 mutex_init(&fd->init_mutex); 1065 1066 return 0; 1067 } 1068 1069 static int mshv_vtl_hvcall_dev_release(struct inode *node, struct file *f) 1070 { 1071 struct mshv_vtl_hvcall_fd *fd; 1072 1073 fd = f->private_data; 1074 if (fd) { 1075 vfree(fd); 1076 f->private_data = NULL; 1077 } 1078 1079 return 0; 1080 } 1081 1082 static int mshv_vtl_hvcall_do_setup(struct mshv_vtl_hvcall_fd *fd, 1083 struct mshv_vtl_hvcall_setup __user *hvcall_setup_user) 1084 { 1085 struct mshv_vtl_hvcall_setup hvcall_setup; 1086 1087 guard(mutex)(&fd->init_mutex); 1088 1089 if (fd->allow_map_initialized) { 1090 dev_err(fd->dev->this_device, 1091 "Hypercall allow map has already been set, pid %d\n", 1092 current->pid); 1093 return -EINVAL; 1094 } 1095 1096 if (copy_from_user(&hvcall_setup, hvcall_setup_user, 1097 sizeof(struct mshv_vtl_hvcall_setup))) { 1098 return -EFAULT; 1099 } 1100 if (hvcall_setup.bitmap_array_size > ARRAY_SIZE(fd->allow_bitmap)) 1101 return -EINVAL; 1102 1103 if (copy_from_user(&fd->allow_bitmap, 1104 (void __user *)hvcall_setup.allow_bitmap_ptr, 1105 hvcall_setup.bitmap_array_size)) { 1106 return -EFAULT; 1107 } 1108 1109 dev_info(fd->dev->this_device, "Hypercall allow map has been set, pid %d\n", 1110 current->pid); 1111 fd->allow_map_initialized = true; 1112 return 0; 1113 } 1114 1115 static bool mshv_vtl_hvcall_is_allowed(struct mshv_vtl_hvcall_fd *fd, u16 call_code) 1116 { 1117 return test_bit(call_code, (unsigned long *)fd->allow_bitmap); 1118 } 1119 1120 static int mshv_vtl_hvcall_call(struct mshv_vtl_hvcall_fd *fd, 1121 struct mshv_vtl_hvcall __user *hvcall_user) 1122 { 1123 struct mshv_vtl_hvcall hvcall; 1124 void *in, *out; 1125 int ret; 1126 1127 if (copy_from_user(&hvcall, hvcall_user, sizeof(struct mshv_vtl_hvcall))) 1128 return -EFAULT; 1129 if (hvcall.input_size > HV_HYP_PAGE_SIZE) 1130 return -EINVAL; 1131 if (hvcall.output_size > HV_HYP_PAGE_SIZE) 1132 return -EINVAL; 1133 1134 /* 1135 * By default, all hypercalls are not allowed. 1136 * The user mode code has to set up the allow bitmap once. 1137 */ 1138 1139 if (!mshv_vtl_hvcall_is_allowed(fd, hvcall.control & 0xFFFF)) { 1140 dev_err(fd->dev->this_device, 1141 "Hypercall with control data %#llx isn't allowed\n", 1142 hvcall.control); 1143 return -EPERM; 1144 } 1145 1146 /* 1147 * This may create a problem for Confidential VM (CVM) usecase where we need to use 1148 * Hyper-V driver allocated per-cpu input and output pages (hyperv_pcpu_input_arg and 1149 * hyperv_pcpu_output_arg) for making a hypervisor call. 1150 * 1151 * TODO: Take care of this when CVM support is added. 1152 */ 1153 in = (void *)__get_free_page(GFP_KERNEL); 1154 out = (void *)__get_free_page(GFP_KERNEL); 1155 if (!in || !out) { 1156 ret = -ENOMEM; 1157 goto free_pages; 1158 } 1159 1160 if (copy_from_user(in, (void __user *)hvcall.input_ptr, hvcall.input_size)) { 1161 ret = -EFAULT; 1162 goto free_pages; 1163 } 1164 1165 /* 1166 * The caller supplies output_size, so clear the range copied back to 1167 * userspace in case the hypercall writes fewer bytes than requested. 1168 */ 1169 memset(out, 0, hvcall.output_size); 1170 1171 hvcall.status = hv_do_hypercall(hvcall.control, in, out); 1172 1173 if (copy_to_user((void __user *)hvcall.output_ptr, out, hvcall.output_size)) { 1174 ret = -EFAULT; 1175 goto free_pages; 1176 } 1177 ret = put_user(hvcall.status, &hvcall_user->status); 1178 free_pages: 1179 free_page((unsigned long)in); 1180 free_page((unsigned long)out); 1181 1182 return ret; 1183 } 1184 1185 static long mshv_vtl_hvcall_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) 1186 { 1187 struct mshv_vtl_hvcall_fd *fd = f->private_data; 1188 1189 switch (cmd) { 1190 case MSHV_HVCALL_SETUP: 1191 return mshv_vtl_hvcall_do_setup(fd, (struct mshv_vtl_hvcall_setup __user *)arg); 1192 case MSHV_HVCALL: 1193 return mshv_vtl_hvcall_call(fd, (struct mshv_vtl_hvcall __user *)arg); 1194 default: 1195 break; 1196 } 1197 1198 return -ENOIOCTLCMD; 1199 } 1200 1201 static const struct file_operations mshv_vtl_hvcall_dev_file_ops = { 1202 .owner = THIS_MODULE, 1203 .open = mshv_vtl_hvcall_dev_open, 1204 .release = mshv_vtl_hvcall_dev_release, 1205 .unlocked_ioctl = mshv_vtl_hvcall_dev_ioctl, 1206 }; 1207 1208 static struct miscdevice mshv_vtl_hvcall_dev = { 1209 .name = "mshv_hvcall", 1210 .nodename = "mshv_hvcall", 1211 .fops = &mshv_vtl_hvcall_dev_file_ops, 1212 .mode = 0600, 1213 .minor = MISC_DYNAMIC_MINOR, 1214 }; 1215 1216 static int mshv_vtl_low_open(struct inode *inodep, struct file *filp) 1217 { 1218 pid_t pid = task_pid_vnr(current); 1219 uid_t uid = current_uid().val; 1220 int ret = 0; 1221 1222 pr_debug("%s: Opening VTL low, task group %d, uid %d\n", __func__, pid, uid); 1223 1224 if (capable(CAP_SYS_ADMIN)) { 1225 filp->private_data = inodep; 1226 } else { 1227 pr_err("%s: VTL low open failed: CAP_SYS_ADMIN required. task group %d, uid %d", 1228 __func__, pid, uid); 1229 ret = -EPERM; 1230 } 1231 1232 return ret; 1233 } 1234 1235 static bool can_fault(struct vm_fault *vmf, unsigned long size, unsigned long *pfn) 1236 { 1237 unsigned long mask = size - 1; 1238 unsigned long start = vmf->address & ~mask; 1239 unsigned long end = start + size; 1240 bool is_valid; 1241 1242 is_valid = (vmf->address & mask) == ((vmf->pgoff << PAGE_SHIFT) & mask) && 1243 start >= vmf->vma->vm_start && 1244 end <= vmf->vma->vm_end; 1245 1246 if (is_valid) 1247 *pfn = vmf->pgoff & ~(mask >> PAGE_SHIFT); 1248 1249 return is_valid; 1250 } 1251 1252 static vm_fault_t mshv_vtl_low_huge_fault(struct vm_fault *vmf, unsigned int order) 1253 { 1254 unsigned long pfn = vmf->pgoff; 1255 vm_fault_t ret = VM_FAULT_FALLBACK; 1256 1257 switch (order) { 1258 case 0: 1259 return vmf_insert_mixed(vmf->vma, vmf->address, pfn); 1260 1261 case PMD_ORDER: 1262 if (can_fault(vmf, PMD_SIZE, &pfn)) 1263 ret = vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); 1264 return ret; 1265 1266 case PUD_ORDER: 1267 if (can_fault(vmf, PUD_SIZE, &pfn)) 1268 ret = vmf_insert_pfn_pud(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); 1269 return ret; 1270 1271 default: 1272 return VM_FAULT_SIGBUS; 1273 } 1274 } 1275 1276 static vm_fault_t mshv_vtl_low_fault(struct vm_fault *vmf) 1277 { 1278 return mshv_vtl_low_huge_fault(vmf, 0); 1279 } 1280 1281 static const struct vm_operations_struct mshv_vtl_low_vm_ops = { 1282 .fault = mshv_vtl_low_fault, 1283 .huge_fault = mshv_vtl_low_huge_fault, 1284 }; 1285 1286 static int mshv_vtl_low_mmap(struct file *filp, struct vm_area_struct *vma) 1287 { 1288 vma->vm_ops = &mshv_vtl_low_vm_ops; 1289 vm_flags_set(vma, VM_HUGEPAGE | VM_MIXEDMAP); 1290 1291 return 0; 1292 } 1293 1294 static const struct file_operations mshv_vtl_low_file_ops = { 1295 .owner = THIS_MODULE, 1296 .open = mshv_vtl_low_open, 1297 .mmap = mshv_vtl_low_mmap, 1298 }; 1299 1300 static struct miscdevice mshv_vtl_low = { 1301 .name = "mshv_vtl_low", 1302 .nodename = "mshv_vtl_low", 1303 .fops = &mshv_vtl_low_file_ops, 1304 .mode = 0600, 1305 .minor = MISC_DYNAMIC_MINOR, 1306 }; 1307 1308 static int __init mshv_vtl_init(void) 1309 { 1310 int ret; 1311 struct device *dev = mshv_dev.this_device; 1312 1313 /* 1314 * This creates /dev/mshv which provides functionality to create VTLs and partitions. 1315 */ 1316 ret = misc_register(&mshv_dev); 1317 if (ret) { 1318 dev_err(dev, "mshv device register failed: %d\n", ret); 1319 goto free_dev; 1320 } 1321 1322 tasklet_init(&msg_dpc, mshv_vtl_sint_on_msg_dpc, 0); 1323 init_waitqueue_head(&fd_wait_queue); 1324 1325 if (mshv_vtl_get_vsm_regs()) { 1326 dev_emerg(dev, "Unable to get VSM capabilities !!\n"); 1327 ret = -ENODEV; 1328 goto free_dev; 1329 } 1330 if (mshv_vtl_configure_vsm_partition(dev)) { 1331 dev_emerg(dev, "VSM configuration failed !!\n"); 1332 ret = -ENODEV; 1333 goto free_dev; 1334 } 1335 1336 mshv_vtl_return_call_init(mshv_vsm_page_offsets.vtl_return_offset); 1337 ret = hv_vtl_setup_synic(); 1338 if (ret) 1339 goto free_dev; 1340 1341 /* 1342 * mshv_sint device adds VMBus relay ioctl support. 1343 * This provides a channel for VTL0 to communicate with VTL2. 1344 */ 1345 ret = misc_register(&mshv_vtl_sint_dev); 1346 if (ret) 1347 goto free_synic; 1348 1349 /* 1350 * mshv_hvcall device adds interface to enable userspace for direct hypercalls support. 1351 */ 1352 ret = misc_register(&mshv_vtl_hvcall_dev); 1353 if (ret) 1354 goto free_sint; 1355 1356 /* 1357 * mshv_vtl_low device is used to map VTL0 address space to a user-mode process in VTL2. 1358 * It implements mmap() to allow a user-mode process in VTL2 to map to the address of VTL0. 1359 */ 1360 ret = misc_register(&mshv_vtl_low); 1361 if (ret) 1362 goto free_hvcall; 1363 1364 /* 1365 * "mshv vtl mem dev" device is later used to setup VTL0 memory. 1366 */ 1367 mem_dev = kzalloc_obj(*mem_dev); 1368 if (!mem_dev) { 1369 ret = -ENOMEM; 1370 goto free_low; 1371 } 1372 1373 mutex_init(&mshv_vtl_poll_file_lock); 1374 1375 device_initialize(mem_dev); 1376 dev_set_name(mem_dev, "mshv vtl mem dev"); 1377 ret = device_add(mem_dev); 1378 if (ret) { 1379 dev_err(dev, "mshv vtl mem dev add: %d\n", ret); 1380 goto free_mem; 1381 } 1382 1383 return 0; 1384 1385 free_mem: 1386 kfree(mem_dev); 1387 free_low: 1388 misc_deregister(&mshv_vtl_low); 1389 free_hvcall: 1390 misc_deregister(&mshv_vtl_hvcall_dev); 1391 free_sint: 1392 misc_deregister(&mshv_vtl_sint_dev); 1393 free_synic: 1394 hv_vtl_remove_synic(); 1395 free_dev: 1396 misc_deregister(&mshv_dev); 1397 1398 return ret; 1399 } 1400 1401 static void __exit mshv_vtl_exit(void) 1402 { 1403 device_del(mem_dev); 1404 kfree(mem_dev); 1405 misc_deregister(&mshv_vtl_low); 1406 misc_deregister(&mshv_vtl_hvcall_dev); 1407 misc_deregister(&mshv_vtl_sint_dev); 1408 hv_vtl_remove_synic(); 1409 misc_deregister(&mshv_dev); 1410 } 1411 1412 module_init(mshv_vtl_init); 1413 module_exit(mshv_vtl_exit); 1414