1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2023, Microsoft Corporation.
4 *
5 * Author:
6 * Roman Kisel <romank@linux.microsoft.com>
7 * Saurabh Sengar <ssengar@linux.microsoft.com>
8 * Naman Jain <namjain@linux.microsoft.com>
9 */
10
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/miscdevice.h>
14 #include <linux/anon_inodes.h>
15 #include <linux/cpuhotplug.h>
16 #include <linux/count_zeros.h>
17 #include <linux/entry-virt.h>
18 #include <linux/eventfd.h>
19 #include <linux/poll.h>
20 #include <linux/file.h>
21 #include <linux/vmalloc.h>
22 #include <asm/debugreg.h>
23 #include <asm/mshyperv.h>
24 #include <trace/events/ipi.h>
25 #include <uapi/asm/mtrr.h>
26 #include <uapi/linux/mshv.h>
27 #include <hyperv/hvhdk.h>
28
29 #include "../../kernel/fpu/legacy.h"
30 #include "mshv.h"
31 #include "mshv_vtl.h"
32 #include "hyperv_vmbus.h"
33
34 MODULE_AUTHOR("Microsoft");
35 MODULE_LICENSE("GPL");
36 MODULE_DESCRIPTION("Microsoft Hyper-V VTL Driver");
37
38 #define MSHV_ENTRY_REASON_LOWER_VTL_CALL 0x1
39 #define MSHV_ENTRY_REASON_INTERRUPT 0x2
40 #define MSHV_ENTRY_REASON_INTERCEPT 0x3
41
42 #define MSHV_REAL_OFF_SHIFT 16
43 #define MSHV_PG_OFF_CPU_MASK (BIT_ULL(MSHV_REAL_OFF_SHIFT) - 1)
44 #define MSHV_RUN_PAGE_OFFSET 0
45 #define MSHV_REG_PAGE_OFFSET 1
46 #define VTL2_VMBUS_SINT_INDEX 7
47
48 static struct device *mem_dev;
49
50 static struct tasklet_struct msg_dpc;
51 static wait_queue_head_t fd_wait_queue;
52 static bool has_message;
53 static struct eventfd_ctx *flag_eventfds[HV_EVENT_FLAGS_COUNT];
54 static DEFINE_MUTEX(flag_lock);
55 static bool __read_mostly mshv_has_reg_page;
56
57 /* hvcall code is of type u16, allocate a bitmap of size (1 << 16) to accommodate it */
58 #define MAX_BITMAP_SIZE ((U16_MAX + 1) / 8)
59
60 struct mshv_vtl_hvcall_fd {
61 u8 allow_bitmap[MAX_BITMAP_SIZE];
62 bool allow_map_initialized;
63 /*
64 * Used to protect hvcall setup in IOCTLs
65 */
66 struct mutex init_mutex;
67 struct miscdevice *dev;
68 };
69
70 struct mshv_vtl_poll_file {
71 struct file *file;
72 wait_queue_entry_t wait;
73 wait_queue_head_t *wqh;
74 poll_table pt;
75 int cpu;
76 };
77
78 struct mshv_vtl {
79 struct device *module_dev;
80 u64 id;
81 };
82
83 struct mshv_vtl_per_cpu {
84 struct mshv_vtl_run *run;
85 struct page *reg_page;
86 };
87
88 /* SYNIC_OVERLAY_PAGE_MSR - internal, identical to hv_synic_simp */
89 union hv_synic_overlay_page_msr {
90 u64 as_uint64;
91 struct {
92 u64 enabled: 1;
93 u64 reserved: 11;
94 u64 pfn: 52;
95 } __packed;
96 };
97
98 static struct mutex mshv_vtl_poll_file_lock;
99 static union hv_register_vsm_page_offsets mshv_vsm_page_offsets;
100 static union hv_register_vsm_capabilities mshv_vsm_capabilities;
101
102 static DEFINE_PER_CPU(struct mshv_vtl_poll_file, mshv_vtl_poll_file);
103 static DEFINE_PER_CPU(unsigned long long, num_vtl0_transitions);
104 static DEFINE_PER_CPU(struct mshv_vtl_per_cpu, mshv_vtl_per_cpu);
105
106 static const union hv_input_vtl input_vtl_zero;
107 static const union hv_input_vtl input_vtl_normal = {
108 .use_target_vtl = 1,
109 };
110
111 static const struct file_operations mshv_vtl_fops;
112
113 static long
mshv_ioctl_create_vtl(void __user * user_arg,struct device * module_dev)114 mshv_ioctl_create_vtl(void __user *user_arg, struct device *module_dev)
115 {
116 struct mshv_vtl *vtl;
117 struct file *file;
118 int fd;
119
120 vtl = kzalloc_obj(*vtl);
121 if (!vtl)
122 return -ENOMEM;
123
124 fd = get_unused_fd_flags(O_CLOEXEC);
125 if (fd < 0) {
126 kfree(vtl);
127 return fd;
128 }
129 file = anon_inode_getfile("mshv_vtl", &mshv_vtl_fops,
130 vtl, O_RDWR);
131 if (IS_ERR(file)) {
132 put_unused_fd(fd);
133 kfree(vtl);
134 return PTR_ERR(file);
135 }
136 vtl->module_dev = module_dev;
137 fd_install(fd, file);
138
139 return fd;
140 }
141
142 static long
mshv_ioctl_check_extension(void __user * user_arg)143 mshv_ioctl_check_extension(void __user *user_arg)
144 {
145 u32 arg;
146
147 if (copy_from_user(&arg, user_arg, sizeof(arg)))
148 return -EFAULT;
149
150 switch (arg) {
151 case MSHV_CAP_CORE_API_STABLE:
152 return 0;
153 case MSHV_CAP_REGISTER_PAGE:
154 return mshv_has_reg_page;
155 case MSHV_CAP_VTL_RETURN_ACTION:
156 return mshv_vsm_capabilities.return_action_available;
157 case MSHV_CAP_DR6_SHARED:
158 return mshv_vsm_capabilities.dr6_shared;
159 }
160
161 return -EOPNOTSUPP;
162 }
163
164 static long
mshv_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)165 mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
166 {
167 struct miscdevice *misc = filp->private_data;
168
169 switch (ioctl) {
170 case MSHV_CHECK_EXTENSION:
171 return mshv_ioctl_check_extension((void __user *)arg);
172 case MSHV_CREATE_VTL:
173 return mshv_ioctl_create_vtl((void __user *)arg, misc->this_device);
174 }
175
176 return -ENOTTY;
177 }
178
179 static const struct file_operations mshv_dev_fops = {
180 .owner = THIS_MODULE,
181 .unlocked_ioctl = mshv_dev_ioctl,
182 .llseek = noop_llseek,
183 };
184
185 static struct miscdevice mshv_dev = {
186 .minor = MISC_DYNAMIC_MINOR,
187 .name = "mshv",
188 .fops = &mshv_dev_fops,
189 .mode = 0600,
190 };
191
mshv_vtl_this_run(void)192 static struct mshv_vtl_run *mshv_vtl_this_run(void)
193 {
194 return *this_cpu_ptr(&mshv_vtl_per_cpu.run);
195 }
196
mshv_vtl_cpu_run(int cpu)197 static struct mshv_vtl_run *mshv_vtl_cpu_run(int cpu)
198 {
199 return *per_cpu_ptr(&mshv_vtl_per_cpu.run, cpu);
200 }
201
mshv_vtl_cpu_reg_page(int cpu)202 static struct page *mshv_vtl_cpu_reg_page(int cpu)
203 {
204 return *per_cpu_ptr(&mshv_vtl_per_cpu.reg_page, cpu);
205 }
206
mshv_vtl_configure_reg_page(struct mshv_vtl_per_cpu * per_cpu)207 static void mshv_vtl_configure_reg_page(struct mshv_vtl_per_cpu *per_cpu)
208 {
209 struct hv_register_assoc reg_assoc = {};
210 union hv_synic_overlay_page_msr overlay = {};
211 struct page *reg_page;
212
213 reg_page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL);
214 if (!reg_page) {
215 WARN(1, "failed to allocate register page\n");
216 return;
217 }
218
219 overlay.enabled = 1;
220 overlay.pfn = page_to_hvpfn(reg_page);
221 reg_assoc.name = HV_X64_REGISTER_REG_PAGE;
222 reg_assoc.value.reg64 = overlay.as_uint64;
223
224 if (hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
225 1, input_vtl_zero, ®_assoc)) {
226 WARN(1, "failed to setup register page\n");
227 __free_page(reg_page);
228 return;
229 }
230
231 per_cpu->reg_page = reg_page;
232 mshv_has_reg_page = true;
233 }
234
mshv_vtl_synic_enable_regs(unsigned int cpu)235 static void mshv_vtl_synic_enable_regs(unsigned int cpu)
236 {
237 union hv_synic_sint sint;
238
239 sint.as_uint64 = 0;
240 sint.vector = HYPERVISOR_CALLBACK_VECTOR;
241 sint.masked = false;
242 sint.auto_eoi = hv_recommend_using_aeoi();
243
244 /* Enable intercepts */
245 if (!mshv_vsm_capabilities.intercept_page_available)
246 hv_set_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX,
247 sint.as_uint64);
248
249 /* VTL2 Host VSP SINT is (un)masked when the user mode requests that */
250 }
251
mshv_vtl_get_vsm_regs(void)252 static int mshv_vtl_get_vsm_regs(void)
253 {
254 struct hv_register_assoc registers[2];
255 int ret, count = 2;
256
257 registers[0].name = HV_REGISTER_VSM_CODE_PAGE_OFFSETS;
258 registers[1].name = HV_REGISTER_VSM_CAPABILITIES;
259
260 ret = hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
261 count, input_vtl_zero, registers);
262 if (ret)
263 return ret;
264
265 mshv_vsm_page_offsets.as_uint64 = registers[0].value.reg64;
266 mshv_vsm_capabilities.as_uint64 = registers[1].value.reg64;
267
268 return ret;
269 }
270
mshv_vtl_configure_vsm_partition(struct device * dev)271 static int mshv_vtl_configure_vsm_partition(struct device *dev)
272 {
273 union hv_register_vsm_partition_config config;
274 struct hv_register_assoc reg_assoc;
275
276 config.as_uint64 = 0;
277 config.default_vtl_protection_mask = HV_MAP_GPA_PERMISSIONS_MASK;
278 config.enable_vtl_protection = 1;
279 config.zero_memory_on_reset = 1;
280 config.intercept_vp_startup = 1;
281 config.intercept_cpuid_unimplemented = 1;
282
283 if (mshv_vsm_capabilities.intercept_page_available) {
284 dev_dbg(dev, "using intercept page\n");
285 config.intercept_page = 1;
286 }
287
288 reg_assoc.name = HV_REGISTER_VSM_PARTITION_CONFIG;
289 reg_assoc.value.reg64 = config.as_uint64;
290
291 return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
292 1, input_vtl_zero, ®_assoc);
293 }
294
mshv_vtl_vmbus_isr(void)295 static void mshv_vtl_vmbus_isr(void)
296 {
297 struct hv_per_cpu_context *per_cpu;
298 struct hv_message *msg;
299 u32 message_type;
300 union hv_synic_event_flags *event_flags;
301 struct eventfd_ctx *eventfd;
302 u16 i;
303
304 per_cpu = this_cpu_ptr(hv_context.cpu_context);
305 if (smp_processor_id() == 0) {
306 msg = (struct hv_message *)per_cpu->hyp_synic_message_page + VTL2_VMBUS_SINT_INDEX;
307 message_type = READ_ONCE(msg->header.message_type);
308 if (message_type != HVMSG_NONE)
309 tasklet_schedule(&msg_dpc);
310 }
311
312 event_flags = (union hv_synic_event_flags *)per_cpu->hyp_synic_event_page +
313 VTL2_VMBUS_SINT_INDEX;
314 for_each_set_bit(i, event_flags->flags, HV_EVENT_FLAGS_COUNT) {
315 if (!sync_test_and_clear_bit(i, event_flags->flags))
316 continue;
317 rcu_read_lock();
318 eventfd = READ_ONCE(flag_eventfds[i]);
319 if (eventfd)
320 eventfd_signal(eventfd);
321 rcu_read_unlock();
322 }
323
324 vmbus_isr();
325 }
326
mshv_vtl_alloc_context(unsigned int cpu)327 static int mshv_vtl_alloc_context(unsigned int cpu)
328 {
329 struct mshv_vtl_per_cpu *per_cpu = this_cpu_ptr(&mshv_vtl_per_cpu);
330
331 per_cpu->run = (struct mshv_vtl_run *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
332 if (!per_cpu->run)
333 return -ENOMEM;
334
335 if (mshv_vsm_capabilities.intercept_page_available)
336 mshv_vtl_configure_reg_page(per_cpu);
337
338 mshv_vtl_synic_enable_regs(cpu);
339
340 return 0;
341 }
342
343 static int mshv_vtl_cpuhp_online;
344
hv_vtl_setup_synic(void)345 static int hv_vtl_setup_synic(void)
346 {
347 int ret;
348
349 /* Use our isr to first filter out packets destined for userspace */
350 hv_setup_vmbus_handler(mshv_vtl_vmbus_isr);
351
352 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hyperv/vtl:online",
353 mshv_vtl_alloc_context, NULL);
354 if (ret < 0) {
355 hv_setup_vmbus_handler(vmbus_isr);
356 return ret;
357 }
358
359 mshv_vtl_cpuhp_online = ret;
360
361 return 0;
362 }
363
hv_vtl_remove_synic(void)364 static void hv_vtl_remove_synic(void)
365 {
366 cpuhp_remove_state(mshv_vtl_cpuhp_online);
367 hv_setup_vmbus_handler(vmbus_isr);
368 }
369
vtl_get_vp_register(struct hv_register_assoc * reg)370 static int vtl_get_vp_register(struct hv_register_assoc *reg)
371 {
372 return hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
373 1, input_vtl_normal, reg);
374 }
375
vtl_set_vp_register(struct hv_register_assoc * reg)376 static int vtl_set_vp_register(struct hv_register_assoc *reg)
377 {
378 return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
379 1, input_vtl_normal, reg);
380 }
381
mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl * vtl,void __user * arg)382 static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg)
383 {
384 struct mshv_vtl_ram_disposition vtl0_mem;
385 struct dev_pagemap *pgmap;
386 void *addr;
387
388 if (copy_from_user(&vtl0_mem, arg, sizeof(vtl0_mem)))
389 return -EFAULT;
390 if (vtl0_mem.last_pfn <= vtl0_mem.start_pfn) {
391 dev_err(vtl->module_dev, "range start pfn (%llx) > end pfn (%llx)\n",
392 vtl0_mem.start_pfn, vtl0_mem.last_pfn);
393 return -EFAULT;
394 }
395
396 pgmap = kzalloc_obj(*pgmap);
397 if (!pgmap)
398 return -ENOMEM;
399
400 /*
401 * vtl0_mem.last_pfn is excluded in the pagemap range for VTL0 as per design.
402 * last_pfn is not reserved or wasted, and reflects 'start_pfn + size' of pagemap range.
403 */
404 pgmap->ranges[0].start = PFN_PHYS(vtl0_mem.start_pfn);
405 pgmap->ranges[0].end = PFN_PHYS(vtl0_mem.last_pfn) - 1;
406 pgmap->nr_range = 1;
407 pgmap->type = MEMORY_DEVICE_GENERIC;
408
409 /*
410 * Determine the highest page order that can be used for the given memory range.
411 * This works best when the range is aligned; i.e. both the start and the length.
412 * Clamp to MAX_FOLIO_ORDER to avoid a WARN in memremap_pages() when the range
413 * alignment exceeds the maximum supported folio order for this kernel config.
414 */
415 pgmap->vmemmap_shift = min(count_trailing_zeros(vtl0_mem.start_pfn | vtl0_mem.last_pfn),
416 MAX_FOLIO_ORDER);
417 dev_dbg(vtl->module_dev,
418 "Add VTL0 memory: start: 0x%llx, end_pfn: 0x%llx, page order: %lu\n",
419 vtl0_mem.start_pfn, vtl0_mem.last_pfn, pgmap->vmemmap_shift);
420
421 addr = devm_memremap_pages(mem_dev, pgmap);
422 if (IS_ERR(addr)) {
423 dev_err(vtl->module_dev, "devm_memremap_pages error: %ld\n", PTR_ERR(addr));
424 kfree(pgmap);
425 return PTR_ERR(addr);
426 }
427
428 /* Don't free pgmap, since it has to stick around until the memory
429 * is unmapped, which will never happen as there is no scenario
430 * where VTL0 can be released/shutdown without bringing down VTL2.
431 */
432 return 0;
433 }
434
mshv_vtl_cancel(int cpu)435 static void mshv_vtl_cancel(int cpu)
436 {
437 int here = get_cpu();
438
439 if (here != cpu) {
440 if (!xchg_relaxed(&mshv_vtl_cpu_run(cpu)->cancel, 1))
441 smp_send_reschedule(cpu);
442 } else {
443 WRITE_ONCE(mshv_vtl_this_run()->cancel, 1);
444 }
445 put_cpu();
446 }
447
mshv_vtl_poll_file_wake(wait_queue_entry_t * wait,unsigned int mode,int sync,void * key)448 static int mshv_vtl_poll_file_wake(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key)
449 {
450 struct mshv_vtl_poll_file *poll_file = container_of(wait, struct mshv_vtl_poll_file, wait);
451
452 mshv_vtl_cancel(poll_file->cpu);
453
454 return 0;
455 }
456
mshv_vtl_ptable_queue_proc(struct file * file,wait_queue_head_t * wqh,poll_table * pt)457 static void mshv_vtl_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, poll_table *pt)
458 {
459 struct mshv_vtl_poll_file *poll_file = container_of(pt, struct mshv_vtl_poll_file, pt);
460
461 WARN_ON(poll_file->wqh);
462 poll_file->wqh = wqh;
463 add_wait_queue(wqh, &poll_file->wait);
464 }
465
mshv_vtl_ioctl_set_poll_file(struct mshv_vtl_set_poll_file __user * user_input)466 static int mshv_vtl_ioctl_set_poll_file(struct mshv_vtl_set_poll_file __user *user_input)
467 {
468 struct file *file, *old_file;
469 struct mshv_vtl_poll_file *poll_file;
470 struct mshv_vtl_set_poll_file input;
471
472 if (copy_from_user(&input, user_input, sizeof(input)))
473 return -EFAULT;
474
475 if (input.cpu >= num_possible_cpus() || !cpu_online(input.cpu))
476 return -EINVAL;
477 /*
478 * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists.
479 * CPU is expected to remain online after above cpu_online() check.
480 */
481
482 file = NULL;
483 file = fget(input.fd);
484 if (!file)
485 return -EBADFD;
486
487 poll_file = per_cpu_ptr(&mshv_vtl_poll_file, READ_ONCE(input.cpu));
488 if (!poll_file)
489 return -EINVAL;
490
491 mutex_lock(&mshv_vtl_poll_file_lock);
492
493 if (poll_file->wqh)
494 remove_wait_queue(poll_file->wqh, &poll_file->wait);
495 poll_file->wqh = NULL;
496
497 old_file = poll_file->file;
498 poll_file->file = file;
499 poll_file->cpu = input.cpu;
500
501 if (file) {
502 init_waitqueue_func_entry(&poll_file->wait, mshv_vtl_poll_file_wake);
503 init_poll_funcptr(&poll_file->pt, mshv_vtl_ptable_queue_proc);
504 vfs_poll(file, &poll_file->pt);
505 }
506
507 mutex_unlock(&mshv_vtl_poll_file_lock);
508
509 if (old_file)
510 fput(old_file);
511
512 return 0;
513 }
514
515 /* Static table mapping register names to their corresponding actions */
516 static const struct {
517 enum hv_register_name reg_name;
518 int debug_reg_num; /* -1 if not a debug register */
519 u32 msr_addr; /* 0 if not an MSR */
520 } reg_table[] = {
521 /* Debug registers */
522 {HV_X64_REGISTER_DR0, 0, 0},
523 {HV_X64_REGISTER_DR1, 1, 0},
524 {HV_X64_REGISTER_DR2, 2, 0},
525 {HV_X64_REGISTER_DR3, 3, 0},
526 {HV_X64_REGISTER_DR6, 6, 0},
527 /* MTRR MSRs */
528 {HV_X64_REGISTER_MSR_MTRR_CAP, -1, MSR_MTRRcap},
529 {HV_X64_REGISTER_MSR_MTRR_DEF_TYPE, -1, MSR_MTRRdefType},
530 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0, -1, MTRRphysBase_MSR(0)},
531 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE1, -1, MTRRphysBase_MSR(1)},
532 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE2, -1, MTRRphysBase_MSR(2)},
533 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE3, -1, MTRRphysBase_MSR(3)},
534 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE4, -1, MTRRphysBase_MSR(4)},
535 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE5, -1, MTRRphysBase_MSR(5)},
536 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE6, -1, MTRRphysBase_MSR(6)},
537 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE7, -1, MTRRphysBase_MSR(7)},
538 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE8, -1, MTRRphysBase_MSR(8)},
539 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE9, -1, MTRRphysBase_MSR(9)},
540 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEA, -1, MTRRphysBase_MSR(0xa)},
541 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEB, -1, MTRRphysBase_MSR(0xb)},
542 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEC, -1, MTRRphysBase_MSR(0xc)},
543 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASED, -1, MTRRphysBase_MSR(0xd)},
544 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEE, -1, MTRRphysBase_MSR(0xe)},
545 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEF, -1, MTRRphysBase_MSR(0xf)},
546 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0, -1, MTRRphysMask_MSR(0)},
547 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK1, -1, MTRRphysMask_MSR(1)},
548 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK2, -1, MTRRphysMask_MSR(2)},
549 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK3, -1, MTRRphysMask_MSR(3)},
550 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK4, -1, MTRRphysMask_MSR(4)},
551 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK5, -1, MTRRphysMask_MSR(5)},
552 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK6, -1, MTRRphysMask_MSR(6)},
553 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK7, -1, MTRRphysMask_MSR(7)},
554 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK8, -1, MTRRphysMask_MSR(8)},
555 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK9, -1, MTRRphysMask_MSR(9)},
556 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKA, -1, MTRRphysMask_MSR(0xa)},
557 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKB, -1, MTRRphysMask_MSR(0xb)},
558 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKC, -1, MTRRphysMask_MSR(0xc)},
559 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKD, -1, MTRRphysMask_MSR(0xd)},
560 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKE, -1, MTRRphysMask_MSR(0xe)},
561 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKF, -1, MTRRphysMask_MSR(0xf)},
562 {HV_X64_REGISTER_MSR_MTRR_FIX64K00000, -1, MSR_MTRRfix64K_00000},
563 {HV_X64_REGISTER_MSR_MTRR_FIX16K80000, -1, MSR_MTRRfix16K_80000},
564 {HV_X64_REGISTER_MSR_MTRR_FIX16KA0000, -1, MSR_MTRRfix16K_A0000},
565 {HV_X64_REGISTER_MSR_MTRR_FIX4KC0000, -1, MSR_MTRRfix4K_C0000},
566 {HV_X64_REGISTER_MSR_MTRR_FIX4KC8000, -1, MSR_MTRRfix4K_C8000},
567 {HV_X64_REGISTER_MSR_MTRR_FIX4KD0000, -1, MSR_MTRRfix4K_D0000},
568 {HV_X64_REGISTER_MSR_MTRR_FIX4KD8000, -1, MSR_MTRRfix4K_D8000},
569 {HV_X64_REGISTER_MSR_MTRR_FIX4KE0000, -1, MSR_MTRRfix4K_E0000},
570 {HV_X64_REGISTER_MSR_MTRR_FIX4KE8000, -1, MSR_MTRRfix4K_E8000},
571 {HV_X64_REGISTER_MSR_MTRR_FIX4KF0000, -1, MSR_MTRRfix4K_F0000},
572 {HV_X64_REGISTER_MSR_MTRR_FIX4KF8000, -1, MSR_MTRRfix4K_F8000},
573 };
574
mshv_vtl_get_set_reg(struct hv_register_assoc * regs,bool set)575 static int mshv_vtl_get_set_reg(struct hv_register_assoc *regs, bool set)
576 {
577 u64 *reg64;
578 enum hv_register_name gpr_name;
579 int i;
580
581 gpr_name = regs->name;
582 reg64 = ®s->value.reg64;
583
584 /* Search for the register in the table */
585 for (i = 0; i < ARRAY_SIZE(reg_table); i++) {
586 if (reg_table[i].reg_name != gpr_name)
587 continue;
588 if (reg_table[i].debug_reg_num != -1) {
589 /* Handle debug registers */
590 if (gpr_name == HV_X64_REGISTER_DR6 &&
591 !mshv_vsm_capabilities.dr6_shared)
592 goto hypercall;
593 if (set)
594 native_set_debugreg(reg_table[i].debug_reg_num, *reg64);
595 else
596 *reg64 = native_get_debugreg(reg_table[i].debug_reg_num);
597 } else {
598 /* Handle MSRs */
599 if (set)
600 wrmsrq(reg_table[i].msr_addr, *reg64);
601 else
602 rdmsrq(reg_table[i].msr_addr, *reg64);
603 }
604 return 0;
605 }
606
607 hypercall:
608 return 1;
609 }
610
mshv_vtl_return(struct mshv_vtl_cpu_context * vtl0)611 static void mshv_vtl_return(struct mshv_vtl_cpu_context *vtl0)
612 {
613 struct hv_vp_assist_page *hvp;
614
615 hvp = hv_vp_assist_page[smp_processor_id()];
616
617 /*
618 * Process signal event direct set in the run page, if any.
619 */
620 if (mshv_vsm_capabilities.return_action_available) {
621 u32 offset = READ_ONCE(mshv_vtl_this_run()->vtl_ret_action_size);
622
623 WRITE_ONCE(mshv_vtl_this_run()->vtl_ret_action_size, 0);
624
625 /*
626 * Hypervisor will take care of clearing out the actions
627 * set in the assist page.
628 */
629 memcpy(hvp->vtl_ret_actions,
630 mshv_vtl_this_run()->vtl_ret_actions,
631 min_t(u32, offset, sizeof(hvp->vtl_ret_actions)));
632 }
633
634 mshv_vtl_return_call(vtl0);
635 }
636
mshv_vtl_process_intercept(void)637 static bool mshv_vtl_process_intercept(void)
638 {
639 struct hv_per_cpu_context *mshv_cpu;
640 void *synic_message_page;
641 struct hv_message *msg;
642 u32 message_type;
643
644 mshv_cpu = this_cpu_ptr(hv_context.cpu_context);
645 synic_message_page = mshv_cpu->hyp_synic_message_page;
646 if (unlikely(!synic_message_page))
647 return true;
648
649 msg = (struct hv_message *)synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX;
650 message_type = READ_ONCE(msg->header.message_type);
651 if (message_type == HVMSG_NONE)
652 return true;
653
654 memcpy(mshv_vtl_this_run()->exit_message, msg, sizeof(*msg));
655 vmbus_signal_eom(msg, message_type);
656
657 return false;
658 }
659
mshv_vtl_ioctl_return_to_lower_vtl(void)660 static int mshv_vtl_ioctl_return_to_lower_vtl(void)
661 {
662 preempt_disable();
663 for (;;) {
664 unsigned long irq_flags;
665 struct hv_vp_assist_page *hvp;
666 int ret;
667
668 if (__xfer_to_guest_mode_work_pending()) {
669 preempt_enable();
670 ret = xfer_to_guest_mode_handle_work();
671 if (ret)
672 return ret;
673 preempt_disable();
674 }
675
676 local_irq_save(irq_flags);
677 if (READ_ONCE(mshv_vtl_this_run()->cancel)) {
678 local_irq_restore(irq_flags);
679 preempt_enable();
680 return -EINTR;
681 }
682
683 mshv_vtl_return(&mshv_vtl_this_run()->cpu_context);
684 local_irq_restore(irq_flags);
685
686 hvp = hv_vp_assist_page[smp_processor_id()];
687 this_cpu_inc(num_vtl0_transitions);
688 switch (hvp->vtl_entry_reason) {
689 case MSHV_ENTRY_REASON_INTERRUPT:
690 if (!mshv_vsm_capabilities.intercept_page_available &&
691 likely(!mshv_vtl_process_intercept()))
692 goto done;
693 break;
694
695 case MSHV_ENTRY_REASON_INTERCEPT:
696 WARN_ON(!mshv_vsm_capabilities.intercept_page_available);
697 memcpy(mshv_vtl_this_run()->exit_message, hvp->intercept_message,
698 sizeof(hvp->intercept_message));
699 goto done;
700
701 default:
702 panic("unknown entry reason: %d", hvp->vtl_entry_reason);
703 }
704 }
705
706 done:
707 preempt_enable();
708
709 return 0;
710 }
711
712 static long
mshv_vtl_ioctl_get_regs(void __user * user_args)713 mshv_vtl_ioctl_get_regs(void __user *user_args)
714 {
715 struct mshv_vp_registers args;
716 struct hv_register_assoc reg;
717 long ret;
718
719 if (copy_from_user(&args, user_args, sizeof(args)))
720 return -EFAULT;
721
722 /* This IOCTL supports processing only one register at a time. */
723 if (args.count != 1)
724 return -EINVAL;
725
726 if (copy_from_user(®, (void __user *)args.regs_ptr,
727 sizeof(reg)))
728 return -EFAULT;
729
730 ret = mshv_vtl_get_set_reg(®, false);
731 if (!ret)
732 goto copy_args; /* No need of hypercall */
733 ret = vtl_get_vp_register(®);
734 if (ret)
735 return ret;
736
737 copy_args:
738 if (copy_to_user((void __user *)args.regs_ptr, ®, sizeof(reg)))
739 ret = -EFAULT;
740
741 return ret;
742 }
743
744 static long
mshv_vtl_ioctl_set_regs(void __user * user_args)745 mshv_vtl_ioctl_set_regs(void __user *user_args)
746 {
747 struct mshv_vp_registers args;
748 struct hv_register_assoc reg;
749 long ret;
750
751 if (copy_from_user(&args, user_args, sizeof(args)))
752 return -EFAULT;
753
754 /* This IOCTL supports processing only one register at a time. */
755 if (args.count != 1)
756 return -EINVAL;
757
758 if (copy_from_user(®, (void __user *)args.regs_ptr, sizeof(reg)))
759 return -EFAULT;
760
761 ret = mshv_vtl_get_set_reg(®, true);
762 if (!ret)
763 return ret; /* No need of hypercall */
764 ret = vtl_set_vp_register(®);
765
766 return ret;
767 }
768
769 static long
mshv_vtl_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)770 mshv_vtl_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
771 {
772 long ret;
773 struct mshv_vtl *vtl = filp->private_data;
774
775 switch (ioctl) {
776 case MSHV_SET_POLL_FILE:
777 ret = mshv_vtl_ioctl_set_poll_file((struct mshv_vtl_set_poll_file __user *)arg);
778 break;
779 case MSHV_GET_VP_REGISTERS:
780 ret = mshv_vtl_ioctl_get_regs((void __user *)arg);
781 break;
782 case MSHV_SET_VP_REGISTERS:
783 ret = mshv_vtl_ioctl_set_regs((void __user *)arg);
784 break;
785 case MSHV_RETURN_TO_LOWER_VTL:
786 ret = mshv_vtl_ioctl_return_to_lower_vtl();
787 break;
788 case MSHV_ADD_VTL0_MEMORY:
789 ret = mshv_vtl_ioctl_add_vtl0_mem(vtl, (void __user *)arg);
790 break;
791 default:
792 dev_err(vtl->module_dev, "invalid vtl ioctl: %#x\n", ioctl);
793 ret = -ENOTTY;
794 }
795
796 return ret;
797 }
798
mshv_vtl_fault(struct vm_fault * vmf)799 static vm_fault_t mshv_vtl_fault(struct vm_fault *vmf)
800 {
801 struct page *page;
802 int cpu = vmf->pgoff & MSHV_PG_OFF_CPU_MASK;
803 int real_off = vmf->pgoff >> MSHV_REAL_OFF_SHIFT;
804
805 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
806 return VM_FAULT_SIGBUS;
807 /*
808 * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists.
809 * CPU is expected to remain online after above cpu_online() check.
810 */
811
812 if (real_off == MSHV_RUN_PAGE_OFFSET) {
813 page = virt_to_page(mshv_vtl_cpu_run(cpu));
814 } else if (real_off == MSHV_REG_PAGE_OFFSET) {
815 if (!mshv_has_reg_page)
816 return VM_FAULT_SIGBUS;
817 page = mshv_vtl_cpu_reg_page(cpu);
818 } else {
819 return VM_FAULT_NOPAGE;
820 }
821
822 if (!page)
823 return VM_FAULT_SIGBUS;
824
825 get_page(page);
826 vmf->page = page;
827
828 return 0;
829 }
830
831 static const struct vm_operations_struct mshv_vtl_vm_ops = {
832 .fault = mshv_vtl_fault,
833 };
834
mshv_vtl_mmap(struct file * filp,struct vm_area_struct * vma)835 static int mshv_vtl_mmap(struct file *filp, struct vm_area_struct *vma)
836 {
837 vma->vm_ops = &mshv_vtl_vm_ops;
838
839 return 0;
840 }
841
mshv_vtl_release(struct inode * inode,struct file * filp)842 static int mshv_vtl_release(struct inode *inode, struct file *filp)
843 {
844 struct mshv_vtl *vtl = filp->private_data;
845
846 kfree(vtl);
847
848 return 0;
849 }
850
851 static const struct file_operations mshv_vtl_fops = {
852 .owner = THIS_MODULE,
853 .unlocked_ioctl = mshv_vtl_ioctl,
854 .release = mshv_vtl_release,
855 .mmap = mshv_vtl_mmap,
856 };
857
mshv_vtl_synic_mask_vmbus_sint(void * info)858 static void mshv_vtl_synic_mask_vmbus_sint(void *info)
859 {
860 union hv_synic_sint sint;
861 const u8 *mask = info;
862
863 sint.as_uint64 = 0;
864 sint.vector = HYPERVISOR_CALLBACK_VECTOR;
865 sint.masked = (*mask != 0);
866 sint.auto_eoi = hv_recommend_using_aeoi();
867
868 hv_set_msr(HV_MSR_SINT0 + VTL2_VMBUS_SINT_INDEX,
869 sint.as_uint64);
870
871 if (!sint.masked)
872 pr_debug("%s: Unmasking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id());
873 else
874 pr_debug("%s: Masking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id());
875 }
876
mshv_vtl_read_remote(void * buffer)877 static void mshv_vtl_read_remote(void *buffer)
878 {
879 struct hv_per_cpu_context *mshv_cpu = this_cpu_ptr(hv_context.cpu_context);
880 struct hv_message *msg = (struct hv_message *)mshv_cpu->hyp_synic_message_page +
881 VTL2_VMBUS_SINT_INDEX;
882 u32 message_type = READ_ONCE(msg->header.message_type);
883
884 WRITE_ONCE(has_message, false);
885 if (message_type == HVMSG_NONE)
886 return;
887
888 memcpy(buffer, msg, sizeof(*msg));
889 vmbus_signal_eom(msg, message_type);
890 }
891
892 static bool vtl_synic_mask_vmbus_sint_masked = true;
893
mshv_vtl_sint_read(struct file * filp,char __user * arg,size_t size,loff_t * offset)894 static ssize_t mshv_vtl_sint_read(struct file *filp, char __user *arg, size_t size, loff_t *offset)
895 {
896 struct hv_message msg = {};
897 int ret;
898
899 if (size < sizeof(msg))
900 return -EINVAL;
901
902 for (;;) {
903 smp_call_function_single(VMBUS_CONNECT_CPU, mshv_vtl_read_remote, &msg, true);
904 if (msg.header.message_type != HVMSG_NONE)
905 break;
906
907 if (READ_ONCE(vtl_synic_mask_vmbus_sint_masked))
908 return 0; /* EOF */
909
910 if (filp->f_flags & O_NONBLOCK)
911 return -EAGAIN;
912
913 ret = wait_event_interruptible(fd_wait_queue,
914 READ_ONCE(has_message) ||
915 READ_ONCE(vtl_synic_mask_vmbus_sint_masked));
916 if (ret)
917 return ret;
918 }
919
920 if (copy_to_user(arg, &msg, sizeof(msg)))
921 return -EFAULT;
922
923 return sizeof(msg);
924 }
925
mshv_vtl_sint_poll(struct file * filp,poll_table * wait)926 static __poll_t mshv_vtl_sint_poll(struct file *filp, poll_table *wait)
927 {
928 __poll_t mask = 0;
929
930 poll_wait(filp, &fd_wait_queue, wait);
931 if (READ_ONCE(has_message) || READ_ONCE(vtl_synic_mask_vmbus_sint_masked))
932 mask |= EPOLLIN | EPOLLRDNORM;
933
934 return mask;
935 }
936
mshv_vtl_sint_on_msg_dpc(unsigned long data)937 static void mshv_vtl_sint_on_msg_dpc(unsigned long data)
938 {
939 WRITE_ONCE(has_message, true);
940 wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN);
941 }
942
mshv_vtl_sint_ioctl_post_msg(struct mshv_vtl_sint_post_msg __user * arg)943 static int mshv_vtl_sint_ioctl_post_msg(struct mshv_vtl_sint_post_msg __user *arg)
944 {
945 struct mshv_vtl_sint_post_msg message;
946 u8 payload[HV_MESSAGE_PAYLOAD_BYTE_COUNT];
947
948 if (copy_from_user(&message, arg, sizeof(message)))
949 return -EFAULT;
950 if (message.payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT)
951 return -EINVAL;
952 if (copy_from_user(payload, (void __user *)message.payload_ptr,
953 message.payload_size))
954 return -EFAULT;
955
956 return hv_post_message((union hv_connection_id)message.connection_id,
957 message.message_type, (void *)payload,
958 message.payload_size);
959 }
960
mshv_vtl_sint_ioctl_signal_event(struct mshv_vtl_signal_event __user * arg)961 static int mshv_vtl_sint_ioctl_signal_event(struct mshv_vtl_signal_event __user *arg)
962 {
963 u64 input, status;
964 struct mshv_vtl_signal_event signal_event;
965
966 if (copy_from_user(&signal_event, arg, sizeof(signal_event)))
967 return -EFAULT;
968
969 input = signal_event.connection_id | ((u64)signal_event.flag << 32);
970
971 status = hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, input);
972
973 return hv_result_to_errno(status);
974 }
975
mshv_vtl_sint_ioctl_set_eventfd(struct mshv_vtl_set_eventfd __user * arg)976 static int mshv_vtl_sint_ioctl_set_eventfd(struct mshv_vtl_set_eventfd __user *arg)
977 {
978 struct mshv_vtl_set_eventfd set_eventfd;
979 struct eventfd_ctx *eventfd, *old_eventfd;
980
981 if (copy_from_user(&set_eventfd, arg, sizeof(set_eventfd)))
982 return -EFAULT;
983 if (set_eventfd.flag >= HV_EVENT_FLAGS_COUNT)
984 return -EINVAL;
985
986 eventfd = NULL;
987 if (set_eventfd.fd >= 0) {
988 eventfd = eventfd_ctx_fdget(set_eventfd.fd);
989 if (IS_ERR(eventfd))
990 return PTR_ERR(eventfd);
991 }
992
993 guard(mutex)(&flag_lock);
994 old_eventfd = READ_ONCE(flag_eventfds[set_eventfd.flag]);
995 WRITE_ONCE(flag_eventfds[set_eventfd.flag], eventfd);
996
997 if (old_eventfd) {
998 synchronize_rcu();
999 eventfd_ctx_put(old_eventfd);
1000 }
1001
1002 return 0;
1003 }
1004
mshv_vtl_sint_ioctl_pause_msg_stream(struct mshv_sint_mask __user * arg)1005 static int mshv_vtl_sint_ioctl_pause_msg_stream(struct mshv_sint_mask __user *arg)
1006 {
1007 static DEFINE_MUTEX(vtl2_vmbus_sint_mask_mutex);
1008 struct mshv_sint_mask mask;
1009
1010 if (copy_from_user(&mask, arg, sizeof(mask)))
1011 return -EFAULT;
1012 guard(mutex)(&vtl2_vmbus_sint_mask_mutex);
1013 on_each_cpu(mshv_vtl_synic_mask_vmbus_sint, &mask.mask, 1);
1014 WRITE_ONCE(vtl_synic_mask_vmbus_sint_masked, mask.mask != 0);
1015 if (mask.mask)
1016 wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN);
1017
1018 return 0;
1019 }
1020
mshv_vtl_sint_ioctl(struct file * f,unsigned int cmd,unsigned long arg)1021 static long mshv_vtl_sint_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1022 {
1023 switch (cmd) {
1024 case MSHV_SINT_POST_MESSAGE:
1025 return mshv_vtl_sint_ioctl_post_msg((struct mshv_vtl_sint_post_msg __user *)arg);
1026 case MSHV_SINT_SIGNAL_EVENT:
1027 return mshv_vtl_sint_ioctl_signal_event((struct mshv_vtl_signal_event __user *)arg);
1028 case MSHV_SINT_SET_EVENTFD:
1029 return mshv_vtl_sint_ioctl_set_eventfd((struct mshv_vtl_set_eventfd __user *)arg);
1030 case MSHV_SINT_PAUSE_MESSAGE_STREAM:
1031 return mshv_vtl_sint_ioctl_pause_msg_stream((struct mshv_sint_mask __user *)arg);
1032 default:
1033 return -ENOIOCTLCMD;
1034 }
1035 }
1036
1037 static const struct file_operations mshv_vtl_sint_ops = {
1038 .owner = THIS_MODULE,
1039 .read = mshv_vtl_sint_read,
1040 .poll = mshv_vtl_sint_poll,
1041 .unlocked_ioctl = mshv_vtl_sint_ioctl,
1042 };
1043
1044 static struct miscdevice mshv_vtl_sint_dev = {
1045 .name = "mshv_sint",
1046 .fops = &mshv_vtl_sint_ops,
1047 .mode = 0600,
1048 .minor = MISC_DYNAMIC_MINOR,
1049 };
1050
mshv_vtl_hvcall_dev_open(struct inode * node,struct file * f)1051 static int mshv_vtl_hvcall_dev_open(struct inode *node, struct file *f)
1052 {
1053 struct miscdevice *dev = f->private_data;
1054 struct mshv_vtl_hvcall_fd *fd;
1055
1056 if (!capable(CAP_SYS_ADMIN))
1057 return -EPERM;
1058
1059 fd = vzalloc(sizeof(*fd));
1060 if (!fd)
1061 return -ENOMEM;
1062 fd->dev = dev;
1063 f->private_data = fd;
1064 mutex_init(&fd->init_mutex);
1065
1066 return 0;
1067 }
1068
mshv_vtl_hvcall_dev_release(struct inode * node,struct file * f)1069 static int mshv_vtl_hvcall_dev_release(struct inode *node, struct file *f)
1070 {
1071 struct mshv_vtl_hvcall_fd *fd;
1072
1073 fd = f->private_data;
1074 if (fd) {
1075 vfree(fd);
1076 f->private_data = NULL;
1077 }
1078
1079 return 0;
1080 }
1081
mshv_vtl_hvcall_do_setup(struct mshv_vtl_hvcall_fd * fd,struct mshv_vtl_hvcall_setup __user * hvcall_setup_user)1082 static int mshv_vtl_hvcall_do_setup(struct mshv_vtl_hvcall_fd *fd,
1083 struct mshv_vtl_hvcall_setup __user *hvcall_setup_user)
1084 {
1085 struct mshv_vtl_hvcall_setup hvcall_setup;
1086
1087 guard(mutex)(&fd->init_mutex);
1088
1089 if (fd->allow_map_initialized) {
1090 dev_err(fd->dev->this_device,
1091 "Hypercall allow map has already been set, pid %d\n",
1092 current->pid);
1093 return -EINVAL;
1094 }
1095
1096 if (copy_from_user(&hvcall_setup, hvcall_setup_user,
1097 sizeof(struct mshv_vtl_hvcall_setup))) {
1098 return -EFAULT;
1099 }
1100 if (hvcall_setup.bitmap_array_size > ARRAY_SIZE(fd->allow_bitmap))
1101 return -EINVAL;
1102
1103 if (copy_from_user(&fd->allow_bitmap,
1104 (void __user *)hvcall_setup.allow_bitmap_ptr,
1105 hvcall_setup.bitmap_array_size)) {
1106 return -EFAULT;
1107 }
1108
1109 dev_info(fd->dev->this_device, "Hypercall allow map has been set, pid %d\n",
1110 current->pid);
1111 fd->allow_map_initialized = true;
1112 return 0;
1113 }
1114
mshv_vtl_hvcall_is_allowed(struct mshv_vtl_hvcall_fd * fd,u16 call_code)1115 static bool mshv_vtl_hvcall_is_allowed(struct mshv_vtl_hvcall_fd *fd, u16 call_code)
1116 {
1117 return test_bit(call_code, (unsigned long *)fd->allow_bitmap);
1118 }
1119
mshv_vtl_hvcall_call(struct mshv_vtl_hvcall_fd * fd,struct mshv_vtl_hvcall __user * hvcall_user)1120 static int mshv_vtl_hvcall_call(struct mshv_vtl_hvcall_fd *fd,
1121 struct mshv_vtl_hvcall __user *hvcall_user)
1122 {
1123 struct mshv_vtl_hvcall hvcall;
1124 void *in, *out;
1125 int ret;
1126
1127 if (copy_from_user(&hvcall, hvcall_user, sizeof(struct mshv_vtl_hvcall)))
1128 return -EFAULT;
1129 if (hvcall.input_size > HV_HYP_PAGE_SIZE)
1130 return -EINVAL;
1131 if (hvcall.output_size > HV_HYP_PAGE_SIZE)
1132 return -EINVAL;
1133
1134 /*
1135 * By default, all hypercalls are not allowed.
1136 * The user mode code has to set up the allow bitmap once.
1137 */
1138
1139 if (!mshv_vtl_hvcall_is_allowed(fd, hvcall.control & 0xFFFF)) {
1140 dev_err(fd->dev->this_device,
1141 "Hypercall with control data %#llx isn't allowed\n",
1142 hvcall.control);
1143 return -EPERM;
1144 }
1145
1146 /*
1147 * This may create a problem for Confidential VM (CVM) usecase where we need to use
1148 * Hyper-V driver allocated per-cpu input and output pages (hyperv_pcpu_input_arg and
1149 * hyperv_pcpu_output_arg) for making a hypervisor call.
1150 *
1151 * TODO: Take care of this when CVM support is added.
1152 */
1153 in = (void *)__get_free_page(GFP_KERNEL);
1154 out = (void *)__get_free_page(GFP_KERNEL);
1155 if (!in || !out) {
1156 ret = -ENOMEM;
1157 goto free_pages;
1158 }
1159
1160 if (copy_from_user(in, (void __user *)hvcall.input_ptr, hvcall.input_size)) {
1161 ret = -EFAULT;
1162 goto free_pages;
1163 }
1164
1165 /*
1166 * The caller supplies output_size, so clear the range copied back to
1167 * userspace in case the hypercall writes fewer bytes than requested.
1168 */
1169 memset(out, 0, hvcall.output_size);
1170
1171 hvcall.status = hv_do_hypercall(hvcall.control, in, out);
1172
1173 if (copy_to_user((void __user *)hvcall.output_ptr, out, hvcall.output_size)) {
1174 ret = -EFAULT;
1175 goto free_pages;
1176 }
1177 ret = put_user(hvcall.status, &hvcall_user->status);
1178 free_pages:
1179 free_page((unsigned long)in);
1180 free_page((unsigned long)out);
1181
1182 return ret;
1183 }
1184
mshv_vtl_hvcall_dev_ioctl(struct file * f,unsigned int cmd,unsigned long arg)1185 static long mshv_vtl_hvcall_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1186 {
1187 struct mshv_vtl_hvcall_fd *fd = f->private_data;
1188
1189 switch (cmd) {
1190 case MSHV_HVCALL_SETUP:
1191 return mshv_vtl_hvcall_do_setup(fd, (struct mshv_vtl_hvcall_setup __user *)arg);
1192 case MSHV_HVCALL:
1193 return mshv_vtl_hvcall_call(fd, (struct mshv_vtl_hvcall __user *)arg);
1194 default:
1195 break;
1196 }
1197
1198 return -ENOIOCTLCMD;
1199 }
1200
1201 static const struct file_operations mshv_vtl_hvcall_dev_file_ops = {
1202 .owner = THIS_MODULE,
1203 .open = mshv_vtl_hvcall_dev_open,
1204 .release = mshv_vtl_hvcall_dev_release,
1205 .unlocked_ioctl = mshv_vtl_hvcall_dev_ioctl,
1206 };
1207
1208 static struct miscdevice mshv_vtl_hvcall_dev = {
1209 .name = "mshv_hvcall",
1210 .nodename = "mshv_hvcall",
1211 .fops = &mshv_vtl_hvcall_dev_file_ops,
1212 .mode = 0600,
1213 .minor = MISC_DYNAMIC_MINOR,
1214 };
1215
mshv_vtl_low_open(struct inode * inodep,struct file * filp)1216 static int mshv_vtl_low_open(struct inode *inodep, struct file *filp)
1217 {
1218 pid_t pid = task_pid_vnr(current);
1219 uid_t uid = current_uid().val;
1220 int ret = 0;
1221
1222 pr_debug("%s: Opening VTL low, task group %d, uid %d\n", __func__, pid, uid);
1223
1224 if (capable(CAP_SYS_ADMIN)) {
1225 filp->private_data = inodep;
1226 } else {
1227 pr_err("%s: VTL low open failed: CAP_SYS_ADMIN required. task group %d, uid %d",
1228 __func__, pid, uid);
1229 ret = -EPERM;
1230 }
1231
1232 return ret;
1233 }
1234
can_fault(struct vm_fault * vmf,unsigned long size,unsigned long * pfn)1235 static bool can_fault(struct vm_fault *vmf, unsigned long size, unsigned long *pfn)
1236 {
1237 unsigned long mask = size - 1;
1238 unsigned long start = vmf->address & ~mask;
1239 unsigned long end = start + size;
1240 bool is_valid;
1241
1242 is_valid = (vmf->address & mask) == ((vmf->pgoff << PAGE_SHIFT) & mask) &&
1243 start >= vmf->vma->vm_start &&
1244 end <= vmf->vma->vm_end;
1245
1246 if (is_valid)
1247 *pfn = vmf->pgoff & ~(mask >> PAGE_SHIFT);
1248
1249 return is_valid;
1250 }
1251
mshv_vtl_low_huge_fault(struct vm_fault * vmf,unsigned int order)1252 static vm_fault_t mshv_vtl_low_huge_fault(struct vm_fault *vmf, unsigned int order)
1253 {
1254 unsigned long pfn = vmf->pgoff;
1255 vm_fault_t ret = VM_FAULT_FALLBACK;
1256
1257 switch (order) {
1258 case 0:
1259 return vmf_insert_mixed(vmf->vma, vmf->address, pfn);
1260
1261 case PMD_ORDER:
1262 if (can_fault(vmf, PMD_SIZE, &pfn))
1263 ret = vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE);
1264 return ret;
1265
1266 case PUD_ORDER:
1267 if (can_fault(vmf, PUD_SIZE, &pfn))
1268 ret = vmf_insert_pfn_pud(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE);
1269 return ret;
1270
1271 default:
1272 return VM_FAULT_SIGBUS;
1273 }
1274 }
1275
mshv_vtl_low_fault(struct vm_fault * vmf)1276 static vm_fault_t mshv_vtl_low_fault(struct vm_fault *vmf)
1277 {
1278 return mshv_vtl_low_huge_fault(vmf, 0);
1279 }
1280
1281 static const struct vm_operations_struct mshv_vtl_low_vm_ops = {
1282 .fault = mshv_vtl_low_fault,
1283 .huge_fault = mshv_vtl_low_huge_fault,
1284 };
1285
mshv_vtl_low_mmap(struct file * filp,struct vm_area_struct * vma)1286 static int mshv_vtl_low_mmap(struct file *filp, struct vm_area_struct *vma)
1287 {
1288 vma->vm_ops = &mshv_vtl_low_vm_ops;
1289 vm_flags_set(vma, VM_HUGEPAGE | VM_MIXEDMAP);
1290
1291 return 0;
1292 }
1293
1294 static const struct file_operations mshv_vtl_low_file_ops = {
1295 .owner = THIS_MODULE,
1296 .open = mshv_vtl_low_open,
1297 .mmap = mshv_vtl_low_mmap,
1298 };
1299
1300 static struct miscdevice mshv_vtl_low = {
1301 .name = "mshv_vtl_low",
1302 .nodename = "mshv_vtl_low",
1303 .fops = &mshv_vtl_low_file_ops,
1304 .mode = 0600,
1305 .minor = MISC_DYNAMIC_MINOR,
1306 };
1307
mshv_vtl_init(void)1308 static int __init mshv_vtl_init(void)
1309 {
1310 int ret;
1311 struct device *dev = mshv_dev.this_device;
1312
1313 /*
1314 * This creates /dev/mshv which provides functionality to create VTLs and partitions.
1315 */
1316 ret = misc_register(&mshv_dev);
1317 if (ret) {
1318 dev_err(dev, "mshv device register failed: %d\n", ret);
1319 goto free_dev;
1320 }
1321
1322 tasklet_init(&msg_dpc, mshv_vtl_sint_on_msg_dpc, 0);
1323 init_waitqueue_head(&fd_wait_queue);
1324
1325 if (mshv_vtl_get_vsm_regs()) {
1326 dev_emerg(dev, "Unable to get VSM capabilities !!\n");
1327 ret = -ENODEV;
1328 goto free_dev;
1329 }
1330 if (mshv_vtl_configure_vsm_partition(dev)) {
1331 dev_emerg(dev, "VSM configuration failed !!\n");
1332 ret = -ENODEV;
1333 goto free_dev;
1334 }
1335
1336 mshv_vtl_return_call_init(mshv_vsm_page_offsets.vtl_return_offset);
1337 ret = hv_vtl_setup_synic();
1338 if (ret)
1339 goto free_dev;
1340
1341 /*
1342 * mshv_sint device adds VMBus relay ioctl support.
1343 * This provides a channel for VTL0 to communicate with VTL2.
1344 */
1345 ret = misc_register(&mshv_vtl_sint_dev);
1346 if (ret)
1347 goto free_synic;
1348
1349 /*
1350 * mshv_hvcall device adds interface to enable userspace for direct hypercalls support.
1351 */
1352 ret = misc_register(&mshv_vtl_hvcall_dev);
1353 if (ret)
1354 goto free_sint;
1355
1356 /*
1357 * mshv_vtl_low device is used to map VTL0 address space to a user-mode process in VTL2.
1358 * It implements mmap() to allow a user-mode process in VTL2 to map to the address of VTL0.
1359 */
1360 ret = misc_register(&mshv_vtl_low);
1361 if (ret)
1362 goto free_hvcall;
1363
1364 /*
1365 * "mshv vtl mem dev" device is later used to setup VTL0 memory.
1366 */
1367 mem_dev = kzalloc_obj(*mem_dev);
1368 if (!mem_dev) {
1369 ret = -ENOMEM;
1370 goto free_low;
1371 }
1372
1373 mutex_init(&mshv_vtl_poll_file_lock);
1374
1375 device_initialize(mem_dev);
1376 dev_set_name(mem_dev, "mshv vtl mem dev");
1377 ret = device_add(mem_dev);
1378 if (ret) {
1379 dev_err(dev, "mshv vtl mem dev add: %d\n", ret);
1380 goto free_mem;
1381 }
1382
1383 return 0;
1384
1385 free_mem:
1386 kfree(mem_dev);
1387 free_low:
1388 misc_deregister(&mshv_vtl_low);
1389 free_hvcall:
1390 misc_deregister(&mshv_vtl_hvcall_dev);
1391 free_sint:
1392 misc_deregister(&mshv_vtl_sint_dev);
1393 free_synic:
1394 hv_vtl_remove_synic();
1395 free_dev:
1396 misc_deregister(&mshv_dev);
1397
1398 return ret;
1399 }
1400
mshv_vtl_exit(void)1401 static void __exit mshv_vtl_exit(void)
1402 {
1403 device_del(mem_dev);
1404 kfree(mem_dev);
1405 misc_deregister(&mshv_vtl_low);
1406 misc_deregister(&mshv_vtl_hvcall_dev);
1407 misc_deregister(&mshv_vtl_sint_dev);
1408 hv_vtl_remove_synic();
1409 misc_deregister(&mshv_dev);
1410 }
1411
1412 module_init(mshv_vtl_init);
1413 module_exit(mshv_vtl_exit);
1414