1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2023, Microsoft Corporation.
4 *
5 * Author:
6 * Roman Kisel <romank@linux.microsoft.com>
7 * Saurabh Sengar <ssengar@linux.microsoft.com>
8 * Naman Jain <namjain@linux.microsoft.com>
9 */
10
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/miscdevice.h>
14 #include <linux/anon_inodes.h>
15 #include <linux/cpuhotplug.h>
16 #include <linux/count_zeros.h>
17 #include <linux/entry-virt.h>
18 #include <linux/eventfd.h>
19 #include <linux/poll.h>
20 #include <linux/file.h>
21 #include <linux/vmalloc.h>
22 #include <asm/debugreg.h>
23 #include <asm/mshyperv.h>
24 #include <trace/events/ipi.h>
25 #include <uapi/asm/mtrr.h>
26 #include <uapi/linux/mshv.h>
27 #include <hyperv/hvhdk.h>
28
29 #include "../../kernel/fpu/legacy.h"
30 #include "mshv.h"
31 #include "mshv_vtl.h"
32 #include "hyperv_vmbus.h"
33
34 MODULE_AUTHOR("Microsoft");
35 MODULE_LICENSE("GPL");
36 MODULE_DESCRIPTION("Microsoft Hyper-V VTL Driver");
37
38 #define MSHV_ENTRY_REASON_LOWER_VTL_CALL 0x1
39 #define MSHV_ENTRY_REASON_INTERRUPT 0x2
40 #define MSHV_ENTRY_REASON_INTERCEPT 0x3
41
42 #define MSHV_REAL_OFF_SHIFT 16
43 #define MSHV_PG_OFF_CPU_MASK (BIT_ULL(MSHV_REAL_OFF_SHIFT) - 1)
44 #define MSHV_RUN_PAGE_OFFSET 0
45 #define MSHV_REG_PAGE_OFFSET 1
46 #define VTL2_VMBUS_SINT_INDEX 7
47
48 static struct device *mem_dev;
49
50 static struct tasklet_struct msg_dpc;
51 static wait_queue_head_t fd_wait_queue;
52 static bool has_message;
53 static struct eventfd_ctx *flag_eventfds[HV_EVENT_FLAGS_COUNT];
54 static DEFINE_MUTEX(flag_lock);
55 static bool __read_mostly mshv_has_reg_page;
56
57 /* hvcall code is of type u16, allocate a bitmap of size (1 << 16) to accommodate it */
58 #define MAX_BITMAP_SIZE ((U16_MAX + 1) / 8)
59
60 struct mshv_vtl_hvcall_fd {
61 u8 allow_bitmap[MAX_BITMAP_SIZE];
62 bool allow_map_initialized;
63 /*
64 * Used to protect hvcall setup in IOCTLs
65 */
66 struct mutex init_mutex;
67 struct miscdevice *dev;
68 };
69
70 struct mshv_vtl_poll_file {
71 struct file *file;
72 wait_queue_entry_t wait;
73 wait_queue_head_t *wqh;
74 poll_table pt;
75 int cpu;
76 };
77
78 struct mshv_vtl {
79 struct device *module_dev;
80 u64 id;
81 };
82
83 struct mshv_vtl_per_cpu {
84 struct mshv_vtl_run *run;
85 struct page *reg_page;
86 };
87
88 /* SYNIC_OVERLAY_PAGE_MSR - internal, identical to hv_synic_simp */
89 union hv_synic_overlay_page_msr {
90 u64 as_uint64;
91 struct {
92 u64 enabled: 1;
93 u64 reserved: 11;
94 u64 pfn: 52;
95 } __packed;
96 };
97
98 static struct mutex mshv_vtl_poll_file_lock;
99 static union hv_register_vsm_page_offsets mshv_vsm_page_offsets;
100 static union hv_register_vsm_capabilities mshv_vsm_capabilities;
101
102 static DEFINE_PER_CPU(struct mshv_vtl_poll_file, mshv_vtl_poll_file);
103 static DEFINE_PER_CPU(unsigned long long, num_vtl0_transitions);
104 static DEFINE_PER_CPU(struct mshv_vtl_per_cpu, mshv_vtl_per_cpu);
105
106 static const union hv_input_vtl input_vtl_zero;
107 static const union hv_input_vtl input_vtl_normal = {
108 .use_target_vtl = 1,
109 };
110
111 static const struct file_operations mshv_vtl_fops;
112
113 static long
mshv_ioctl_create_vtl(void __user * user_arg,struct device * module_dev)114 mshv_ioctl_create_vtl(void __user *user_arg, struct device *module_dev)
115 {
116 struct mshv_vtl *vtl;
117 struct file *file;
118 int fd;
119
120 vtl = kzalloc_obj(*vtl);
121 if (!vtl)
122 return -ENOMEM;
123
124 fd = get_unused_fd_flags(O_CLOEXEC);
125 if (fd < 0) {
126 kfree(vtl);
127 return fd;
128 }
129 file = anon_inode_getfile("mshv_vtl", &mshv_vtl_fops,
130 vtl, O_RDWR);
131 if (IS_ERR(file)) {
132 put_unused_fd(fd);
133 kfree(vtl);
134 return PTR_ERR(file);
135 }
136 vtl->module_dev = module_dev;
137 fd_install(fd, file);
138
139 return fd;
140 }
141
142 static long
mshv_ioctl_check_extension(void __user * user_arg)143 mshv_ioctl_check_extension(void __user *user_arg)
144 {
145 u32 arg;
146
147 if (copy_from_user(&arg, user_arg, sizeof(arg)))
148 return -EFAULT;
149
150 switch (arg) {
151 case MSHV_CAP_CORE_API_STABLE:
152 return 0;
153 case MSHV_CAP_REGISTER_PAGE:
154 return mshv_has_reg_page;
155 case MSHV_CAP_VTL_RETURN_ACTION:
156 return mshv_vsm_capabilities.return_action_available;
157 case MSHV_CAP_DR6_SHARED:
158 return mshv_vsm_capabilities.dr6_shared;
159 }
160
161 return -EOPNOTSUPP;
162 }
163
164 static long
mshv_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)165 mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
166 {
167 struct miscdevice *misc = filp->private_data;
168
169 switch (ioctl) {
170 case MSHV_CHECK_EXTENSION:
171 return mshv_ioctl_check_extension((void __user *)arg);
172 case MSHV_CREATE_VTL:
173 return mshv_ioctl_create_vtl((void __user *)arg, misc->this_device);
174 }
175
176 return -ENOTTY;
177 }
178
179 static const struct file_operations mshv_dev_fops = {
180 .owner = THIS_MODULE,
181 .unlocked_ioctl = mshv_dev_ioctl,
182 .llseek = noop_llseek,
183 };
184
185 static struct miscdevice mshv_dev = {
186 .minor = MISC_DYNAMIC_MINOR,
187 .name = "mshv",
188 .fops = &mshv_dev_fops,
189 .mode = 0600,
190 };
191
mshv_vtl_this_run(void)192 static struct mshv_vtl_run *mshv_vtl_this_run(void)
193 {
194 return *this_cpu_ptr(&mshv_vtl_per_cpu.run);
195 }
196
mshv_vtl_cpu_run(int cpu)197 static struct mshv_vtl_run *mshv_vtl_cpu_run(int cpu)
198 {
199 return *per_cpu_ptr(&mshv_vtl_per_cpu.run, cpu);
200 }
201
mshv_vtl_cpu_reg_page(int cpu)202 static struct page *mshv_vtl_cpu_reg_page(int cpu)
203 {
204 return *per_cpu_ptr(&mshv_vtl_per_cpu.reg_page, cpu);
205 }
206
mshv_vtl_configure_reg_page(struct mshv_vtl_per_cpu * per_cpu)207 static void mshv_vtl_configure_reg_page(struct mshv_vtl_per_cpu *per_cpu)
208 {
209 struct hv_register_assoc reg_assoc = {};
210 union hv_synic_overlay_page_msr overlay = {};
211 struct page *reg_page;
212
213 reg_page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL);
214 if (!reg_page) {
215 WARN(1, "failed to allocate register page\n");
216 return;
217 }
218
219 overlay.enabled = 1;
220 overlay.pfn = page_to_hvpfn(reg_page);
221 reg_assoc.name = HV_X64_REGISTER_REG_PAGE;
222 reg_assoc.value.reg64 = overlay.as_uint64;
223
224 if (hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
225 1, input_vtl_zero, ®_assoc)) {
226 WARN(1, "failed to setup register page\n");
227 __free_page(reg_page);
228 return;
229 }
230
231 per_cpu->reg_page = reg_page;
232 mshv_has_reg_page = true;
233 }
234
mshv_vtl_synic_enable_regs(unsigned int cpu)235 static void mshv_vtl_synic_enable_regs(unsigned int cpu)
236 {
237 union hv_synic_sint sint;
238
239 sint.as_uint64 = 0;
240 sint.vector = HYPERVISOR_CALLBACK_VECTOR;
241 sint.masked = false;
242 sint.auto_eoi = hv_recommend_using_aeoi();
243
244 /* Enable intercepts */
245 if (!mshv_vsm_capabilities.intercept_page_available)
246 hv_set_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX,
247 sint.as_uint64);
248
249 /* VTL2 Host VSP SINT is (un)masked when the user mode requests that */
250 }
251
mshv_vtl_get_vsm_regs(void)252 static int mshv_vtl_get_vsm_regs(void)
253 {
254 struct hv_register_assoc registers[2];
255 int ret, count = 2;
256
257 registers[0].name = HV_REGISTER_VSM_CODE_PAGE_OFFSETS;
258 registers[1].name = HV_REGISTER_VSM_CAPABILITIES;
259
260 ret = hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
261 count, input_vtl_zero, registers);
262 if (ret)
263 return ret;
264
265 mshv_vsm_page_offsets.as_uint64 = registers[0].value.reg64;
266 mshv_vsm_capabilities.as_uint64 = registers[1].value.reg64;
267
268 return ret;
269 }
270
mshv_vtl_configure_vsm_partition(struct device * dev)271 static int mshv_vtl_configure_vsm_partition(struct device *dev)
272 {
273 union hv_register_vsm_partition_config config;
274 struct hv_register_assoc reg_assoc;
275
276 config.as_uint64 = 0;
277 config.default_vtl_protection_mask = HV_MAP_GPA_PERMISSIONS_MASK;
278 config.enable_vtl_protection = 1;
279 config.zero_memory_on_reset = 1;
280 config.intercept_vp_startup = 1;
281 config.intercept_cpuid_unimplemented = 1;
282
283 if (mshv_vsm_capabilities.intercept_page_available) {
284 dev_dbg(dev, "using intercept page\n");
285 config.intercept_page = 1;
286 }
287
288 reg_assoc.name = HV_REGISTER_VSM_PARTITION_CONFIG;
289 reg_assoc.value.reg64 = config.as_uint64;
290
291 return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
292 1, input_vtl_zero, ®_assoc);
293 }
294
mshv_vtl_vmbus_isr(void)295 static void mshv_vtl_vmbus_isr(void)
296 {
297 struct hv_per_cpu_context *per_cpu;
298 struct hv_message *msg;
299 u32 message_type;
300 union hv_synic_event_flags *event_flags;
301 struct eventfd_ctx *eventfd;
302 u16 i;
303
304 per_cpu = this_cpu_ptr(hv_context.cpu_context);
305 if (smp_processor_id() == 0) {
306 msg = (struct hv_message *)per_cpu->hyp_synic_message_page + VTL2_VMBUS_SINT_INDEX;
307 message_type = READ_ONCE(msg->header.message_type);
308 if (message_type != HVMSG_NONE)
309 tasklet_schedule(&msg_dpc);
310 }
311
312 event_flags = (union hv_synic_event_flags *)per_cpu->hyp_synic_event_page +
313 VTL2_VMBUS_SINT_INDEX;
314 for_each_set_bit(i, event_flags->flags, HV_EVENT_FLAGS_COUNT) {
315 if (!sync_test_and_clear_bit(i, event_flags->flags))
316 continue;
317 rcu_read_lock();
318 eventfd = READ_ONCE(flag_eventfds[i]);
319 if (eventfd)
320 eventfd_signal(eventfd);
321 rcu_read_unlock();
322 }
323
324 vmbus_isr();
325 }
326
mshv_vtl_alloc_context(unsigned int cpu)327 static int mshv_vtl_alloc_context(unsigned int cpu)
328 {
329 struct mshv_vtl_per_cpu *per_cpu = this_cpu_ptr(&mshv_vtl_per_cpu);
330
331 per_cpu->run = (struct mshv_vtl_run *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
332 if (!per_cpu->run)
333 return -ENOMEM;
334
335 if (mshv_vsm_capabilities.intercept_page_available)
336 mshv_vtl_configure_reg_page(per_cpu);
337
338 mshv_vtl_synic_enable_regs(cpu);
339
340 return 0;
341 }
342
343 static int mshv_vtl_cpuhp_online;
344
hv_vtl_setup_synic(void)345 static int hv_vtl_setup_synic(void)
346 {
347 int ret;
348
349 /* Use our isr to first filter out packets destined for userspace */
350 hv_setup_vmbus_handler(mshv_vtl_vmbus_isr);
351
352 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hyperv/vtl:online",
353 mshv_vtl_alloc_context, NULL);
354 if (ret < 0) {
355 hv_setup_vmbus_handler(vmbus_isr);
356 return ret;
357 }
358
359 mshv_vtl_cpuhp_online = ret;
360
361 return 0;
362 }
363
hv_vtl_remove_synic(void)364 static void hv_vtl_remove_synic(void)
365 {
366 cpuhp_remove_state(mshv_vtl_cpuhp_online);
367 hv_setup_vmbus_handler(vmbus_isr);
368 }
369
vtl_get_vp_register(struct hv_register_assoc * reg)370 static int vtl_get_vp_register(struct hv_register_assoc *reg)
371 {
372 return hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
373 1, input_vtl_normal, reg);
374 }
375
vtl_set_vp_register(struct hv_register_assoc * reg)376 static int vtl_set_vp_register(struct hv_register_assoc *reg)
377 {
378 return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
379 1, input_vtl_normal, reg);
380 }
381
mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl * vtl,void __user * arg)382 static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg)
383 {
384 struct mshv_vtl_ram_disposition vtl0_mem;
385 struct dev_pagemap *pgmap;
386 void *addr;
387
388 if (copy_from_user(&vtl0_mem, arg, sizeof(vtl0_mem)))
389 return -EFAULT;
390 if (vtl0_mem.last_pfn <= vtl0_mem.start_pfn) {
391 dev_err(vtl->module_dev, "range start pfn (%llx) > end pfn (%llx)\n",
392 vtl0_mem.start_pfn, vtl0_mem.last_pfn);
393 return -EFAULT;
394 }
395
396 pgmap = kzalloc_obj(*pgmap);
397 if (!pgmap)
398 return -ENOMEM;
399
400 /*
401 * vtl0_mem.last_pfn is excluded in the pagemap range for VTL0 as per design.
402 * last_pfn is not reserved or wasted, and reflects 'start_pfn + size' of pagemap range.
403 */
404 pgmap->ranges[0].start = PFN_PHYS(vtl0_mem.start_pfn);
405 pgmap->ranges[0].end = PFN_PHYS(vtl0_mem.last_pfn) - 1;
406 pgmap->nr_range = 1;
407 pgmap->type = MEMORY_DEVICE_GENERIC;
408
409 /*
410 * Determine the highest page order that can be used for the given memory range.
411 * This works best when the range is aligned; i.e. both the start and the length.
412 * Clamp to MAX_FOLIO_ORDER to avoid a WARN in memremap_pages() when the range
413 * alignment exceeds the maximum supported folio order for this kernel config.
414 */
415 pgmap->vmemmap_shift = min(count_trailing_zeros(vtl0_mem.start_pfn | vtl0_mem.last_pfn),
416 MAX_FOLIO_ORDER);
417 dev_dbg(vtl->module_dev,
418 "Add VTL0 memory: start: 0x%llx, end_pfn: 0x%llx, page order: %lu\n",
419 vtl0_mem.start_pfn, vtl0_mem.last_pfn, pgmap->vmemmap_shift);
420
421 addr = devm_memremap_pages(mem_dev, pgmap);
422 if (IS_ERR(addr)) {
423 dev_err(vtl->module_dev, "devm_memremap_pages error: %ld\n", PTR_ERR(addr));
424 kfree(pgmap);
425 return PTR_ERR(addr);
426 }
427
428 /* Don't free pgmap, since it has to stick around until the memory
429 * is unmapped, which will never happen as there is no scenario
430 * where VTL0 can be released/shutdown without bringing down VTL2.
431 */
432 return 0;
433 }
434
mshv_vtl_cancel(int cpu)435 static void mshv_vtl_cancel(int cpu)
436 {
437 int here = get_cpu();
438
439 if (here != cpu) {
440 if (!xchg_relaxed(&mshv_vtl_cpu_run(cpu)->cancel, 1))
441 smp_send_reschedule(cpu);
442 } else {
443 WRITE_ONCE(mshv_vtl_this_run()->cancel, 1);
444 }
445 put_cpu();
446 }
447
mshv_vtl_poll_file_wake(wait_queue_entry_t * wait,unsigned int mode,int sync,void * key)448 static int mshv_vtl_poll_file_wake(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key)
449 {
450 struct mshv_vtl_poll_file *poll_file = container_of(wait, struct mshv_vtl_poll_file, wait);
451
452 mshv_vtl_cancel(poll_file->cpu);
453
454 return 0;
455 }
456
mshv_vtl_ptable_queue_proc(struct file * file,wait_queue_head_t * wqh,poll_table * pt)457 static void mshv_vtl_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, poll_table *pt)
458 {
459 struct mshv_vtl_poll_file *poll_file = container_of(pt, struct mshv_vtl_poll_file, pt);
460
461 WARN_ON(poll_file->wqh);
462 poll_file->wqh = wqh;
463 add_wait_queue(wqh, &poll_file->wait);
464 }
465
mshv_vtl_ioctl_set_poll_file(struct mshv_vtl_set_poll_file __user * user_input)466 static int mshv_vtl_ioctl_set_poll_file(struct mshv_vtl_set_poll_file __user *user_input)
467 {
468 struct file *file, *old_file;
469 struct mshv_vtl_poll_file *poll_file;
470 struct mshv_vtl_set_poll_file input;
471
472 if (copy_from_user(&input, user_input, sizeof(input)))
473 return -EFAULT;
474
475 if (input.cpu >= num_possible_cpus() || !cpu_online(input.cpu))
476 return -EINVAL;
477 /*
478 * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists.
479 * CPU is expected to remain online after above cpu_online() check.
480 */
481
482 file = NULL;
483 file = fget(input.fd);
484 if (!file)
485 return -EBADFD;
486
487 poll_file = per_cpu_ptr(&mshv_vtl_poll_file, READ_ONCE(input.cpu));
488 if (!poll_file)
489 return -EINVAL;
490
491 mutex_lock(&mshv_vtl_poll_file_lock);
492
493 if (poll_file->wqh)
494 remove_wait_queue(poll_file->wqh, &poll_file->wait);
495 poll_file->wqh = NULL;
496
497 old_file = poll_file->file;
498 poll_file->file = file;
499 poll_file->cpu = input.cpu;
500
501 if (file) {
502 init_waitqueue_func_entry(&poll_file->wait, mshv_vtl_poll_file_wake);
503 init_poll_funcptr(&poll_file->pt, mshv_vtl_ptable_queue_proc);
504 vfs_poll(file, &poll_file->pt);
505 }
506
507 mutex_unlock(&mshv_vtl_poll_file_lock);
508
509 if (old_file)
510 fput(old_file);
511
512 return 0;
513 }
514
515 /* Static table mapping register names to their corresponding actions */
516 static const struct {
517 enum hv_register_name reg_name;
518 int debug_reg_num; /* -1 if not a debug register */
519 u32 msr_addr; /* 0 if not an MSR */
520 } reg_table[] = {
521 /* Debug registers */
522 {HV_X64_REGISTER_DR0, 0, 0},
523 {HV_X64_REGISTER_DR1, 1, 0},
524 {HV_X64_REGISTER_DR2, 2, 0},
525 {HV_X64_REGISTER_DR3, 3, 0},
526 {HV_X64_REGISTER_DR6, 6, 0},
527 /* MTRR MSRs */
528 {HV_X64_REGISTER_MSR_MTRR_CAP, -1, MSR_MTRRcap},
529 {HV_X64_REGISTER_MSR_MTRR_DEF_TYPE, -1, MSR_MTRRdefType},
530 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0, -1, MTRRphysBase_MSR(0)},
531 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE1, -1, MTRRphysBase_MSR(1)},
532 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE2, -1, MTRRphysBase_MSR(2)},
533 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE3, -1, MTRRphysBase_MSR(3)},
534 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE4, -1, MTRRphysBase_MSR(4)},
535 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE5, -1, MTRRphysBase_MSR(5)},
536 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE6, -1, MTRRphysBase_MSR(6)},
537 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE7, -1, MTRRphysBase_MSR(7)},
538 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE8, -1, MTRRphysBase_MSR(8)},
539 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE9, -1, MTRRphysBase_MSR(9)},
540 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEA, -1, MTRRphysBase_MSR(0xa)},
541 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEB, -1, MTRRphysBase_MSR(0xb)},
542 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEC, -1, MTRRphysBase_MSR(0xc)},
543 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASED, -1, MTRRphysBase_MSR(0xd)},
544 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEE, -1, MTRRphysBase_MSR(0xe)},
545 {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEF, -1, MTRRphysBase_MSR(0xf)},
546 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0, -1, MTRRphysMask_MSR(0)},
547 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK1, -1, MTRRphysMask_MSR(1)},
548 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK2, -1, MTRRphysMask_MSR(2)},
549 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK3, -1, MTRRphysMask_MSR(3)},
550 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK4, -1, MTRRphysMask_MSR(4)},
551 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK5, -1, MTRRphysMask_MSR(5)},
552 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK6, -1, MTRRphysMask_MSR(6)},
553 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK7, -1, MTRRphysMask_MSR(7)},
554 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK8, -1, MTRRphysMask_MSR(8)},
555 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK9, -1, MTRRphysMask_MSR(9)},
556 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKA, -1, MTRRphysMask_MSR(0xa)},
557 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKB, -1, MTRRphysMask_MSR(0xb)},
558 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKC, -1, MTRRphysMask_MSR(0xc)},
559 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKD, -1, MTRRphysMask_MSR(0xd)},
560 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKE, -1, MTRRphysMask_MSR(0xe)},
561 {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKF, -1, MTRRphysMask_MSR(0xf)},
562 {HV_X64_REGISTER_MSR_MTRR_FIX64K00000, -1, MSR_MTRRfix64K_00000},
563 {HV_X64_REGISTER_MSR_MTRR_FIX16K80000, -1, MSR_MTRRfix16K_80000},
564 {HV_X64_REGISTER_MSR_MTRR_FIX16KA0000, -1, MSR_MTRRfix16K_A0000},
565 {HV_X64_REGISTER_MSR_MTRR_FIX4KC0000, -1, MSR_MTRRfix4K_C0000},
566 {HV_X64_REGISTER_MSR_MTRR_FIX4KC8000, -1, MSR_MTRRfix4K_C8000},
567 {HV_X64_REGISTER_MSR_MTRR_FIX4KD0000, -1, MSR_MTRRfix4K_D0000},
568 {HV_X64_REGISTER_MSR_MTRR_FIX4KD8000, -1, MSR_MTRRfix4K_D8000},
569 {HV_X64_REGISTER_MSR_MTRR_FIX4KE0000, -1, MSR_MTRRfix4K_E0000},
570 {HV_X64_REGISTER_MSR_MTRR_FIX4KE8000, -1, MSR_MTRRfix4K_E8000},
571 {HV_X64_REGISTER_MSR_MTRR_FIX4KF0000, -1, MSR_MTRRfix4K_F0000},
572 {HV_X64_REGISTER_MSR_MTRR_FIX4KF8000, -1, MSR_MTRRfix4K_F8000},
573 };
574
mshv_vtl_get_set_reg(struct hv_register_assoc * regs,bool set)575 static int mshv_vtl_get_set_reg(struct hv_register_assoc *regs, bool set)
576 {
577 u64 *reg64;
578 enum hv_register_name gpr_name;
579 int i;
580
581 gpr_name = regs->name;
582 reg64 = ®s->value.reg64;
583
584 /* Search for the register in the table */
585 for (i = 0; i < ARRAY_SIZE(reg_table); i++) {
586 if (reg_table[i].reg_name != gpr_name)
587 continue;
588 if (reg_table[i].debug_reg_num != -1) {
589 /* Handle debug registers */
590 if (gpr_name == HV_X64_REGISTER_DR6 &&
591 !mshv_vsm_capabilities.dr6_shared)
592 goto hypercall;
593 if (set)
594 native_set_debugreg(reg_table[i].debug_reg_num, *reg64);
595 else
596 *reg64 = native_get_debugreg(reg_table[i].debug_reg_num);
597 } else {
598 /* Handle MSRs */
599 if (set)
600 wrmsrq(reg_table[i].msr_addr, *reg64);
601 else
602 rdmsrq(reg_table[i].msr_addr, *reg64);
603 }
604 return 0;
605 }
606
607 hypercall:
608 return 1;
609 }
610
mshv_vtl_return(struct mshv_vtl_cpu_context * vtl0)611 static void mshv_vtl_return(struct mshv_vtl_cpu_context *vtl0)
612 {
613 struct hv_vp_assist_page *hvp;
614
615 hvp = hv_vp_assist_page[smp_processor_id()];
616
617 /*
618 * Process signal event direct set in the run page, if any.
619 */
620 if (mshv_vsm_capabilities.return_action_available) {
621 u32 offset = READ_ONCE(mshv_vtl_this_run()->vtl_ret_action_size);
622
623 WRITE_ONCE(mshv_vtl_this_run()->vtl_ret_action_size, 0);
624
625 /*
626 * Hypervisor will take care of clearing out the actions
627 * set in the assist page.
628 */
629 memcpy(hvp->vtl_ret_actions,
630 mshv_vtl_this_run()->vtl_ret_actions,
631 min_t(u32, offset, sizeof(hvp->vtl_ret_actions)));
632 }
633
634 mshv_vtl_return_call(vtl0);
635 }
636
mshv_vtl_process_intercept(void)637 static bool mshv_vtl_process_intercept(void)
638 {
639 struct hv_per_cpu_context *mshv_cpu;
640 void *synic_message_page;
641 struct hv_message *msg;
642 u32 message_type;
643
644 mshv_cpu = this_cpu_ptr(hv_context.cpu_context);
645 synic_message_page = mshv_cpu->hyp_synic_message_page;
646 if (unlikely(!synic_message_page))
647 return true;
648
649 msg = (struct hv_message *)synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX;
650 message_type = READ_ONCE(msg->header.message_type);
651 if (message_type == HVMSG_NONE)
652 return true;
653
654 memcpy(mshv_vtl_this_run()->exit_message, msg, sizeof(*msg));
655 vmbus_signal_eom(msg, message_type);
656
657 return false;
658 }
659
mshv_vtl_ioctl_return_to_lower_vtl(void)660 static int mshv_vtl_ioctl_return_to_lower_vtl(void)
661 {
662 preempt_disable();
663 for (;;) {
664 unsigned long irq_flags;
665 struct hv_vp_assist_page *hvp;
666 int ret;
667
668 if (__xfer_to_guest_mode_work_pending()) {
669 preempt_enable();
670 ret = xfer_to_guest_mode_handle_work();
671 if (ret)
672 return ret;
673 preempt_disable();
674 }
675
676 local_irq_save(irq_flags);
677 if (READ_ONCE(mshv_vtl_this_run()->cancel)) {
678 local_irq_restore(irq_flags);
679 preempt_enable();
680 return -EINTR;
681 }
682
683 mshv_vtl_return(&mshv_vtl_this_run()->cpu_context);
684 local_irq_restore(irq_flags);
685
686 hvp = hv_vp_assist_page[smp_processor_id()];
687 this_cpu_inc(num_vtl0_transitions);
688 switch (hvp->vtl_entry_reason) {
689 case MSHV_ENTRY_REASON_INTERRUPT:
690 if (!mshv_vsm_capabilities.intercept_page_available &&
691 likely(!mshv_vtl_process_intercept()))
692 goto done;
693 break;
694
695 case MSHV_ENTRY_REASON_INTERCEPT:
696 WARN_ON(!mshv_vsm_capabilities.intercept_page_available);
697 memcpy(mshv_vtl_this_run()->exit_message, hvp->intercept_message,
698 sizeof(hvp->intercept_message));
699 goto done;
700
701 default:
702 panic("unknown entry reason: %d", hvp->vtl_entry_reason);
703 }
704 }
705
706 done:
707 preempt_enable();
708
709 return 0;
710 }
711
712 static long
mshv_vtl_ioctl_get_regs(void __user * user_args)713 mshv_vtl_ioctl_get_regs(void __user *user_args)
714 {
715 struct mshv_vp_registers args;
716 struct hv_register_assoc reg;
717 long ret;
718
719 if (copy_from_user(&args, user_args, sizeof(args)))
720 return -EFAULT;
721
722 /* This IOCTL supports processing only one register at a time. */
723 if (args.count != 1)
724 return -EINVAL;
725
726 if (copy_from_user(®, (void __user *)args.regs_ptr,
727 sizeof(reg)))
728 return -EFAULT;
729
730 ret = mshv_vtl_get_set_reg(®, false);
731 if (!ret)
732 goto copy_args; /* No need of hypercall */
733 ret = vtl_get_vp_register(®);
734 if (ret)
735 return ret;
736
737 copy_args:
738 if (copy_to_user((void __user *)args.regs_ptr, ®, sizeof(reg)))
739 ret = -EFAULT;
740
741 return ret;
742 }
743
744 static long
mshv_vtl_ioctl_set_regs(void __user * user_args)745 mshv_vtl_ioctl_set_regs(void __user *user_args)
746 {
747 struct mshv_vp_registers args;
748 struct hv_register_assoc reg;
749 long ret;
750
751 if (copy_from_user(&args, user_args, sizeof(args)))
752 return -EFAULT;
753
754 /* This IOCTL supports processing only one register at a time. */
755 if (args.count != 1)
756 return -EINVAL;
757
758 if (copy_from_user(®, (void __user *)args.regs_ptr, sizeof(reg)))
759 return -EFAULT;
760
761 ret = mshv_vtl_get_set_reg(®, true);
762 if (!ret)
763 return ret; /* No need of hypercall */
764 ret = vtl_set_vp_register(®);
765
766 return ret;
767 }
768
769 static long
mshv_vtl_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)770 mshv_vtl_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
771 {
772 long ret;
773 struct mshv_vtl *vtl = filp->private_data;
774
775 switch (ioctl) {
776 case MSHV_SET_POLL_FILE:
777 ret = mshv_vtl_ioctl_set_poll_file((struct mshv_vtl_set_poll_file __user *)arg);
778 break;
779 case MSHV_GET_VP_REGISTERS:
780 ret = mshv_vtl_ioctl_get_regs((void __user *)arg);
781 break;
782 case MSHV_SET_VP_REGISTERS:
783 ret = mshv_vtl_ioctl_set_regs((void __user *)arg);
784 break;
785 case MSHV_RETURN_TO_LOWER_VTL:
786 ret = mshv_vtl_ioctl_return_to_lower_vtl();
787 break;
788 case MSHV_ADD_VTL0_MEMORY:
789 ret = mshv_vtl_ioctl_add_vtl0_mem(vtl, (void __user *)arg);
790 break;
791 default:
792 dev_err(vtl->module_dev, "invalid vtl ioctl: %#x\n", ioctl);
793 ret = -ENOTTY;
794 }
795
796 return ret;
797 }
798
mshv_vtl_fault(struct vm_fault * vmf)799 static vm_fault_t mshv_vtl_fault(struct vm_fault *vmf)
800 {
801 struct page *page;
802 int cpu = vmf->pgoff & MSHV_PG_OFF_CPU_MASK;
803 int real_off = vmf->pgoff >> MSHV_REAL_OFF_SHIFT;
804
805 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
806 return VM_FAULT_SIGBUS;
807 /*
808 * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists.
809 * CPU is expected to remain online after above cpu_online() check.
810 */
811
812 if (real_off == MSHV_RUN_PAGE_OFFSET) {
813 page = virt_to_page(mshv_vtl_cpu_run(cpu));
814 } else if (real_off == MSHV_REG_PAGE_OFFSET) {
815 if (!mshv_has_reg_page)
816 return VM_FAULT_SIGBUS;
817 page = mshv_vtl_cpu_reg_page(cpu);
818 } else {
819 return VM_FAULT_NOPAGE;
820 }
821
822 get_page(page);
823 vmf->page = page;
824
825 return 0;
826 }
827
828 static const struct vm_operations_struct mshv_vtl_vm_ops = {
829 .fault = mshv_vtl_fault,
830 };
831
mshv_vtl_mmap(struct file * filp,struct vm_area_struct * vma)832 static int mshv_vtl_mmap(struct file *filp, struct vm_area_struct *vma)
833 {
834 vma->vm_ops = &mshv_vtl_vm_ops;
835
836 return 0;
837 }
838
mshv_vtl_release(struct inode * inode,struct file * filp)839 static int mshv_vtl_release(struct inode *inode, struct file *filp)
840 {
841 struct mshv_vtl *vtl = filp->private_data;
842
843 kfree(vtl);
844
845 return 0;
846 }
847
848 static const struct file_operations mshv_vtl_fops = {
849 .owner = THIS_MODULE,
850 .unlocked_ioctl = mshv_vtl_ioctl,
851 .release = mshv_vtl_release,
852 .mmap = mshv_vtl_mmap,
853 };
854
mshv_vtl_synic_mask_vmbus_sint(void * info)855 static void mshv_vtl_synic_mask_vmbus_sint(void *info)
856 {
857 union hv_synic_sint sint;
858 const u8 *mask = info;
859
860 sint.as_uint64 = 0;
861 sint.vector = HYPERVISOR_CALLBACK_VECTOR;
862 sint.masked = (*mask != 0);
863 sint.auto_eoi = hv_recommend_using_aeoi();
864
865 hv_set_msr(HV_MSR_SINT0 + VTL2_VMBUS_SINT_INDEX,
866 sint.as_uint64);
867
868 if (!sint.masked)
869 pr_debug("%s: Unmasking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id());
870 else
871 pr_debug("%s: Masking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id());
872 }
873
mshv_vtl_read_remote(void * buffer)874 static void mshv_vtl_read_remote(void *buffer)
875 {
876 struct hv_per_cpu_context *mshv_cpu = this_cpu_ptr(hv_context.cpu_context);
877 struct hv_message *msg = (struct hv_message *)mshv_cpu->hyp_synic_message_page +
878 VTL2_VMBUS_SINT_INDEX;
879 u32 message_type = READ_ONCE(msg->header.message_type);
880
881 WRITE_ONCE(has_message, false);
882 if (message_type == HVMSG_NONE)
883 return;
884
885 memcpy(buffer, msg, sizeof(*msg));
886 vmbus_signal_eom(msg, message_type);
887 }
888
889 static bool vtl_synic_mask_vmbus_sint_masked = true;
890
mshv_vtl_sint_read(struct file * filp,char __user * arg,size_t size,loff_t * offset)891 static ssize_t mshv_vtl_sint_read(struct file *filp, char __user *arg, size_t size, loff_t *offset)
892 {
893 struct hv_message msg = {};
894 int ret;
895
896 if (size < sizeof(msg))
897 return -EINVAL;
898
899 for (;;) {
900 smp_call_function_single(VMBUS_CONNECT_CPU, mshv_vtl_read_remote, &msg, true);
901 if (msg.header.message_type != HVMSG_NONE)
902 break;
903
904 if (READ_ONCE(vtl_synic_mask_vmbus_sint_masked))
905 return 0; /* EOF */
906
907 if (filp->f_flags & O_NONBLOCK)
908 return -EAGAIN;
909
910 ret = wait_event_interruptible(fd_wait_queue,
911 READ_ONCE(has_message) ||
912 READ_ONCE(vtl_synic_mask_vmbus_sint_masked));
913 if (ret)
914 return ret;
915 }
916
917 if (copy_to_user(arg, &msg, sizeof(msg)))
918 return -EFAULT;
919
920 return sizeof(msg);
921 }
922
mshv_vtl_sint_poll(struct file * filp,poll_table * wait)923 static __poll_t mshv_vtl_sint_poll(struct file *filp, poll_table *wait)
924 {
925 __poll_t mask = 0;
926
927 poll_wait(filp, &fd_wait_queue, wait);
928 if (READ_ONCE(has_message) || READ_ONCE(vtl_synic_mask_vmbus_sint_masked))
929 mask |= EPOLLIN | EPOLLRDNORM;
930
931 return mask;
932 }
933
mshv_vtl_sint_on_msg_dpc(unsigned long data)934 static void mshv_vtl_sint_on_msg_dpc(unsigned long data)
935 {
936 WRITE_ONCE(has_message, true);
937 wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN);
938 }
939
mshv_vtl_sint_ioctl_post_msg(struct mshv_vtl_sint_post_msg __user * arg)940 static int mshv_vtl_sint_ioctl_post_msg(struct mshv_vtl_sint_post_msg __user *arg)
941 {
942 struct mshv_vtl_sint_post_msg message;
943 u8 payload[HV_MESSAGE_PAYLOAD_BYTE_COUNT];
944
945 if (copy_from_user(&message, arg, sizeof(message)))
946 return -EFAULT;
947 if (message.payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT)
948 return -EINVAL;
949 if (copy_from_user(payload, (void __user *)message.payload_ptr,
950 message.payload_size))
951 return -EFAULT;
952
953 return hv_post_message((union hv_connection_id)message.connection_id,
954 message.message_type, (void *)payload,
955 message.payload_size);
956 }
957
mshv_vtl_sint_ioctl_signal_event(struct mshv_vtl_signal_event __user * arg)958 static int mshv_vtl_sint_ioctl_signal_event(struct mshv_vtl_signal_event __user *arg)
959 {
960 u64 input, status;
961 struct mshv_vtl_signal_event signal_event;
962
963 if (copy_from_user(&signal_event, arg, sizeof(signal_event)))
964 return -EFAULT;
965
966 input = signal_event.connection_id | ((u64)signal_event.flag << 32);
967
968 status = hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, input);
969
970 return hv_result_to_errno(status);
971 }
972
mshv_vtl_sint_ioctl_set_eventfd(struct mshv_vtl_set_eventfd __user * arg)973 static int mshv_vtl_sint_ioctl_set_eventfd(struct mshv_vtl_set_eventfd __user *arg)
974 {
975 struct mshv_vtl_set_eventfd set_eventfd;
976 struct eventfd_ctx *eventfd, *old_eventfd;
977
978 if (copy_from_user(&set_eventfd, arg, sizeof(set_eventfd)))
979 return -EFAULT;
980 if (set_eventfd.flag >= HV_EVENT_FLAGS_COUNT)
981 return -EINVAL;
982
983 eventfd = NULL;
984 if (set_eventfd.fd >= 0) {
985 eventfd = eventfd_ctx_fdget(set_eventfd.fd);
986 if (IS_ERR(eventfd))
987 return PTR_ERR(eventfd);
988 }
989
990 guard(mutex)(&flag_lock);
991 old_eventfd = READ_ONCE(flag_eventfds[set_eventfd.flag]);
992 WRITE_ONCE(flag_eventfds[set_eventfd.flag], eventfd);
993
994 if (old_eventfd) {
995 synchronize_rcu();
996 eventfd_ctx_put(old_eventfd);
997 }
998
999 return 0;
1000 }
1001
mshv_vtl_sint_ioctl_pause_msg_stream(struct mshv_sint_mask __user * arg)1002 static int mshv_vtl_sint_ioctl_pause_msg_stream(struct mshv_sint_mask __user *arg)
1003 {
1004 static DEFINE_MUTEX(vtl2_vmbus_sint_mask_mutex);
1005 struct mshv_sint_mask mask;
1006
1007 if (copy_from_user(&mask, arg, sizeof(mask)))
1008 return -EFAULT;
1009 guard(mutex)(&vtl2_vmbus_sint_mask_mutex);
1010 on_each_cpu(mshv_vtl_synic_mask_vmbus_sint, &mask.mask, 1);
1011 WRITE_ONCE(vtl_synic_mask_vmbus_sint_masked, mask.mask != 0);
1012 if (mask.mask)
1013 wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN);
1014
1015 return 0;
1016 }
1017
mshv_vtl_sint_ioctl(struct file * f,unsigned int cmd,unsigned long arg)1018 static long mshv_vtl_sint_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1019 {
1020 switch (cmd) {
1021 case MSHV_SINT_POST_MESSAGE:
1022 return mshv_vtl_sint_ioctl_post_msg((struct mshv_vtl_sint_post_msg __user *)arg);
1023 case MSHV_SINT_SIGNAL_EVENT:
1024 return mshv_vtl_sint_ioctl_signal_event((struct mshv_vtl_signal_event __user *)arg);
1025 case MSHV_SINT_SET_EVENTFD:
1026 return mshv_vtl_sint_ioctl_set_eventfd((struct mshv_vtl_set_eventfd __user *)arg);
1027 case MSHV_SINT_PAUSE_MESSAGE_STREAM:
1028 return mshv_vtl_sint_ioctl_pause_msg_stream((struct mshv_sint_mask __user *)arg);
1029 default:
1030 return -ENOIOCTLCMD;
1031 }
1032 }
1033
1034 static const struct file_operations mshv_vtl_sint_ops = {
1035 .owner = THIS_MODULE,
1036 .read = mshv_vtl_sint_read,
1037 .poll = mshv_vtl_sint_poll,
1038 .unlocked_ioctl = mshv_vtl_sint_ioctl,
1039 };
1040
1041 static struct miscdevice mshv_vtl_sint_dev = {
1042 .name = "mshv_sint",
1043 .fops = &mshv_vtl_sint_ops,
1044 .mode = 0600,
1045 .minor = MISC_DYNAMIC_MINOR,
1046 };
1047
mshv_vtl_hvcall_dev_open(struct inode * node,struct file * f)1048 static int mshv_vtl_hvcall_dev_open(struct inode *node, struct file *f)
1049 {
1050 struct miscdevice *dev = f->private_data;
1051 struct mshv_vtl_hvcall_fd *fd;
1052
1053 if (!capable(CAP_SYS_ADMIN))
1054 return -EPERM;
1055
1056 fd = vzalloc(sizeof(*fd));
1057 if (!fd)
1058 return -ENOMEM;
1059 fd->dev = dev;
1060 f->private_data = fd;
1061 mutex_init(&fd->init_mutex);
1062
1063 return 0;
1064 }
1065
mshv_vtl_hvcall_dev_release(struct inode * node,struct file * f)1066 static int mshv_vtl_hvcall_dev_release(struct inode *node, struct file *f)
1067 {
1068 struct mshv_vtl_hvcall_fd *fd;
1069
1070 fd = f->private_data;
1071 if (fd) {
1072 vfree(fd);
1073 f->private_data = NULL;
1074 }
1075
1076 return 0;
1077 }
1078
mshv_vtl_hvcall_do_setup(struct mshv_vtl_hvcall_fd * fd,struct mshv_vtl_hvcall_setup __user * hvcall_setup_user)1079 static int mshv_vtl_hvcall_do_setup(struct mshv_vtl_hvcall_fd *fd,
1080 struct mshv_vtl_hvcall_setup __user *hvcall_setup_user)
1081 {
1082 struct mshv_vtl_hvcall_setup hvcall_setup;
1083
1084 guard(mutex)(&fd->init_mutex);
1085
1086 if (fd->allow_map_initialized) {
1087 dev_err(fd->dev->this_device,
1088 "Hypercall allow map has already been set, pid %d\n",
1089 current->pid);
1090 return -EINVAL;
1091 }
1092
1093 if (copy_from_user(&hvcall_setup, hvcall_setup_user,
1094 sizeof(struct mshv_vtl_hvcall_setup))) {
1095 return -EFAULT;
1096 }
1097 if (hvcall_setup.bitmap_array_size > ARRAY_SIZE(fd->allow_bitmap))
1098 return -EINVAL;
1099
1100 if (copy_from_user(&fd->allow_bitmap,
1101 (void __user *)hvcall_setup.allow_bitmap_ptr,
1102 hvcall_setup.bitmap_array_size)) {
1103 return -EFAULT;
1104 }
1105
1106 dev_info(fd->dev->this_device, "Hypercall allow map has been set, pid %d\n",
1107 current->pid);
1108 fd->allow_map_initialized = true;
1109 return 0;
1110 }
1111
mshv_vtl_hvcall_is_allowed(struct mshv_vtl_hvcall_fd * fd,u16 call_code)1112 static bool mshv_vtl_hvcall_is_allowed(struct mshv_vtl_hvcall_fd *fd, u16 call_code)
1113 {
1114 return test_bit(call_code, (unsigned long *)fd->allow_bitmap);
1115 }
1116
mshv_vtl_hvcall_call(struct mshv_vtl_hvcall_fd * fd,struct mshv_vtl_hvcall __user * hvcall_user)1117 static int mshv_vtl_hvcall_call(struct mshv_vtl_hvcall_fd *fd,
1118 struct mshv_vtl_hvcall __user *hvcall_user)
1119 {
1120 struct mshv_vtl_hvcall hvcall;
1121 void *in, *out;
1122 int ret;
1123
1124 if (copy_from_user(&hvcall, hvcall_user, sizeof(struct mshv_vtl_hvcall)))
1125 return -EFAULT;
1126 if (hvcall.input_size > HV_HYP_PAGE_SIZE)
1127 return -EINVAL;
1128 if (hvcall.output_size > HV_HYP_PAGE_SIZE)
1129 return -EINVAL;
1130
1131 /*
1132 * By default, all hypercalls are not allowed.
1133 * The user mode code has to set up the allow bitmap once.
1134 */
1135
1136 if (!mshv_vtl_hvcall_is_allowed(fd, hvcall.control & 0xFFFF)) {
1137 dev_err(fd->dev->this_device,
1138 "Hypercall with control data %#llx isn't allowed\n",
1139 hvcall.control);
1140 return -EPERM;
1141 }
1142
1143 /*
1144 * This may create a problem for Confidential VM (CVM) usecase where we need to use
1145 * Hyper-V driver allocated per-cpu input and output pages (hyperv_pcpu_input_arg and
1146 * hyperv_pcpu_output_arg) for making a hypervisor call.
1147 *
1148 * TODO: Take care of this when CVM support is added.
1149 */
1150 in = (void *)__get_free_page(GFP_KERNEL);
1151 out = (void *)__get_free_page(GFP_KERNEL);
1152 if (!in || !out) {
1153 ret = -ENOMEM;
1154 goto free_pages;
1155 }
1156
1157 if (copy_from_user(in, (void __user *)hvcall.input_ptr, hvcall.input_size)) {
1158 ret = -EFAULT;
1159 goto free_pages;
1160 }
1161
1162 /*
1163 * The caller supplies output_size, so clear the range copied back to
1164 * userspace in case the hypercall writes fewer bytes than requested.
1165 */
1166 memset(out, 0, hvcall.output_size);
1167
1168 hvcall.status = hv_do_hypercall(hvcall.control, in, out);
1169
1170 if (copy_to_user((void __user *)hvcall.output_ptr, out, hvcall.output_size)) {
1171 ret = -EFAULT;
1172 goto free_pages;
1173 }
1174 ret = put_user(hvcall.status, &hvcall_user->status);
1175 free_pages:
1176 free_page((unsigned long)in);
1177 free_page((unsigned long)out);
1178
1179 return ret;
1180 }
1181
mshv_vtl_hvcall_dev_ioctl(struct file * f,unsigned int cmd,unsigned long arg)1182 static long mshv_vtl_hvcall_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1183 {
1184 struct mshv_vtl_hvcall_fd *fd = f->private_data;
1185
1186 switch (cmd) {
1187 case MSHV_HVCALL_SETUP:
1188 return mshv_vtl_hvcall_do_setup(fd, (struct mshv_vtl_hvcall_setup __user *)arg);
1189 case MSHV_HVCALL:
1190 return mshv_vtl_hvcall_call(fd, (struct mshv_vtl_hvcall __user *)arg);
1191 default:
1192 break;
1193 }
1194
1195 return -ENOIOCTLCMD;
1196 }
1197
1198 static const struct file_operations mshv_vtl_hvcall_dev_file_ops = {
1199 .owner = THIS_MODULE,
1200 .open = mshv_vtl_hvcall_dev_open,
1201 .release = mshv_vtl_hvcall_dev_release,
1202 .unlocked_ioctl = mshv_vtl_hvcall_dev_ioctl,
1203 };
1204
1205 static struct miscdevice mshv_vtl_hvcall_dev = {
1206 .name = "mshv_hvcall",
1207 .nodename = "mshv_hvcall",
1208 .fops = &mshv_vtl_hvcall_dev_file_ops,
1209 .mode = 0600,
1210 .minor = MISC_DYNAMIC_MINOR,
1211 };
1212
mshv_vtl_low_open(struct inode * inodep,struct file * filp)1213 static int mshv_vtl_low_open(struct inode *inodep, struct file *filp)
1214 {
1215 pid_t pid = task_pid_vnr(current);
1216 uid_t uid = current_uid().val;
1217 int ret = 0;
1218
1219 pr_debug("%s: Opening VTL low, task group %d, uid %d\n", __func__, pid, uid);
1220
1221 if (capable(CAP_SYS_ADMIN)) {
1222 filp->private_data = inodep;
1223 } else {
1224 pr_err("%s: VTL low open failed: CAP_SYS_ADMIN required. task group %d, uid %d",
1225 __func__, pid, uid);
1226 ret = -EPERM;
1227 }
1228
1229 return ret;
1230 }
1231
can_fault(struct vm_fault * vmf,unsigned long size,unsigned long * pfn)1232 static bool can_fault(struct vm_fault *vmf, unsigned long size, unsigned long *pfn)
1233 {
1234 unsigned long mask = size - 1;
1235 unsigned long start = vmf->address & ~mask;
1236 unsigned long end = start + size;
1237 bool is_valid;
1238
1239 is_valid = (vmf->address & mask) == ((vmf->pgoff << PAGE_SHIFT) & mask) &&
1240 start >= vmf->vma->vm_start &&
1241 end <= vmf->vma->vm_end;
1242
1243 if (is_valid)
1244 *pfn = vmf->pgoff & ~(mask >> PAGE_SHIFT);
1245
1246 return is_valid;
1247 }
1248
mshv_vtl_low_huge_fault(struct vm_fault * vmf,unsigned int order)1249 static vm_fault_t mshv_vtl_low_huge_fault(struct vm_fault *vmf, unsigned int order)
1250 {
1251 unsigned long pfn = vmf->pgoff;
1252 vm_fault_t ret = VM_FAULT_FALLBACK;
1253
1254 switch (order) {
1255 case 0:
1256 return vmf_insert_mixed(vmf->vma, vmf->address, pfn);
1257
1258 case PMD_ORDER:
1259 if (can_fault(vmf, PMD_SIZE, &pfn))
1260 ret = vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE);
1261 return ret;
1262
1263 case PUD_ORDER:
1264 if (can_fault(vmf, PUD_SIZE, &pfn))
1265 ret = vmf_insert_pfn_pud(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE);
1266 return ret;
1267
1268 default:
1269 return VM_FAULT_SIGBUS;
1270 }
1271 }
1272
mshv_vtl_low_fault(struct vm_fault * vmf)1273 static vm_fault_t mshv_vtl_low_fault(struct vm_fault *vmf)
1274 {
1275 return mshv_vtl_low_huge_fault(vmf, 0);
1276 }
1277
1278 static const struct vm_operations_struct mshv_vtl_low_vm_ops = {
1279 .fault = mshv_vtl_low_fault,
1280 .huge_fault = mshv_vtl_low_huge_fault,
1281 };
1282
mshv_vtl_low_mmap(struct file * filp,struct vm_area_struct * vma)1283 static int mshv_vtl_low_mmap(struct file *filp, struct vm_area_struct *vma)
1284 {
1285 vma->vm_ops = &mshv_vtl_low_vm_ops;
1286 vm_flags_set(vma, VM_HUGEPAGE | VM_MIXEDMAP);
1287
1288 return 0;
1289 }
1290
1291 static const struct file_operations mshv_vtl_low_file_ops = {
1292 .owner = THIS_MODULE,
1293 .open = mshv_vtl_low_open,
1294 .mmap = mshv_vtl_low_mmap,
1295 };
1296
1297 static struct miscdevice mshv_vtl_low = {
1298 .name = "mshv_vtl_low",
1299 .nodename = "mshv_vtl_low",
1300 .fops = &mshv_vtl_low_file_ops,
1301 .mode = 0600,
1302 .minor = MISC_DYNAMIC_MINOR,
1303 };
1304
mshv_vtl_init(void)1305 static int __init mshv_vtl_init(void)
1306 {
1307 int ret;
1308 struct device *dev = mshv_dev.this_device;
1309
1310 /*
1311 * This creates /dev/mshv which provides functionality to create VTLs and partitions.
1312 */
1313 ret = misc_register(&mshv_dev);
1314 if (ret) {
1315 dev_err(dev, "mshv device register failed: %d\n", ret);
1316 goto free_dev;
1317 }
1318
1319 tasklet_init(&msg_dpc, mshv_vtl_sint_on_msg_dpc, 0);
1320 init_waitqueue_head(&fd_wait_queue);
1321
1322 if (mshv_vtl_get_vsm_regs()) {
1323 dev_emerg(dev, "Unable to get VSM capabilities !!\n");
1324 ret = -ENODEV;
1325 goto free_dev;
1326 }
1327 if (mshv_vtl_configure_vsm_partition(dev)) {
1328 dev_emerg(dev, "VSM configuration failed !!\n");
1329 ret = -ENODEV;
1330 goto free_dev;
1331 }
1332
1333 mshv_vtl_return_call_init(mshv_vsm_page_offsets.vtl_return_offset);
1334 ret = hv_vtl_setup_synic();
1335 if (ret)
1336 goto free_dev;
1337
1338 /*
1339 * mshv_sint device adds VMBus relay ioctl support.
1340 * This provides a channel for VTL0 to communicate with VTL2.
1341 */
1342 ret = misc_register(&mshv_vtl_sint_dev);
1343 if (ret)
1344 goto free_synic;
1345
1346 /*
1347 * mshv_hvcall device adds interface to enable userspace for direct hypercalls support.
1348 */
1349 ret = misc_register(&mshv_vtl_hvcall_dev);
1350 if (ret)
1351 goto free_sint;
1352
1353 /*
1354 * mshv_vtl_low device is used to map VTL0 address space to a user-mode process in VTL2.
1355 * It implements mmap() to allow a user-mode process in VTL2 to map to the address of VTL0.
1356 */
1357 ret = misc_register(&mshv_vtl_low);
1358 if (ret)
1359 goto free_hvcall;
1360
1361 /*
1362 * "mshv vtl mem dev" device is later used to setup VTL0 memory.
1363 */
1364 mem_dev = kzalloc_obj(*mem_dev);
1365 if (!mem_dev) {
1366 ret = -ENOMEM;
1367 goto free_low;
1368 }
1369
1370 mutex_init(&mshv_vtl_poll_file_lock);
1371
1372 device_initialize(mem_dev);
1373 dev_set_name(mem_dev, "mshv vtl mem dev");
1374 ret = device_add(mem_dev);
1375 if (ret) {
1376 dev_err(dev, "mshv vtl mem dev add: %d\n", ret);
1377 goto free_mem;
1378 }
1379
1380 return 0;
1381
1382 free_mem:
1383 kfree(mem_dev);
1384 free_low:
1385 misc_deregister(&mshv_vtl_low);
1386 free_hvcall:
1387 misc_deregister(&mshv_vtl_hvcall_dev);
1388 free_sint:
1389 misc_deregister(&mshv_vtl_sint_dev);
1390 free_synic:
1391 hv_vtl_remove_synic();
1392 free_dev:
1393 misc_deregister(&mshv_dev);
1394
1395 return ret;
1396 }
1397
mshv_vtl_exit(void)1398 static void __exit mshv_vtl_exit(void)
1399 {
1400 device_del(mem_dev);
1401 kfree(mem_dev);
1402 misc_deregister(&mshv_vtl_low);
1403 misc_deregister(&mshv_vtl_hvcall_dev);
1404 misc_deregister(&mshv_vtl_sint_dev);
1405 hv_vtl_remove_synic();
1406 misc_deregister(&mshv_dev);
1407 }
1408
1409 module_init(mshv_vtl_init);
1410 module_exit(mshv_vtl_exit);
1411