1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * Authors:
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
8 * K. Y. Srinivasan <kys@microsoft.com>
9 */
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12 #include <linux/init.h>
13 #include <linux/module.h>
14 #include <linux/device.h>
15 #include <linux/platform_device.h>
16 #include <linux/interrupt.h>
17 #include <linux/sysctl.h>
18 #include <linux/slab.h>
19 #include <linux/acpi.h>
20 #include <linux/completion.h>
21 #include <linux/hyperv.h>
22 #include <linux/kernel_stat.h>
23 #include <linux/of_address.h>
24 #include <linux/clockchips.h>
25 #include <linux/cpu.h>
26 #include <linux/sched/isolation.h>
27 #include <linux/sched/task_stack.h>
28 #include <linux/smpboot.h>
29
30 #include <linux/delay.h>
31 #include <linux/panic_notifier.h>
32 #include <linux/ptrace.h>
33 #include <linux/sysfb.h>
34 #include <linux/efi.h>
35 #include <linux/kernel.h>
36 #include <linux/syscore_ops.h>
37 #include <linux/dma-map-ops.h>
38 #include <linux/pci.h>
39 #include <linux/export.h>
40 #include <clocksource/hyperv_timer.h>
41 #include <asm/mshyperv.h>
42 #include "hyperv_vmbus.h"
43
44 struct vmbus_dynid {
45 struct list_head node;
46 struct hv_vmbus_device_id id;
47 };
48
49 /* VMBus Root Device */
50 static struct device *vmbus_root_device;
51
52 static int hyperv_cpuhp_online;
53
54 static DEFINE_PER_CPU(long, vmbus_evt);
55
56 /* Values parsed from ACPI DSDT */
57 int vmbus_irq;
58 int vmbus_interrupt;
59
60 /*
61 * If the Confidential VMBus is used, the data on the "wire" is not
62 * visible to either the host or the hypervisor.
63 */
64 static bool is_confidential;
65
vmbus_is_confidential(void)66 bool vmbus_is_confidential(void)
67 {
68 return is_confidential;
69 }
70 EXPORT_SYMBOL_GPL(vmbus_is_confidential);
71
72 static bool skip_vmbus_unload;
73
74 /*
75 * Allow a VMBus framebuffer driver to specify that in the case of a panic,
76 * it will do the VMbus unload operation once it has flushed any dirty
77 * portions of the framebuffer to the Hyper-V host.
78 */
vmbus_set_skip_unload(bool skip)79 void vmbus_set_skip_unload(bool skip)
80 {
81 skip_vmbus_unload = skip;
82 }
83 EXPORT_SYMBOL_GPL(vmbus_set_skip_unload);
84
85 /*
86 * The panic notifier below is responsible solely for unloading the
87 * vmbus connection, which is necessary in a panic event.
88 */
hv_panic_vmbus_unload(struct notifier_block * nb,unsigned long val,void * args)89 static int hv_panic_vmbus_unload(struct notifier_block *nb, unsigned long val,
90 void *args)
91 {
92 if (!skip_vmbus_unload)
93 vmbus_initiate_unload(true);
94
95 return NOTIFY_DONE;
96 }
97 static struct notifier_block hyperv_panic_vmbus_unload_block = {
98 .notifier_call = hv_panic_vmbus_unload,
99 .priority = INT_MIN + 1, /* almost the latest one to execute */
100 };
101
102 static const char *fb_mmio_name = "fb_range";
103 static struct resource *fb_mmio;
104 static struct resource *hyperv_mmio;
105 static DEFINE_MUTEX(hyperv_mmio_lock);
106
hv_get_vmbus_root_device(void)107 struct device *hv_get_vmbus_root_device(void)
108 {
109 return vmbus_root_device;
110 }
111 EXPORT_SYMBOL_GPL(hv_get_vmbus_root_device);
112
hv_vmbus_exists(void)113 bool hv_vmbus_exists(void)
114 {
115 return vmbus_root_device != NULL;
116 }
117 EXPORT_SYMBOL_GPL(hv_vmbus_exists);
118
channel_monitor_group(const struct vmbus_channel * channel)119 static u8 channel_monitor_group(const struct vmbus_channel *channel)
120 {
121 return (u8)channel->offermsg.monitorid / 32;
122 }
123
channel_monitor_offset(const struct vmbus_channel * channel)124 static u8 channel_monitor_offset(const struct vmbus_channel *channel)
125 {
126 return (u8)channel->offermsg.monitorid % 32;
127 }
128
channel_pending(const struct vmbus_channel * channel,const struct hv_monitor_page * monitor_page)129 static u32 channel_pending(const struct vmbus_channel *channel,
130 const struct hv_monitor_page *monitor_page)
131 {
132 u8 monitor_group = channel_monitor_group(channel);
133
134 return monitor_page->trigger_group[monitor_group].pending;
135 }
136
channel_latency(const struct vmbus_channel * channel,const struct hv_monitor_page * monitor_page)137 static u32 channel_latency(const struct vmbus_channel *channel,
138 const struct hv_monitor_page *monitor_page)
139 {
140 u8 monitor_group = channel_monitor_group(channel);
141 u8 monitor_offset = channel_monitor_offset(channel);
142
143 return monitor_page->latency[monitor_group][monitor_offset];
144 }
145
channel_conn_id(struct vmbus_channel * channel,struct hv_monitor_page * monitor_page)146 static u32 channel_conn_id(struct vmbus_channel *channel,
147 struct hv_monitor_page *monitor_page)
148 {
149 u8 monitor_group = channel_monitor_group(channel);
150 u8 monitor_offset = channel_monitor_offset(channel);
151
152 return monitor_page->parameter[monitor_group][monitor_offset].connectionid.u.id;
153 }
154
id_show(struct device * dev,struct device_attribute * dev_attr,char * buf)155 static ssize_t id_show(struct device *dev, struct device_attribute *dev_attr,
156 char *buf)
157 {
158 struct hv_device *hv_dev = device_to_hv_device(dev);
159
160 if (!hv_dev->channel)
161 return -ENODEV;
162 return sysfs_emit(buf, "%d\n", hv_dev->channel->offermsg.child_relid);
163 }
164 static DEVICE_ATTR_RO(id);
165
state_show(struct device * dev,struct device_attribute * dev_attr,char * buf)166 static ssize_t state_show(struct device *dev, struct device_attribute *dev_attr,
167 char *buf)
168 {
169 struct hv_device *hv_dev = device_to_hv_device(dev);
170
171 if (!hv_dev->channel)
172 return -ENODEV;
173 return sysfs_emit(buf, "%d\n", hv_dev->channel->state);
174 }
175 static DEVICE_ATTR_RO(state);
176
monitor_id_show(struct device * dev,struct device_attribute * dev_attr,char * buf)177 static ssize_t monitor_id_show(struct device *dev,
178 struct device_attribute *dev_attr, char *buf)
179 {
180 struct hv_device *hv_dev = device_to_hv_device(dev);
181
182 if (!hv_dev->channel)
183 return -ENODEV;
184 return sysfs_emit(buf, "%d\n", hv_dev->channel->offermsg.monitorid);
185 }
186 static DEVICE_ATTR_RO(monitor_id);
187
class_id_show(struct device * dev,struct device_attribute * dev_attr,char * buf)188 static ssize_t class_id_show(struct device *dev,
189 struct device_attribute *dev_attr, char *buf)
190 {
191 struct hv_device *hv_dev = device_to_hv_device(dev);
192
193 if (!hv_dev->channel)
194 return -ENODEV;
195 return sysfs_emit(buf, "{%pUl}\n",
196 &hv_dev->channel->offermsg.offer.if_type);
197 }
198 static DEVICE_ATTR_RO(class_id);
199
device_id_show(struct device * dev,struct device_attribute * dev_attr,char * buf)200 static ssize_t device_id_show(struct device *dev,
201 struct device_attribute *dev_attr, char *buf)
202 {
203 struct hv_device *hv_dev = device_to_hv_device(dev);
204
205 if (!hv_dev->channel)
206 return -ENODEV;
207 return sysfs_emit(buf, "{%pUl}\n",
208 &hv_dev->channel->offermsg.offer.if_instance);
209 }
210 static DEVICE_ATTR_RO(device_id);
211
modalias_show(struct device * dev,struct device_attribute * dev_attr,char * buf)212 static ssize_t modalias_show(struct device *dev,
213 struct device_attribute *dev_attr, char *buf)
214 {
215 struct hv_device *hv_dev = device_to_hv_device(dev);
216
217 return sysfs_emit(buf, "vmbus:%*phN\n", UUID_SIZE, &hv_dev->dev_type);
218 }
219 static DEVICE_ATTR_RO(modalias);
220
221 #ifdef CONFIG_NUMA
numa_node_show(struct device * dev,struct device_attribute * attr,char * buf)222 static ssize_t numa_node_show(struct device *dev,
223 struct device_attribute *attr, char *buf)
224 {
225 struct hv_device *hv_dev = device_to_hv_device(dev);
226
227 if (!hv_dev->channel)
228 return -ENODEV;
229
230 return sysfs_emit(buf, "%d\n", cpu_to_node(hv_dev->channel->target_cpu));
231 }
232 static DEVICE_ATTR_RO(numa_node);
233 #endif
234
server_monitor_pending_show(struct device * dev,struct device_attribute * dev_attr,char * buf)235 static ssize_t server_monitor_pending_show(struct device *dev,
236 struct device_attribute *dev_attr,
237 char *buf)
238 {
239 struct hv_device *hv_dev = device_to_hv_device(dev);
240
241 if (!hv_dev->channel)
242 return -ENODEV;
243 return sysfs_emit(buf, "%d\n", channel_pending(hv_dev->channel,
244 vmbus_connection.monitor_pages[0]));
245 }
246 static DEVICE_ATTR_RO(server_monitor_pending);
247
client_monitor_pending_show(struct device * dev,struct device_attribute * dev_attr,char * buf)248 static ssize_t client_monitor_pending_show(struct device *dev,
249 struct device_attribute *dev_attr,
250 char *buf)
251 {
252 struct hv_device *hv_dev = device_to_hv_device(dev);
253
254 if (!hv_dev->channel)
255 return -ENODEV;
256 return sysfs_emit(buf, "%d\n", channel_pending(hv_dev->channel,
257 vmbus_connection.monitor_pages[1]));
258 }
259 static DEVICE_ATTR_RO(client_monitor_pending);
260
server_monitor_latency_show(struct device * dev,struct device_attribute * dev_attr,char * buf)261 static ssize_t server_monitor_latency_show(struct device *dev,
262 struct device_attribute *dev_attr,
263 char *buf)
264 {
265 struct hv_device *hv_dev = device_to_hv_device(dev);
266
267 if (!hv_dev->channel)
268 return -ENODEV;
269 return sysfs_emit(buf, "%d\n", channel_latency(hv_dev->channel,
270 vmbus_connection.monitor_pages[0]));
271 }
272 static DEVICE_ATTR_RO(server_monitor_latency);
273
client_monitor_latency_show(struct device * dev,struct device_attribute * dev_attr,char * buf)274 static ssize_t client_monitor_latency_show(struct device *dev,
275 struct device_attribute *dev_attr,
276 char *buf)
277 {
278 struct hv_device *hv_dev = device_to_hv_device(dev);
279
280 if (!hv_dev->channel)
281 return -ENODEV;
282 return sysfs_emit(buf, "%d\n", channel_latency(hv_dev->channel,
283 vmbus_connection.monitor_pages[1]));
284 }
285 static DEVICE_ATTR_RO(client_monitor_latency);
286
server_monitor_conn_id_show(struct device * dev,struct device_attribute * dev_attr,char * buf)287 static ssize_t server_monitor_conn_id_show(struct device *dev,
288 struct device_attribute *dev_attr,
289 char *buf)
290 {
291 struct hv_device *hv_dev = device_to_hv_device(dev);
292
293 if (!hv_dev->channel)
294 return -ENODEV;
295 return sysfs_emit(buf, "%d\n", channel_conn_id(hv_dev->channel,
296 vmbus_connection.monitor_pages[0]));
297 }
298 static DEVICE_ATTR_RO(server_monitor_conn_id);
299
client_monitor_conn_id_show(struct device * dev,struct device_attribute * dev_attr,char * buf)300 static ssize_t client_monitor_conn_id_show(struct device *dev,
301 struct device_attribute *dev_attr,
302 char *buf)
303 {
304 struct hv_device *hv_dev = device_to_hv_device(dev);
305
306 if (!hv_dev->channel)
307 return -ENODEV;
308 return sysfs_emit(buf, "%d\n", channel_conn_id(hv_dev->channel,
309 vmbus_connection.monitor_pages[1]));
310 }
311 static DEVICE_ATTR_RO(client_monitor_conn_id);
312
out_intr_mask_show(struct device * dev,struct device_attribute * dev_attr,char * buf)313 static ssize_t out_intr_mask_show(struct device *dev,
314 struct device_attribute *dev_attr, char *buf)
315 {
316 struct hv_device *hv_dev = device_to_hv_device(dev);
317 struct hv_ring_buffer_debug_info outbound;
318 int ret;
319
320 if (!hv_dev->channel)
321 return -ENODEV;
322
323 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound,
324 &outbound);
325 if (ret < 0)
326 return ret;
327
328 return sysfs_emit(buf, "%d\n", outbound.current_interrupt_mask);
329 }
330 static DEVICE_ATTR_RO(out_intr_mask);
331
out_read_index_show(struct device * dev,struct device_attribute * dev_attr,char * buf)332 static ssize_t out_read_index_show(struct device *dev,
333 struct device_attribute *dev_attr, char *buf)
334 {
335 struct hv_device *hv_dev = device_to_hv_device(dev);
336 struct hv_ring_buffer_debug_info outbound;
337 int ret;
338
339 if (!hv_dev->channel)
340 return -ENODEV;
341
342 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound,
343 &outbound);
344 if (ret < 0)
345 return ret;
346 return sysfs_emit(buf, "%u\n", outbound.current_read_index);
347 }
348 static DEVICE_ATTR_RO(out_read_index);
349
out_write_index_show(struct device * dev,struct device_attribute * dev_attr,char * buf)350 static ssize_t out_write_index_show(struct device *dev,
351 struct device_attribute *dev_attr,
352 char *buf)
353 {
354 struct hv_device *hv_dev = device_to_hv_device(dev);
355 struct hv_ring_buffer_debug_info outbound;
356 int ret;
357
358 if (!hv_dev->channel)
359 return -ENODEV;
360
361 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound,
362 &outbound);
363 if (ret < 0)
364 return ret;
365 return sysfs_emit(buf, "%u\n", outbound.current_write_index);
366 }
367 static DEVICE_ATTR_RO(out_write_index);
368
out_read_bytes_avail_show(struct device * dev,struct device_attribute * dev_attr,char * buf)369 static ssize_t out_read_bytes_avail_show(struct device *dev,
370 struct device_attribute *dev_attr,
371 char *buf)
372 {
373 struct hv_device *hv_dev = device_to_hv_device(dev);
374 struct hv_ring_buffer_debug_info outbound;
375 int ret;
376
377 if (!hv_dev->channel)
378 return -ENODEV;
379
380 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound,
381 &outbound);
382 if (ret < 0)
383 return ret;
384 return sysfs_emit(buf, "%d\n", outbound.bytes_avail_toread);
385 }
386 static DEVICE_ATTR_RO(out_read_bytes_avail);
387
out_write_bytes_avail_show(struct device * dev,struct device_attribute * dev_attr,char * buf)388 static ssize_t out_write_bytes_avail_show(struct device *dev,
389 struct device_attribute *dev_attr,
390 char *buf)
391 {
392 struct hv_device *hv_dev = device_to_hv_device(dev);
393 struct hv_ring_buffer_debug_info outbound;
394 int ret;
395
396 if (!hv_dev->channel)
397 return -ENODEV;
398
399 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound,
400 &outbound);
401 if (ret < 0)
402 return ret;
403 return sysfs_emit(buf, "%d\n", outbound.bytes_avail_towrite);
404 }
405 static DEVICE_ATTR_RO(out_write_bytes_avail);
406
in_intr_mask_show(struct device * dev,struct device_attribute * dev_attr,char * buf)407 static ssize_t in_intr_mask_show(struct device *dev,
408 struct device_attribute *dev_attr, char *buf)
409 {
410 struct hv_device *hv_dev = device_to_hv_device(dev);
411 struct hv_ring_buffer_debug_info inbound;
412 int ret;
413
414 if (!hv_dev->channel)
415 return -ENODEV;
416
417 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound);
418 if (ret < 0)
419 return ret;
420
421 return sysfs_emit(buf, "%d\n", inbound.current_interrupt_mask);
422 }
423 static DEVICE_ATTR_RO(in_intr_mask);
424
in_read_index_show(struct device * dev,struct device_attribute * dev_attr,char * buf)425 static ssize_t in_read_index_show(struct device *dev,
426 struct device_attribute *dev_attr, char *buf)
427 {
428 struct hv_device *hv_dev = device_to_hv_device(dev);
429 struct hv_ring_buffer_debug_info inbound;
430 int ret;
431
432 if (!hv_dev->channel)
433 return -ENODEV;
434
435 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound);
436 if (ret < 0)
437 return ret;
438
439 return sysfs_emit(buf, "%d\n", inbound.current_read_index);
440 }
441 static DEVICE_ATTR_RO(in_read_index);
442
in_write_index_show(struct device * dev,struct device_attribute * dev_attr,char * buf)443 static ssize_t in_write_index_show(struct device *dev,
444 struct device_attribute *dev_attr, char *buf)
445 {
446 struct hv_device *hv_dev = device_to_hv_device(dev);
447 struct hv_ring_buffer_debug_info inbound;
448 int ret;
449
450 if (!hv_dev->channel)
451 return -ENODEV;
452
453 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound);
454 if (ret < 0)
455 return ret;
456
457 return sysfs_emit(buf, "%d\n", inbound.current_write_index);
458 }
459 static DEVICE_ATTR_RO(in_write_index);
460
in_read_bytes_avail_show(struct device * dev,struct device_attribute * dev_attr,char * buf)461 static ssize_t in_read_bytes_avail_show(struct device *dev,
462 struct device_attribute *dev_attr,
463 char *buf)
464 {
465 struct hv_device *hv_dev = device_to_hv_device(dev);
466 struct hv_ring_buffer_debug_info inbound;
467 int ret;
468
469 if (!hv_dev->channel)
470 return -ENODEV;
471
472 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound);
473 if (ret < 0)
474 return ret;
475
476 return sysfs_emit(buf, "%d\n", inbound.bytes_avail_toread);
477 }
478 static DEVICE_ATTR_RO(in_read_bytes_avail);
479
in_write_bytes_avail_show(struct device * dev,struct device_attribute * dev_attr,char * buf)480 static ssize_t in_write_bytes_avail_show(struct device *dev,
481 struct device_attribute *dev_attr,
482 char *buf)
483 {
484 struct hv_device *hv_dev = device_to_hv_device(dev);
485 struct hv_ring_buffer_debug_info inbound;
486 int ret;
487
488 if (!hv_dev->channel)
489 return -ENODEV;
490
491 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound);
492 if (ret < 0)
493 return ret;
494
495 return sysfs_emit(buf, "%d\n", inbound.bytes_avail_towrite);
496 }
497 static DEVICE_ATTR_RO(in_write_bytes_avail);
498
channel_vp_mapping_show(struct device * dev,struct device_attribute * dev_attr,char * buf)499 static ssize_t channel_vp_mapping_show(struct device *dev,
500 struct device_attribute *dev_attr,
501 char *buf)
502 {
503 struct hv_device *hv_dev = device_to_hv_device(dev);
504 struct vmbus_channel *channel = hv_dev->channel, *cur_sc;
505 int n_written;
506 struct list_head *cur;
507
508 if (!channel)
509 return -ENODEV;
510
511 mutex_lock(&vmbus_connection.channel_mutex);
512
513 n_written = sysfs_emit(buf, "%u:%u\n",
514 channel->offermsg.child_relid,
515 channel->target_cpu);
516
517 list_for_each(cur, &channel->sc_list) {
518
519 cur_sc = list_entry(cur, struct vmbus_channel, sc_list);
520 n_written += sysfs_emit_at(buf, n_written, "%u:%u\n",
521 cur_sc->offermsg.child_relid,
522 cur_sc->target_cpu);
523 }
524
525 mutex_unlock(&vmbus_connection.channel_mutex);
526
527 return n_written;
528 }
529 static DEVICE_ATTR_RO(channel_vp_mapping);
530
vendor_show(struct device * dev,struct device_attribute * dev_attr,char * buf)531 static ssize_t vendor_show(struct device *dev,
532 struct device_attribute *dev_attr,
533 char *buf)
534 {
535 struct hv_device *hv_dev = device_to_hv_device(dev);
536
537 return sysfs_emit(buf, "0x%x\n", hv_dev->vendor_id);
538 }
539 static DEVICE_ATTR_RO(vendor);
540
device_show(struct device * dev,struct device_attribute * dev_attr,char * buf)541 static ssize_t device_show(struct device *dev,
542 struct device_attribute *dev_attr,
543 char *buf)
544 {
545 struct hv_device *hv_dev = device_to_hv_device(dev);
546
547 return sysfs_emit(buf, "0x%x\n", hv_dev->device_id);
548 }
549 static DEVICE_ATTR_RO(device);
550
551 /* Set up per device attributes in /sys/bus/vmbus/devices/<bus device> */
552 static struct attribute *vmbus_dev_attrs[] = {
553 &dev_attr_id.attr,
554 &dev_attr_state.attr,
555 &dev_attr_monitor_id.attr,
556 &dev_attr_class_id.attr,
557 &dev_attr_device_id.attr,
558 &dev_attr_modalias.attr,
559 #ifdef CONFIG_NUMA
560 &dev_attr_numa_node.attr,
561 #endif
562 &dev_attr_server_monitor_pending.attr,
563 &dev_attr_client_monitor_pending.attr,
564 &dev_attr_server_monitor_latency.attr,
565 &dev_attr_client_monitor_latency.attr,
566 &dev_attr_server_monitor_conn_id.attr,
567 &dev_attr_client_monitor_conn_id.attr,
568 &dev_attr_out_intr_mask.attr,
569 &dev_attr_out_read_index.attr,
570 &dev_attr_out_write_index.attr,
571 &dev_attr_out_read_bytes_avail.attr,
572 &dev_attr_out_write_bytes_avail.attr,
573 &dev_attr_in_intr_mask.attr,
574 &dev_attr_in_read_index.attr,
575 &dev_attr_in_write_index.attr,
576 &dev_attr_in_read_bytes_avail.attr,
577 &dev_attr_in_write_bytes_avail.attr,
578 &dev_attr_channel_vp_mapping.attr,
579 &dev_attr_vendor.attr,
580 &dev_attr_device.attr,
581 NULL,
582 };
583
584 /*
585 * Device-level attribute_group callback function. Returns the permission for
586 * each attribute, and returns 0 if an attribute is not visible.
587 */
vmbus_dev_attr_is_visible(struct kobject * kobj,struct attribute * attr,int idx)588 static umode_t vmbus_dev_attr_is_visible(struct kobject *kobj,
589 struct attribute *attr, int idx)
590 {
591 struct device *dev = kobj_to_dev(kobj);
592 const struct hv_device *hv_dev = device_to_hv_device(dev);
593
594 /* Hide the monitor attributes if the monitor mechanism is not used. */
595 if (!hv_dev->channel->offermsg.monitor_allocated &&
596 (attr == &dev_attr_monitor_id.attr ||
597 attr == &dev_attr_server_monitor_pending.attr ||
598 attr == &dev_attr_client_monitor_pending.attr ||
599 attr == &dev_attr_server_monitor_latency.attr ||
600 attr == &dev_attr_client_monitor_latency.attr ||
601 attr == &dev_attr_server_monitor_conn_id.attr ||
602 attr == &dev_attr_client_monitor_conn_id.attr))
603 return 0;
604
605 return attr->mode;
606 }
607
608 static const struct attribute_group vmbus_dev_group = {
609 .attrs = vmbus_dev_attrs,
610 .is_visible = vmbus_dev_attr_is_visible
611 };
612 __ATTRIBUTE_GROUPS(vmbus_dev);
613
614 /* Set up the attribute for /sys/bus/vmbus/hibernation */
hibernation_show(const struct bus_type * bus,char * buf)615 static ssize_t hibernation_show(const struct bus_type *bus, char *buf)
616 {
617 return sprintf(buf, "%d\n", !!hv_is_hibernation_supported());
618 }
619
620 static BUS_ATTR_RO(hibernation);
621
622 static struct attribute *vmbus_bus_attrs[] = {
623 &bus_attr_hibernation.attr,
624 NULL,
625 };
626 static const struct attribute_group vmbus_bus_group = {
627 .attrs = vmbus_bus_attrs,
628 };
629 __ATTRIBUTE_GROUPS(vmbus_bus);
630
631 /*
632 * vmbus_uevent - add uevent for our device
633 *
634 * This routine is invoked when a device is added or removed on the vmbus to
635 * generate a uevent to udev in the userspace. The udev will then look at its
636 * rule and the uevent generated here to load the appropriate driver
637 *
638 * The alias string will be of the form vmbus:guid where guid is the string
639 * representation of the device guid (each byte of the guid will be
640 * represented with two hex characters.
641 */
vmbus_uevent(const struct device * device,struct kobj_uevent_env * env)642 static int vmbus_uevent(const struct device *device, struct kobj_uevent_env *env)
643 {
644 const struct hv_device *dev = device_to_hv_device(device);
645 const char *format = "MODALIAS=vmbus:%*phN";
646
647 return add_uevent_var(env, format, UUID_SIZE, &dev->dev_type);
648 }
649
650 static const struct hv_vmbus_device_id *
hv_vmbus_dev_match(const struct hv_vmbus_device_id * id,const guid_t * guid)651 hv_vmbus_dev_match(const struct hv_vmbus_device_id *id, const guid_t *guid)
652 {
653 if (id == NULL)
654 return NULL; /* empty device table */
655
656 for (; !guid_is_null(&id->guid); id++)
657 if (guid_equal(&id->guid, guid))
658 return id;
659
660 return NULL;
661 }
662
663 static const struct hv_vmbus_device_id *
hv_vmbus_dynid_match(struct hv_driver * drv,const guid_t * guid)664 hv_vmbus_dynid_match(struct hv_driver *drv, const guid_t *guid)
665 {
666 const struct hv_vmbus_device_id *id = NULL;
667 struct vmbus_dynid *dynid;
668
669 spin_lock(&drv->dynids.lock);
670 list_for_each_entry(dynid, &drv->dynids.list, node) {
671 if (guid_equal(&dynid->id.guid, guid)) {
672 id = &dynid->id;
673 break;
674 }
675 }
676 spin_unlock(&drv->dynids.lock);
677
678 return id;
679 }
680
681 static const struct hv_vmbus_device_id vmbus_device_null;
682
683 /*
684 * Return a matching hv_vmbus_device_id pointer.
685 * If there is no match, return NULL.
686 */
hv_vmbus_get_id(const struct hv_driver * drv,struct hv_device * dev)687 static const struct hv_vmbus_device_id *hv_vmbus_get_id(const struct hv_driver *drv,
688 struct hv_device *dev)
689 {
690 const guid_t *guid = &dev->dev_type;
691 const struct hv_vmbus_device_id *id;
692 int ret;
693
694 /* If a driver override is set, only bind to the matching driver */
695 ret = device_match_driver_override(&dev->device, &drv->driver);
696 if (ret == 0)
697 return NULL;
698
699 /* Look at the dynamic ids first, before the static ones */
700 id = hv_vmbus_dynid_match((struct hv_driver *)drv, guid);
701 if (!id)
702 id = hv_vmbus_dev_match(drv->id_table, guid);
703
704 /*
705 * If there's a matching driver override, this function should succeed,
706 * thus return a dummy device ID if no matching ID is found.
707 */
708 if (!id && ret > 0)
709 id = &vmbus_device_null;
710
711 return id;
712 }
713
714 /* vmbus_add_dynid - add a new device ID to this driver and re-probe devices
715 *
716 * This function can race with vmbus_device_register(). This function is
717 * typically running on a user thread in response to writing to the "new_id"
718 * sysfs entry for a driver. vmbus_device_register() is running on a
719 * workqueue thread in response to the Hyper-V host offering a device to the
720 * guest. This function calls driver_attach(), which looks for an existing
721 * device matching the new id, and attaches the driver to which the new id
722 * has been assigned. vmbus_device_register() calls device_register(), which
723 * looks for a driver that matches the device being registered. If both
724 * operations are running simultaneously, the device driver probe function runs
725 * on whichever thread establishes the linkage between the driver and device.
726 *
727 * In most cases, it doesn't matter which thread runs the driver probe
728 * function. But if vmbus_device_register() does not find a matching driver,
729 * it proceeds to create the "channels" subdirectory and numbered per-channel
730 * subdirectory in sysfs. While that multi-step creation is in progress, this
731 * function could run the driver probe function. If the probe function checks
732 * for, or operates on, entries in the "channels" subdirectory, including by
733 * calling hv_create_ring_sysfs(), the operation may or may not succeed
734 * depending on the race. The race can't create a kernel failure in VMBus
735 * or device subsystem code, but probe functions in VMBus drivers doing such
736 * operations must be prepared for the failure case.
737 */
vmbus_add_dynid(struct hv_driver * drv,guid_t * guid)738 static int vmbus_add_dynid(struct hv_driver *drv, guid_t *guid)
739 {
740 struct vmbus_dynid *dynid;
741
742 dynid = kzalloc_obj(*dynid);
743 if (!dynid)
744 return -ENOMEM;
745
746 dynid->id.guid = *guid;
747
748 spin_lock(&drv->dynids.lock);
749 list_add_tail(&dynid->node, &drv->dynids.list);
750 spin_unlock(&drv->dynids.lock);
751
752 return driver_attach(&drv->driver);
753 }
754
vmbus_free_dynids(struct hv_driver * drv)755 static void vmbus_free_dynids(struct hv_driver *drv)
756 {
757 struct vmbus_dynid *dynid, *n;
758
759 spin_lock(&drv->dynids.lock);
760 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
761 list_del(&dynid->node);
762 kfree(dynid);
763 }
764 spin_unlock(&drv->dynids.lock);
765 }
766
767 /*
768 * store_new_id - sysfs frontend to vmbus_add_dynid()
769 *
770 * Allow GUIDs to be added to an existing driver via sysfs.
771 */
new_id_store(struct device_driver * driver,const char * buf,size_t count)772 static ssize_t new_id_store(struct device_driver *driver, const char *buf,
773 size_t count)
774 {
775 struct hv_driver *drv = drv_to_hv_drv(driver);
776 guid_t guid;
777 ssize_t retval;
778
779 retval = guid_parse(buf, &guid);
780 if (retval)
781 return retval;
782
783 if (hv_vmbus_dynid_match(drv, &guid))
784 return -EEXIST;
785
786 retval = vmbus_add_dynid(drv, &guid);
787 if (retval)
788 return retval;
789 return count;
790 }
791 static DRIVER_ATTR_WO(new_id);
792
793 /*
794 * store_remove_id - remove a PCI device ID from this driver
795 *
796 * Removes a dynamic pci device ID to this driver.
797 */
remove_id_store(struct device_driver * driver,const char * buf,size_t count)798 static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
799 size_t count)
800 {
801 struct hv_driver *drv = drv_to_hv_drv(driver);
802 struct vmbus_dynid *dynid, *n;
803 guid_t guid;
804 ssize_t retval;
805
806 retval = guid_parse(buf, &guid);
807 if (retval)
808 return retval;
809
810 retval = -ENODEV;
811 spin_lock(&drv->dynids.lock);
812 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
813 struct hv_vmbus_device_id *id = &dynid->id;
814
815 if (guid_equal(&id->guid, &guid)) {
816 list_del(&dynid->node);
817 kfree(dynid);
818 retval = count;
819 break;
820 }
821 }
822 spin_unlock(&drv->dynids.lock);
823
824 return retval;
825 }
826 static DRIVER_ATTR_WO(remove_id);
827
828 static struct attribute *vmbus_drv_attrs[] = {
829 &driver_attr_new_id.attr,
830 &driver_attr_remove_id.attr,
831 NULL,
832 };
833 ATTRIBUTE_GROUPS(vmbus_drv);
834
835
836 /*
837 * vmbus_match - Attempt to match the specified device to the specified driver
838 */
vmbus_match(struct device * device,const struct device_driver * driver)839 static int vmbus_match(struct device *device, const struct device_driver *driver)
840 {
841 const struct hv_driver *drv = drv_to_hv_drv(driver);
842 struct hv_device *hv_dev = device_to_hv_device(device);
843
844 /* The hv_sock driver handles all hv_sock offers. */
845 if (is_hvsock_channel(hv_dev->channel))
846 return drv->hvsock;
847
848 if (hv_vmbus_get_id(drv, hv_dev))
849 return 1;
850
851 return 0;
852 }
853
854 /*
855 * vmbus_probe - Add the new vmbus's child device
856 */
vmbus_probe(struct device * child_device)857 static int vmbus_probe(struct device *child_device)
858 {
859 int ret = 0;
860 struct hv_driver *drv =
861 drv_to_hv_drv(child_device->driver);
862 struct hv_device *dev = device_to_hv_device(child_device);
863 const struct hv_vmbus_device_id *dev_id;
864
865 dev_id = hv_vmbus_get_id(drv, dev);
866 if (drv->probe) {
867 ret = drv->probe(dev, dev_id);
868 if (ret != 0)
869 pr_err("probe failed for device %s (%d)\n",
870 dev_name(child_device), ret);
871
872 } else {
873 pr_err("probe not set for driver %s\n",
874 dev_name(child_device));
875 ret = -ENODEV;
876 }
877 return ret;
878 }
879
880 /*
881 * vmbus_dma_configure -- Configure DMA coherence for VMbus device
882 */
vmbus_dma_configure(struct device * child_device)883 static int vmbus_dma_configure(struct device *child_device)
884 {
885 /*
886 * On ARM64, propagate the DMA coherence setting from the top level
887 * VMbus ACPI device to the child VMbus device being added here.
888 * On x86/x64 coherence is assumed and these calls have no effect.
889 */
890 hv_setup_dma_ops(child_device,
891 device_get_dma_attr(vmbus_root_device) == DEV_DMA_COHERENT);
892 return 0;
893 }
894
895 /*
896 * vmbus_remove - Remove a vmbus device
897 */
vmbus_remove(struct device * child_device)898 static void vmbus_remove(struct device *child_device)
899 {
900 struct hv_driver *drv;
901 struct hv_device *dev = device_to_hv_device(child_device);
902
903 if (child_device->driver) {
904 drv = drv_to_hv_drv(child_device->driver);
905 if (drv->remove)
906 drv->remove(dev);
907 }
908 }
909
910 /*
911 * vmbus_shutdown - Shutdown a vmbus device
912 */
vmbus_shutdown(struct device * child_device)913 static void vmbus_shutdown(struct device *child_device)
914 {
915 struct hv_driver *drv;
916 struct hv_device *dev = device_to_hv_device(child_device);
917
918
919 /* The device may not be attached yet */
920 if (!child_device->driver)
921 return;
922
923 drv = drv_to_hv_drv(child_device->driver);
924
925 if (drv->shutdown)
926 drv->shutdown(dev);
927 }
928
929 #ifdef CONFIG_PM_SLEEP
930 /*
931 * vmbus_suspend - Suspend a vmbus device
932 */
vmbus_suspend(struct device * child_device)933 static int vmbus_suspend(struct device *child_device)
934 {
935 struct hv_driver *drv;
936 struct hv_device *dev = device_to_hv_device(child_device);
937
938 /* The device may not be attached yet */
939 if (!child_device->driver)
940 return 0;
941
942 drv = drv_to_hv_drv(child_device->driver);
943 if (!drv->suspend)
944 return -EOPNOTSUPP;
945
946 return drv->suspend(dev);
947 }
948
949 /*
950 * vmbus_resume - Resume a vmbus device
951 */
vmbus_resume(struct device * child_device)952 static int vmbus_resume(struct device *child_device)
953 {
954 struct hv_driver *drv;
955 struct hv_device *dev = device_to_hv_device(child_device);
956
957 /* The device may not be attached yet */
958 if (!child_device->driver)
959 return 0;
960
961 drv = drv_to_hv_drv(child_device->driver);
962 if (!drv->resume)
963 return -EOPNOTSUPP;
964
965 return drv->resume(dev);
966 }
967 #else
968 #define vmbus_suspend NULL
969 #define vmbus_resume NULL
970 #endif /* CONFIG_PM_SLEEP */
971
972 /*
973 * vmbus_device_release - Final callback release of the vmbus child device
974 */
vmbus_device_release(struct device * device)975 static void vmbus_device_release(struct device *device)
976 {
977 struct hv_device *hv_dev = device_to_hv_device(device);
978 struct vmbus_channel *channel = hv_dev->channel;
979
980 hv_debug_rm_dev_dir(hv_dev);
981
982 mutex_lock(&vmbus_connection.channel_mutex);
983 hv_process_channel_removal(channel);
984 mutex_unlock(&vmbus_connection.channel_mutex);
985 kfree(hv_dev);
986 }
987
988 /*
989 * Note: we must use the "noirq" ops: see the comment before vmbus_bus_pm.
990 *
991 * suspend_noirq/resume_noirq are set to NULL to support Suspend-to-Idle: we
992 * shouldn't suspend the vmbus devices upon Suspend-to-Idle, otherwise there
993 * is no way to wake up a Generation-2 VM.
994 *
995 * The other 4 ops are for hibernation.
996 */
997
998 static const struct dev_pm_ops vmbus_pm = {
999 .suspend_noirq = NULL,
1000 .resume_noirq = NULL,
1001 .freeze_noirq = vmbus_suspend,
1002 .thaw_noirq = vmbus_resume,
1003 .poweroff_noirq = vmbus_suspend,
1004 .restore_noirq = vmbus_resume,
1005 };
1006
1007 /* The one and only one */
1008 static const struct bus_type hv_bus = {
1009 .name = "vmbus",
1010 .driver_override = true,
1011 .match = vmbus_match,
1012 .shutdown = vmbus_shutdown,
1013 .remove = vmbus_remove,
1014 .probe = vmbus_probe,
1015 .uevent = vmbus_uevent,
1016 .dma_configure = vmbus_dma_configure,
1017 .dev_groups = vmbus_dev_groups,
1018 .drv_groups = vmbus_drv_groups,
1019 .bus_groups = vmbus_bus_groups,
1020 .pm = &vmbus_pm,
1021 };
1022
1023 struct onmessage_work_context {
1024 struct work_struct work;
1025 struct {
1026 struct hv_message_header header;
1027 u8 payload[];
1028 } msg;
1029 };
1030
vmbus_onmessage_work(struct work_struct * work)1031 static void vmbus_onmessage_work(struct work_struct *work)
1032 {
1033 struct onmessage_work_context *ctx;
1034
1035 /* Do not process messages if we're in DISCONNECTED state */
1036 if (vmbus_connection.conn_state == DISCONNECTED)
1037 return;
1038
1039 ctx = container_of(work, struct onmessage_work_context,
1040 work);
1041 vmbus_onmessage((struct vmbus_channel_message_header *)
1042 &ctx->msg.payload);
1043 kfree(ctx);
1044 }
1045
__vmbus_on_msg_dpc(void * message_page_addr)1046 static void __vmbus_on_msg_dpc(void *message_page_addr)
1047 {
1048 struct hv_message msg_copy, *msg;
1049 struct vmbus_channel_message_header *hdr;
1050 enum vmbus_channel_message_type msgtype;
1051 const struct vmbus_channel_message_table_entry *entry;
1052 struct onmessage_work_context *ctx;
1053 __u8 payload_size;
1054 u32 message_type;
1055
1056 if (!message_page_addr)
1057 return;
1058 msg = (struct hv_message *)message_page_addr + VMBUS_MESSAGE_SINT;
1059
1060 /*
1061 * 'enum vmbus_channel_message_type' is supposed to always be 'u32' as
1062 * it is being used in 'struct vmbus_channel_message_header' definition
1063 * which is supposed to match hypervisor ABI.
1064 */
1065 BUILD_BUG_ON(sizeof(enum vmbus_channel_message_type) != sizeof(u32));
1066
1067 /*
1068 * Since the message is in memory shared with the host, an erroneous or
1069 * malicious Hyper-V could modify the message while vmbus_on_msg_dpc()
1070 * or individual message handlers are executing; to prevent this, copy
1071 * the message into private memory.
1072 */
1073 memcpy(&msg_copy, msg, sizeof(struct hv_message));
1074
1075 message_type = msg_copy.header.message_type;
1076 if (message_type == HVMSG_NONE)
1077 /* no msg */
1078 return;
1079
1080 hdr = (struct vmbus_channel_message_header *)msg_copy.u.payload;
1081 msgtype = hdr->msgtype;
1082
1083 trace_vmbus_on_msg_dpc(hdr);
1084
1085 if (msgtype >= CHANNELMSG_COUNT) {
1086 WARN_ONCE(1, "unknown msgtype=%d\n", msgtype);
1087 goto msg_handled;
1088 }
1089
1090 payload_size = msg_copy.header.payload_size;
1091 if (payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) {
1092 WARN_ONCE(1, "payload size is too large (%d)\n", payload_size);
1093 goto msg_handled;
1094 }
1095
1096 entry = &channel_message_table[msgtype];
1097
1098 if (!entry->message_handler)
1099 goto msg_handled;
1100
1101 if (payload_size < entry->min_payload_len) {
1102 WARN_ONCE(1, "message too short: msgtype=%d len=%d\n", msgtype, payload_size);
1103 goto msg_handled;
1104 }
1105
1106 if (entry->handler_type == VMHT_BLOCKING) {
1107 ctx = kmalloc_flex(*ctx, msg.payload, payload_size, GFP_ATOMIC);
1108 if (ctx == NULL)
1109 return;
1110
1111 INIT_WORK(&ctx->work, vmbus_onmessage_work);
1112 ctx->msg.header = msg_copy.header;
1113 memcpy(&ctx->msg.payload, msg_copy.u.payload, payload_size);
1114
1115 /*
1116 * The host can generate a rescind message while we
1117 * may still be handling the original offer. We deal with
1118 * this condition by relying on the synchronization provided
1119 * by offer_in_progress and by channel_mutex. See also the
1120 * inline comments in vmbus_onoffer_rescind().
1121 */
1122 switch (msgtype) {
1123 case CHANNELMSG_RESCIND_CHANNELOFFER:
1124 /*
1125 * If we are handling the rescind message;
1126 * schedule the work on the global work queue.
1127 *
1128 * The OFFER message and the RESCIND message should
1129 * not be handled by the same serialized work queue,
1130 * because the OFFER handler may call vmbus_open(),
1131 * which tries to open the channel by sending an
1132 * OPEN_CHANNEL message to the host and waits for
1133 * the host's response; however, if the host has
1134 * rescinded the channel before it receives the
1135 * OPEN_CHANNEL message, the host just silently
1136 * ignores the OPEN_CHANNEL message; as a result,
1137 * the guest's OFFER handler hangs for ever, if we
1138 * handle the RESCIND message in the same serialized
1139 * work queue: the RESCIND handler can not start to
1140 * run before the OFFER handler finishes.
1141 */
1142 if (vmbus_connection.ignore_any_offer_msg)
1143 break;
1144 queue_work(vmbus_connection.rescind_work_queue, &ctx->work);
1145 break;
1146
1147 case CHANNELMSG_OFFERCHANNEL:
1148 /*
1149 * The host sends the offer message of a given channel
1150 * before sending the rescind message of the same
1151 * channel. These messages are sent to the guest's
1152 * connect CPU; the guest then starts processing them
1153 * in the tasklet handler on this CPU:
1154 *
1155 * VMBUS_CONNECT_CPU
1156 *
1157 * [vmbus_on_msg_dpc()]
1158 * atomic_inc() // CHANNELMSG_OFFERCHANNEL
1159 * queue_work()
1160 * ...
1161 * [vmbus_on_msg_dpc()]
1162 * schedule_work() // CHANNELMSG_RESCIND_CHANNELOFFER
1163 *
1164 * We rely on the memory-ordering properties of the
1165 * queue_work() and schedule_work() primitives, which
1166 * guarantee that the atomic increment will be visible
1167 * to the CPUs which will execute the offer & rescind
1168 * works by the time these works will start execution.
1169 */
1170 if (vmbus_connection.ignore_any_offer_msg)
1171 break;
1172 atomic_inc(&vmbus_connection.offer_in_progress);
1173 fallthrough;
1174
1175 default:
1176 queue_work(vmbus_connection.work_queue, &ctx->work);
1177 }
1178 } else
1179 entry->message_handler(hdr);
1180
1181 msg_handled:
1182 vmbus_signal_eom(msg, message_type);
1183 }
1184
vmbus_on_msg_dpc(unsigned long data)1185 void vmbus_on_msg_dpc(unsigned long data)
1186 {
1187 struct hv_per_cpu_context *hv_cpu = (void *)data;
1188
1189 __vmbus_on_msg_dpc(hv_cpu->hyp_synic_message_page);
1190 __vmbus_on_msg_dpc(hv_cpu->para_synic_message_page);
1191 }
1192
1193 #ifdef CONFIG_PM_SLEEP
1194 /*
1195 * Fake RESCIND_CHANNEL messages to clean up hv_sock channels by force for
1196 * hibernation, because hv_sock connections can not persist across hibernation.
1197 */
vmbus_force_channel_rescinded(struct vmbus_channel * channel)1198 static void vmbus_force_channel_rescinded(struct vmbus_channel *channel)
1199 {
1200 struct onmessage_work_context *ctx;
1201 struct vmbus_channel_rescind_offer *rescind;
1202
1203 WARN_ON(!is_hvsock_channel(channel));
1204
1205 /*
1206 * Allocation size is small and the allocation should really not fail,
1207 * otherwise the state of the hv_sock connections ends up in limbo.
1208 */
1209 ctx = kzalloc(sizeof(*ctx) + sizeof(*rescind),
1210 GFP_KERNEL | __GFP_NOFAIL);
1211
1212 /*
1213 * So far, these are not really used by Linux. Just set them to the
1214 * reasonable values conforming to the definitions of the fields.
1215 */
1216 ctx->msg.header.message_type = 1;
1217 ctx->msg.header.payload_size = sizeof(*rescind);
1218
1219 /* These values are actually used by Linux. */
1220 rescind = (struct vmbus_channel_rescind_offer *)ctx->msg.payload;
1221 rescind->header.msgtype = CHANNELMSG_RESCIND_CHANNELOFFER;
1222 rescind->child_relid = channel->offermsg.child_relid;
1223
1224 INIT_WORK(&ctx->work, vmbus_onmessage_work);
1225
1226 queue_work(vmbus_connection.work_queue, &ctx->work);
1227 }
1228 #endif /* CONFIG_PM_SLEEP */
1229
1230 /*
1231 * Schedule all channels with events pending.
1232 * The event page can be directly checked to get the id of
1233 * the channel that has the interrupt pending.
1234 */
vmbus_chan_sched(void * event_page_addr)1235 static void vmbus_chan_sched(void *event_page_addr)
1236 {
1237 unsigned long *recv_int_page;
1238 u32 maxbits, relid;
1239 union hv_synic_event_flags *event;
1240
1241 if (!event_page_addr)
1242 return;
1243 event = (union hv_synic_event_flags *)event_page_addr + VMBUS_MESSAGE_SINT;
1244
1245 maxbits = READ_ONCE(vmbus_connection.relid_hiwater) + 1;
1246 recv_int_page = event->flags;
1247
1248 if (unlikely(!recv_int_page))
1249 return;
1250
1251 for_each_set_bit(relid, recv_int_page, maxbits) {
1252 void (*callback_fn)(void *context);
1253 struct vmbus_channel *channel;
1254
1255 if (!sync_test_and_clear_bit(relid, recv_int_page))
1256 continue;
1257
1258 /* Special case - vmbus channel protocol msg */
1259 if (relid == 0)
1260 continue;
1261
1262 /*
1263 * Pairs with the kfree_rcu() in vmbus_chan_release().
1264 * Guarantees that the channel data structure doesn't
1265 * get freed while the channel pointer below is being
1266 * dereferenced.
1267 */
1268 rcu_read_lock();
1269
1270 /* Find channel based on relid */
1271 channel = relid2channel(relid);
1272 if (channel == NULL)
1273 goto sched_unlock_rcu;
1274
1275 if (channel->rescind)
1276 goto sched_unlock_rcu;
1277
1278 /*
1279 * Make sure that the ring buffer data structure doesn't get
1280 * freed while we dereference the ring buffer pointer. Test
1281 * for the channel's onchannel_callback being NULL within a
1282 * sched_lock critical section. See also the inline comments
1283 * in vmbus_reset_channel_cb().
1284 */
1285 spin_lock(&channel->sched_lock);
1286
1287 callback_fn = channel->onchannel_callback;
1288 if (unlikely(callback_fn == NULL))
1289 goto sched_unlock;
1290
1291 trace_vmbus_chan_sched(channel);
1292
1293 ++channel->interrupts;
1294
1295 switch (channel->callback_mode) {
1296 case HV_CALL_ISR:
1297 (*callback_fn)(channel->channel_callback_context);
1298 break;
1299
1300 case HV_CALL_BATCHED:
1301 hv_begin_read(&channel->inbound);
1302 fallthrough;
1303 case HV_CALL_DIRECT:
1304 tasklet_schedule(&channel->callback_event);
1305 }
1306
1307 sched_unlock:
1308 spin_unlock(&channel->sched_lock);
1309 sched_unlock_rcu:
1310 rcu_read_unlock();
1311 }
1312 }
1313
vmbus_message_sched(struct hv_per_cpu_context * hv_cpu,void * message_page_addr)1314 static void vmbus_message_sched(struct hv_per_cpu_context *hv_cpu, void *message_page_addr)
1315 {
1316 struct hv_message *msg;
1317
1318 if (!message_page_addr)
1319 return;
1320 msg = (struct hv_message *)message_page_addr + VMBUS_MESSAGE_SINT;
1321
1322 /* Check if there are actual msgs to be processed */
1323 if (msg->header.message_type != HVMSG_NONE) {
1324 if (msg->header.message_type == HVMSG_TIMER_EXPIRED) {
1325 hv_stimer0_isr();
1326 vmbus_signal_eom(msg, HVMSG_TIMER_EXPIRED);
1327 } else {
1328 tasklet_schedule(&hv_cpu->msg_dpc);
1329 }
1330 }
1331 }
1332
__vmbus_isr(void)1333 static void __vmbus_isr(void)
1334 {
1335 struct hv_per_cpu_context *hv_cpu
1336 = this_cpu_ptr(hv_context.cpu_context);
1337
1338 vmbus_chan_sched(hv_cpu->hyp_synic_event_page);
1339 vmbus_chan_sched(hv_cpu->para_synic_event_page);
1340
1341 vmbus_message_sched(hv_cpu, hv_cpu->hyp_synic_message_page);
1342 vmbus_message_sched(hv_cpu, hv_cpu->para_synic_message_page);
1343 }
1344
1345 static DEFINE_PER_CPU(bool, vmbus_irq_pending);
1346 static DEFINE_PER_CPU(struct task_struct *, vmbus_irqd);
1347
vmbus_irqd_wake(void)1348 static void vmbus_irqd_wake(void)
1349 {
1350 struct task_struct *tsk = __this_cpu_read(vmbus_irqd);
1351
1352 __this_cpu_write(vmbus_irq_pending, true);
1353 wake_up_process(tsk);
1354 }
1355
vmbus_irqd_setup(unsigned int cpu)1356 static void vmbus_irqd_setup(unsigned int cpu)
1357 {
1358 sched_set_fifo(current);
1359 }
1360
vmbus_irqd_should_run(unsigned int cpu)1361 static int vmbus_irqd_should_run(unsigned int cpu)
1362 {
1363 return __this_cpu_read(vmbus_irq_pending);
1364 }
1365
run_vmbus_irqd(unsigned int cpu)1366 static void run_vmbus_irqd(unsigned int cpu)
1367 {
1368 __this_cpu_write(vmbus_irq_pending, false);
1369 __vmbus_isr();
1370 }
1371
1372 static struct smp_hotplug_thread vmbus_irq_threads = {
1373 .store = &vmbus_irqd,
1374 .setup = vmbus_irqd_setup,
1375 .thread_should_run = vmbus_irqd_should_run,
1376 .thread_fn = run_vmbus_irqd,
1377 .thread_comm = "vmbus_irq/%u",
1378 };
1379
vmbus_isr(void)1380 void vmbus_isr(void)
1381 {
1382 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
1383 vmbus_irqd_wake();
1384 } else {
1385 static DEFINE_WAIT_OVERRIDE_MAP(vmbus_map, LD_WAIT_CONFIG);
1386
1387 /*
1388 * vmbus_isr is never force-threaded and always invoked at hard
1389 * IRQ level. __vmbus_isr() below can acquire a spinlock_t
1390 * which becomes a sleeping lock and must not be acquired in
1391 * this context. Therefore on PREEMPT_RT this will be threaded
1392 * via vmbus_irqd_wake(). On non-PREEMPT the annotation lets
1393 * lockdep know that acquiring a spinlock_t is not an issue.
1394 */
1395 lock_map_acquire_try(&vmbus_map);
1396 __vmbus_isr();
1397 lock_map_release(&vmbus_map);
1398 }
1399 }
1400 EXPORT_SYMBOL_FOR_MODULES(vmbus_isr, "mshv_vtl");
1401
vmbus_percpu_isr(int irq,void * dev_id)1402 static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id)
1403 {
1404 vmbus_isr();
1405 return IRQ_HANDLED;
1406 }
1407
vmbus_percpu_work(struct work_struct * work)1408 static void vmbus_percpu_work(struct work_struct *work)
1409 {
1410 unsigned int cpu = smp_processor_id();
1411
1412 hv_synic_init(cpu);
1413 }
1414
vmbus_alloc_synic_and_connect(void)1415 static int vmbus_alloc_synic_and_connect(void)
1416 {
1417 int ret, cpu;
1418 struct work_struct __percpu *works;
1419
1420 ret = hv_synic_alloc();
1421 if (ret < 0)
1422 goto err_alloc;
1423
1424 works = alloc_percpu(struct work_struct);
1425 if (!works) {
1426 ret = -ENOMEM;
1427 goto err_alloc;
1428 }
1429
1430 /*
1431 * Initialize the per-cpu interrupt state and stimer state.
1432 * Then connect to the host.
1433 */
1434 cpus_read_lock();
1435 for_each_online_cpu(cpu) {
1436 struct work_struct *work = per_cpu_ptr(works, cpu);
1437
1438 INIT_WORK(work, vmbus_percpu_work);
1439 schedule_work_on(cpu, work);
1440 }
1441
1442 for_each_online_cpu(cpu)
1443 flush_work(per_cpu_ptr(works, cpu));
1444
1445 /* Register the callbacks for possible CPU online/offline'ing */
1446 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, "hyperv/vmbus:online",
1447 hv_synic_init, hv_synic_cleanup);
1448 cpus_read_unlock();
1449 free_percpu(works);
1450 if (ret < 0)
1451 goto err_alloc;
1452 hyperv_cpuhp_online = ret;
1453
1454 ret = vmbus_connect();
1455 if (ret)
1456 goto err_connect;
1457 return 0;
1458
1459 err_connect:
1460 cpuhp_remove_state(hyperv_cpuhp_online);
1461 return -ENODEV;
1462 err_alloc:
1463 hv_synic_free();
1464 return -ENOMEM;
1465 }
1466
1467 /*
1468 * vmbus_bus_init -Main vmbus driver initialization routine.
1469 *
1470 * Here, we
1471 * - initialize the vmbus driver context
1472 * - invoke the vmbus hv main init routine
1473 * - retrieve the channel offers
1474 */
vmbus_bus_init(void)1475 static int vmbus_bus_init(void)
1476 {
1477 int ret;
1478
1479 ret = hv_init();
1480 if (ret != 0) {
1481 pr_err("Unable to initialize the hypervisor - 0x%x\n", ret);
1482 return ret;
1483 }
1484
1485 ret = bus_register(&hv_bus);
1486 if (ret)
1487 return ret;
1488
1489 /*
1490 * VMbus interrupts are best modeled as per-cpu interrupts. If
1491 * on an architecture with support for per-cpu IRQs (e.g. ARM64),
1492 * allocate a per-cpu IRQ using standard Linux kernel functionality.
1493 * If not on such an architecture (e.g., x86/x64), then rely on
1494 * code in the arch-specific portion of the code tree to connect
1495 * the VMbus interrupt handler.
1496 */
1497
1498 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
1499 ret = smpboot_register_percpu_thread(&vmbus_irq_threads);
1500 if (ret)
1501 goto err_kthread;
1502 }
1503
1504 if (vmbus_irq == -1) {
1505 hv_setup_vmbus_handler(vmbus_isr);
1506 } else {
1507 ret = request_percpu_irq(vmbus_irq, vmbus_percpu_isr,
1508 "Hyper-V VMbus", &vmbus_evt);
1509 if (ret) {
1510 pr_err("Can't request Hyper-V VMbus IRQ %d, Err %d",
1511 vmbus_irq, ret);
1512 goto err_setup;
1513 }
1514 }
1515
1516 /*
1517 * Cache the value as getting it involves a VM exit on x86(_64), and
1518 * doing that on each VP while initializing SynIC's wastes time.
1519 */
1520 is_confidential = ms_hyperv.confidential_vmbus_available;
1521 if (is_confidential)
1522 pr_info("Establishing connection to the confidential VMBus\n");
1523 hv_para_set_sint_proxy(!is_confidential);
1524 ret = vmbus_alloc_synic_and_connect();
1525 if (ret)
1526 goto err_connect;
1527
1528 /*
1529 * Always register the vmbus unload panic notifier because we
1530 * need to shut the VMbus channel connection on panic.
1531 */
1532 atomic_notifier_chain_register(&panic_notifier_list,
1533 &hyperv_panic_vmbus_unload_block);
1534
1535 vmbus_request_offers();
1536
1537 return 0;
1538
1539 err_connect:
1540 if (vmbus_irq == -1)
1541 hv_remove_vmbus_handler();
1542 else
1543 free_percpu_irq(vmbus_irq, &vmbus_evt);
1544 err_setup:
1545 if (IS_ENABLED(CONFIG_PREEMPT_RT))
1546 smpboot_unregister_percpu_thread(&vmbus_irq_threads);
1547 err_kthread:
1548 bus_unregister(&hv_bus);
1549 return ret;
1550 }
1551
1552 /**
1553 * __vmbus_driver_register() - Register a vmbus's driver
1554 * @hv_driver: Pointer to driver structure you want to register
1555 * @owner: owner module of the drv
1556 * @mod_name: module name string
1557 *
1558 * Registers the given driver with Linux through the 'driver_register()' call
1559 * and sets up the hyper-v vmbus handling for this driver.
1560 * It will return the state of the 'driver_register()' call.
1561 *
1562 */
__vmbus_driver_register(struct hv_driver * hv_driver,struct module * owner,const char * mod_name)1563 int __vmbus_driver_register(struct hv_driver *hv_driver, struct module *owner, const char *mod_name)
1564 {
1565 int ret;
1566
1567 if (!hv_vmbus_exists())
1568 return -ENODEV;
1569
1570 pr_info("registering driver %s\n", hv_driver->name);
1571
1572 hv_driver->driver.name = hv_driver->name;
1573 hv_driver->driver.owner = owner;
1574 hv_driver->driver.mod_name = mod_name;
1575 hv_driver->driver.bus = &hv_bus;
1576
1577 spin_lock_init(&hv_driver->dynids.lock);
1578 INIT_LIST_HEAD(&hv_driver->dynids.list);
1579
1580 ret = driver_register(&hv_driver->driver);
1581
1582 return ret;
1583 }
1584 EXPORT_SYMBOL_GPL(__vmbus_driver_register);
1585
1586 /**
1587 * vmbus_driver_unregister() - Unregister a vmbus's driver
1588 * @hv_driver: Pointer to driver structure you want to
1589 * un-register
1590 *
1591 * Un-register the given driver that was previous registered with a call to
1592 * vmbus_driver_register()
1593 */
vmbus_driver_unregister(struct hv_driver * hv_driver)1594 void vmbus_driver_unregister(struct hv_driver *hv_driver)
1595 {
1596 if (hv_vmbus_exists()) {
1597 pr_info("unregistering driver %s\n", hv_driver->name);
1598 driver_unregister(&hv_driver->driver);
1599 vmbus_free_dynids(hv_driver);
1600 }
1601 }
1602 EXPORT_SYMBOL_GPL(vmbus_driver_unregister);
1603
1604
1605 /*
1606 * Called when last reference to channel is gone.
1607 */
vmbus_chan_release(struct kobject * kobj)1608 static void vmbus_chan_release(struct kobject *kobj)
1609 {
1610 struct vmbus_channel *channel
1611 = container_of(kobj, struct vmbus_channel, kobj);
1612
1613 kfree_rcu(channel, rcu);
1614 }
1615
1616 struct vmbus_chan_attribute {
1617 struct attribute attr;
1618 ssize_t (*show)(struct vmbus_channel *chan, char *buf);
1619 ssize_t (*store)(struct vmbus_channel *chan,
1620 const char *buf, size_t count);
1621 };
1622 #define VMBUS_CHAN_ATTR(_name, _mode, _show, _store) \
1623 struct vmbus_chan_attribute chan_attr_##_name \
1624 = __ATTR(_name, _mode, _show, _store)
1625 #define VMBUS_CHAN_ATTR_RW(_name) \
1626 struct vmbus_chan_attribute chan_attr_##_name = __ATTR_RW(_name)
1627 #define VMBUS_CHAN_ATTR_RO(_name) \
1628 struct vmbus_chan_attribute chan_attr_##_name = __ATTR_RO(_name)
1629 #define VMBUS_CHAN_ATTR_WO(_name) \
1630 struct vmbus_chan_attribute chan_attr_##_name = __ATTR_WO(_name)
1631
vmbus_chan_attr_show(struct kobject * kobj,struct attribute * attr,char * buf)1632 static ssize_t vmbus_chan_attr_show(struct kobject *kobj,
1633 struct attribute *attr, char *buf)
1634 {
1635 const struct vmbus_chan_attribute *attribute
1636 = container_of(attr, struct vmbus_chan_attribute, attr);
1637 struct vmbus_channel *chan
1638 = container_of(kobj, struct vmbus_channel, kobj);
1639
1640 if (!attribute->show)
1641 return -EIO;
1642
1643 return attribute->show(chan, buf);
1644 }
1645
vmbus_chan_attr_store(struct kobject * kobj,struct attribute * attr,const char * buf,size_t count)1646 static ssize_t vmbus_chan_attr_store(struct kobject *kobj,
1647 struct attribute *attr, const char *buf,
1648 size_t count)
1649 {
1650 const struct vmbus_chan_attribute *attribute
1651 = container_of(attr, struct vmbus_chan_attribute, attr);
1652 struct vmbus_channel *chan
1653 = container_of(kobj, struct vmbus_channel, kobj);
1654
1655 if (!attribute->store)
1656 return -EIO;
1657
1658 return attribute->store(chan, buf, count);
1659 }
1660
1661 static const struct sysfs_ops vmbus_chan_sysfs_ops = {
1662 .show = vmbus_chan_attr_show,
1663 .store = vmbus_chan_attr_store,
1664 };
1665
out_mask_show(struct vmbus_channel * channel,char * buf)1666 static ssize_t out_mask_show(struct vmbus_channel *channel, char *buf)
1667 {
1668 struct hv_ring_buffer_info *rbi = &channel->outbound;
1669 ssize_t ret;
1670
1671 mutex_lock(&rbi->ring_buffer_mutex);
1672 if (!rbi->ring_buffer) {
1673 mutex_unlock(&rbi->ring_buffer_mutex);
1674 return -EINVAL;
1675 }
1676
1677 ret = sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask);
1678 mutex_unlock(&rbi->ring_buffer_mutex);
1679 return ret;
1680 }
1681 static VMBUS_CHAN_ATTR_RO(out_mask);
1682
in_mask_show(struct vmbus_channel * channel,char * buf)1683 static ssize_t in_mask_show(struct vmbus_channel *channel, char *buf)
1684 {
1685 struct hv_ring_buffer_info *rbi = &channel->inbound;
1686 ssize_t ret;
1687
1688 mutex_lock(&rbi->ring_buffer_mutex);
1689 if (!rbi->ring_buffer) {
1690 mutex_unlock(&rbi->ring_buffer_mutex);
1691 return -EINVAL;
1692 }
1693
1694 ret = sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask);
1695 mutex_unlock(&rbi->ring_buffer_mutex);
1696 return ret;
1697 }
1698 static VMBUS_CHAN_ATTR_RO(in_mask);
1699
read_avail_show(struct vmbus_channel * channel,char * buf)1700 static ssize_t read_avail_show(struct vmbus_channel *channel, char *buf)
1701 {
1702 struct hv_ring_buffer_info *rbi = &channel->inbound;
1703 ssize_t ret;
1704
1705 mutex_lock(&rbi->ring_buffer_mutex);
1706 if (!rbi->ring_buffer) {
1707 mutex_unlock(&rbi->ring_buffer_mutex);
1708 return -EINVAL;
1709 }
1710
1711 ret = sprintf(buf, "%u\n", hv_get_bytes_to_read(rbi));
1712 mutex_unlock(&rbi->ring_buffer_mutex);
1713 return ret;
1714 }
1715 static VMBUS_CHAN_ATTR_RO(read_avail);
1716
write_avail_show(struct vmbus_channel * channel,char * buf)1717 static ssize_t write_avail_show(struct vmbus_channel *channel, char *buf)
1718 {
1719 struct hv_ring_buffer_info *rbi = &channel->outbound;
1720 ssize_t ret;
1721
1722 mutex_lock(&rbi->ring_buffer_mutex);
1723 if (!rbi->ring_buffer) {
1724 mutex_unlock(&rbi->ring_buffer_mutex);
1725 return -EINVAL;
1726 }
1727
1728 ret = sprintf(buf, "%u\n", hv_get_bytes_to_write(rbi));
1729 mutex_unlock(&rbi->ring_buffer_mutex);
1730 return ret;
1731 }
1732 static VMBUS_CHAN_ATTR_RO(write_avail);
1733
target_cpu_show(struct vmbus_channel * channel,char * buf)1734 static ssize_t target_cpu_show(struct vmbus_channel *channel, char *buf)
1735 {
1736 return sprintf(buf, "%u\n", channel->target_cpu);
1737 }
1738
vmbus_channel_set_cpu(struct vmbus_channel * channel,u32 target_cpu)1739 int vmbus_channel_set_cpu(struct vmbus_channel *channel, u32 target_cpu)
1740 {
1741 u32 origin_cpu;
1742 int ret = 0;
1743
1744 lockdep_assert_cpus_held();
1745 lockdep_assert_held(&vmbus_connection.channel_mutex);
1746
1747 if (vmbus_proto_version < VERSION_WIN10_V4_1)
1748 return -EIO;
1749
1750 /* Validate target_cpu for the cpumask_test_cpu() operation below. */
1751 if (target_cpu >= nr_cpumask_bits)
1752 return -EINVAL;
1753
1754 if (!cpumask_test_cpu(target_cpu, housekeeping_cpumask(HK_TYPE_MANAGED_IRQ)))
1755 return -EINVAL;
1756
1757 if (!cpu_online(target_cpu))
1758 return -EINVAL;
1759
1760 /*
1761 * Synchronizes vmbus_channel_set_cpu() and channel closure:
1762 *
1763 * { Initially: state = CHANNEL_OPENED }
1764 *
1765 * CPU1 CPU2
1766 *
1767 * [vmbus_channel_set_cpu()] [vmbus_disconnect_ring()]
1768 *
1769 * LOCK channel_mutex LOCK channel_mutex
1770 * LOAD r1 = state LOAD r2 = state
1771 * IF (r1 == CHANNEL_OPENED) IF (r2 == CHANNEL_OPENED)
1772 * SEND MODIFYCHANNEL STORE state = CHANNEL_OPEN
1773 * [...] SEND CLOSECHANNEL
1774 * UNLOCK channel_mutex UNLOCK channel_mutex
1775 *
1776 * Forbids: r1 == r2 == CHANNEL_OPENED (i.e., CPU1's LOCK precedes
1777 * CPU2's LOCK) && CPU2's SEND precedes CPU1's SEND
1778 *
1779 * Note. The host processes the channel messages "sequentially", in
1780 * the order in which they are received on a per-partition basis.
1781 */
1782
1783 /*
1784 * Hyper-V will ignore MODIFYCHANNEL messages for "non-open" channels;
1785 * avoid sending the message and fail here for such channels.
1786 */
1787 if (channel->state != CHANNEL_OPENED_STATE) {
1788 ret = -EIO;
1789 goto end;
1790 }
1791
1792 origin_cpu = channel->target_cpu;
1793 if (target_cpu == origin_cpu)
1794 goto end;
1795
1796 if (vmbus_send_modifychannel(channel,
1797 hv_cpu_number_to_vp_number(target_cpu))) {
1798 ret = -EIO;
1799 goto end;
1800 }
1801
1802 /*
1803 * For version before VERSION_WIN10_V5_3, the following warning holds:
1804 *
1805 * Warning. At this point, there is *no* guarantee that the host will
1806 * have successfully processed the vmbus_send_modifychannel() request.
1807 * See the header comment of vmbus_send_modifychannel() for more info.
1808 *
1809 * Lags in the processing of the above vmbus_send_modifychannel() can
1810 * result in missed interrupts if the "old" target CPU is taken offline
1811 * before Hyper-V starts sending interrupts to the "new" target CPU.
1812 * But apart from this offlining scenario, the code tolerates such
1813 * lags. It will function correctly even if a channel interrupt comes
1814 * in on a CPU that is different from the channel target_cpu value.
1815 */
1816
1817 channel->target_cpu = target_cpu;
1818
1819 /* See init_vp_index(). */
1820 if (hv_is_perf_channel(channel))
1821 hv_update_allocated_cpus(origin_cpu, target_cpu);
1822
1823 /* Currently set only for storvsc channels. */
1824 if (channel->change_target_cpu_callback) {
1825 (*channel->change_target_cpu_callback)(channel,
1826 origin_cpu, target_cpu);
1827 }
1828
1829 end:
1830 return ret;
1831 }
1832
target_cpu_store(struct vmbus_channel * channel,const char * buf,size_t count)1833 static ssize_t target_cpu_store(struct vmbus_channel *channel,
1834 const char *buf, size_t count)
1835 {
1836 u32 target_cpu;
1837 ssize_t ret;
1838
1839 if (sscanf(buf, "%u", &target_cpu) != 1)
1840 return -EIO;
1841
1842 cpus_read_lock();
1843 mutex_lock(&vmbus_connection.channel_mutex);
1844 ret = vmbus_channel_set_cpu(channel, target_cpu);
1845 mutex_unlock(&vmbus_connection.channel_mutex);
1846 cpus_read_unlock();
1847
1848 return ret ?: count;
1849 }
1850 static VMBUS_CHAN_ATTR(cpu, 0644, target_cpu_show, target_cpu_store);
1851
channel_pending_show(struct vmbus_channel * channel,char * buf)1852 static ssize_t channel_pending_show(struct vmbus_channel *channel,
1853 char *buf)
1854 {
1855 return sprintf(buf, "%d\n",
1856 channel_pending(channel,
1857 vmbus_connection.monitor_pages[1]));
1858 }
1859 static VMBUS_CHAN_ATTR(pending, 0444, channel_pending_show, NULL);
1860
channel_latency_show(struct vmbus_channel * channel,char * buf)1861 static ssize_t channel_latency_show(struct vmbus_channel *channel,
1862 char *buf)
1863 {
1864 return sprintf(buf, "%d\n",
1865 channel_latency(channel,
1866 vmbus_connection.monitor_pages[1]));
1867 }
1868 static VMBUS_CHAN_ATTR(latency, 0444, channel_latency_show, NULL);
1869
channel_interrupts_show(struct vmbus_channel * channel,char * buf)1870 static ssize_t channel_interrupts_show(struct vmbus_channel *channel, char *buf)
1871 {
1872 return sprintf(buf, "%llu\n", channel->interrupts);
1873 }
1874 static VMBUS_CHAN_ATTR(interrupts, 0444, channel_interrupts_show, NULL);
1875
channel_events_show(struct vmbus_channel * channel,char * buf)1876 static ssize_t channel_events_show(struct vmbus_channel *channel, char *buf)
1877 {
1878 return sprintf(buf, "%llu\n", channel->sig_events);
1879 }
1880 static VMBUS_CHAN_ATTR(events, 0444, channel_events_show, NULL);
1881
channel_intr_in_full_show(struct vmbus_channel * channel,char * buf)1882 static ssize_t channel_intr_in_full_show(struct vmbus_channel *channel,
1883 char *buf)
1884 {
1885 return sprintf(buf, "%llu\n",
1886 (unsigned long long)channel->intr_in_full);
1887 }
1888 static VMBUS_CHAN_ATTR(intr_in_full, 0444, channel_intr_in_full_show, NULL);
1889
channel_intr_out_empty_show(struct vmbus_channel * channel,char * buf)1890 static ssize_t channel_intr_out_empty_show(struct vmbus_channel *channel,
1891 char *buf)
1892 {
1893 return sprintf(buf, "%llu\n",
1894 (unsigned long long)channel->intr_out_empty);
1895 }
1896 static VMBUS_CHAN_ATTR(intr_out_empty, 0444, channel_intr_out_empty_show, NULL);
1897
channel_out_full_first_show(struct vmbus_channel * channel,char * buf)1898 static ssize_t channel_out_full_first_show(struct vmbus_channel *channel,
1899 char *buf)
1900 {
1901 return sprintf(buf, "%llu\n",
1902 (unsigned long long)channel->out_full_first);
1903 }
1904 static VMBUS_CHAN_ATTR(out_full_first, 0444, channel_out_full_first_show, NULL);
1905
channel_out_full_total_show(struct vmbus_channel * channel,char * buf)1906 static ssize_t channel_out_full_total_show(struct vmbus_channel *channel,
1907 char *buf)
1908 {
1909 return sprintf(buf, "%llu\n",
1910 (unsigned long long)channel->out_full_total);
1911 }
1912 static VMBUS_CHAN_ATTR(out_full_total, 0444, channel_out_full_total_show, NULL);
1913
subchannel_monitor_id_show(struct vmbus_channel * channel,char * buf)1914 static ssize_t subchannel_monitor_id_show(struct vmbus_channel *channel,
1915 char *buf)
1916 {
1917 return sprintf(buf, "%u\n", channel->offermsg.monitorid);
1918 }
1919 static VMBUS_CHAN_ATTR(monitor_id, 0444, subchannel_monitor_id_show, NULL);
1920
subchannel_id_show(struct vmbus_channel * channel,char * buf)1921 static ssize_t subchannel_id_show(struct vmbus_channel *channel,
1922 char *buf)
1923 {
1924 return sprintf(buf, "%u\n",
1925 channel->offermsg.offer.sub_channel_index);
1926 }
1927 static VMBUS_CHAN_ATTR_RO(subchannel_id);
1928
hv_mmap_ring_buffer_wrapper(struct file * filp,struct kobject * kobj,const struct bin_attribute * attr,struct vm_area_struct * vma)1929 static int hv_mmap_ring_buffer_wrapper(struct file *filp, struct kobject *kobj,
1930 const struct bin_attribute *attr,
1931 struct vm_area_struct *vma)
1932 {
1933 struct vmbus_channel *channel = container_of(kobj, struct vmbus_channel, kobj);
1934 struct vm_area_desc desc;
1935 int err;
1936
1937 /*
1938 * hv_(create|remove)_ring_sysfs implementation ensures that
1939 * mmap_prepare_ring_buffer is not NULL.
1940 */
1941 compat_set_desc_from_vma(&desc, filp, vma);
1942 err = channel->mmap_prepare_ring_buffer(channel, &desc);
1943 if (err)
1944 return err;
1945
1946 return __compat_vma_mmap(&desc, vma);
1947 }
1948
1949 static struct bin_attribute chan_attr_ring_buffer = {
1950 .attr = {
1951 .name = "ring",
1952 .mode = 0600,
1953 },
1954 .mmap = hv_mmap_ring_buffer_wrapper,
1955 };
1956 static struct attribute *vmbus_chan_attrs[] = {
1957 &chan_attr_out_mask.attr,
1958 &chan_attr_in_mask.attr,
1959 &chan_attr_read_avail.attr,
1960 &chan_attr_write_avail.attr,
1961 &chan_attr_cpu.attr,
1962 &chan_attr_pending.attr,
1963 &chan_attr_latency.attr,
1964 &chan_attr_interrupts.attr,
1965 &chan_attr_events.attr,
1966 &chan_attr_intr_in_full.attr,
1967 &chan_attr_intr_out_empty.attr,
1968 &chan_attr_out_full_first.attr,
1969 &chan_attr_out_full_total.attr,
1970 &chan_attr_monitor_id.attr,
1971 &chan_attr_subchannel_id.attr,
1972 NULL
1973 };
1974
1975 static const struct bin_attribute *vmbus_chan_bin_attrs[] = {
1976 &chan_attr_ring_buffer,
1977 NULL
1978 };
1979
1980 /*
1981 * Channel-level attribute_group callback function. Returns the permission for
1982 * each attribute, and returns 0 if an attribute is not visible.
1983 */
vmbus_chan_attr_is_visible(struct kobject * kobj,struct attribute * attr,int idx)1984 static umode_t vmbus_chan_attr_is_visible(struct kobject *kobj,
1985 struct attribute *attr, int idx)
1986 {
1987 const struct vmbus_channel *channel =
1988 container_of(kobj, struct vmbus_channel, kobj);
1989
1990 /* Hide the monitor attributes if the monitor mechanism is not used. */
1991 if (!channel->offermsg.monitor_allocated &&
1992 (attr == &chan_attr_pending.attr ||
1993 attr == &chan_attr_latency.attr ||
1994 attr == &chan_attr_monitor_id.attr))
1995 return 0;
1996
1997 return attr->mode;
1998 }
1999
vmbus_chan_bin_attr_is_visible(struct kobject * kobj,const struct bin_attribute * attr,int idx)2000 static umode_t vmbus_chan_bin_attr_is_visible(struct kobject *kobj,
2001 const struct bin_attribute *attr, int idx)
2002 {
2003 const struct vmbus_channel *channel =
2004 container_of(kobj, struct vmbus_channel, kobj);
2005
2006 /* Hide ring attribute if channel's ring_sysfs_visible is set to false */
2007 if (attr == &chan_attr_ring_buffer && !channel->ring_sysfs_visible)
2008 return 0;
2009
2010 return attr->attr.mode;
2011 }
2012
vmbus_chan_bin_size(struct kobject * kobj,const struct bin_attribute * bin_attr,int a)2013 static size_t vmbus_chan_bin_size(struct kobject *kobj,
2014 const struct bin_attribute *bin_attr, int a)
2015 {
2016 const struct vmbus_channel *channel =
2017 container_of(kobj, struct vmbus_channel, kobj);
2018
2019 return channel->ringbuffer_pagecount << PAGE_SHIFT;
2020 }
2021
2022 static const struct attribute_group vmbus_chan_group = {
2023 .attrs = vmbus_chan_attrs,
2024 .bin_attrs = vmbus_chan_bin_attrs,
2025 .is_visible = vmbus_chan_attr_is_visible,
2026 .is_bin_visible = vmbus_chan_bin_attr_is_visible,
2027 .bin_size = vmbus_chan_bin_size,
2028 };
2029
2030 static const struct kobj_type vmbus_chan_ktype = {
2031 .sysfs_ops = &vmbus_chan_sysfs_ops,
2032 .release = vmbus_chan_release,
2033 };
2034
2035 /**
2036 * hv_create_ring_sysfs() - create "ring" sysfs entry corresponding to ring buffers for a channel.
2037 * @channel: Pointer to vmbus_channel structure
2038 * @hv_mmap_prepare_ring_buffer: function pointer for initializing the function to be called on mmap
2039 * channel's "ring" sysfs node, which is for the ring buffer of that channel.
2040 * Function pointer is of below type:
2041 * int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel,
2042 * struct vm_area_desc *desc))
2043 * This has a pointer to the channel and a pointer to vm_area_desc,
2044 * used for mmap_prepare, as arguments.
2045 *
2046 * Sysfs node for ring buffer of a channel is created along with other fields, however its
2047 * visibility is disabled by default. Sysfs creation needs to be controlled when the use-case
2048 * is running.
2049 * For example, HV_NIC device is used either by uio_hv_generic or hv_netvsc at any given point of
2050 * time, and "ring" sysfs is needed only when uio_hv_generic is bound to that device. To avoid
2051 * exposing the ring buffer by default, this function is responsible to enable visibility of
2052 * ring for userspace to use.
2053 * Note: Race conditions can happen with userspace and it is not encouraged to create new
2054 * use-cases for this. This was added to maintain backward compatibility, while solving
2055 * one of the race conditions in uio_hv_generic while creating sysfs. See comments with
2056 * vmbus_add_dynid() and vmbus_device_register().
2057 *
2058 * Returns 0 on success or error code on failure.
2059 */
hv_create_ring_sysfs(struct vmbus_channel * channel,int (* hv_mmap_prepare_ring_buffer)(struct vmbus_channel * channel,struct vm_area_desc * desc))2060 int hv_create_ring_sysfs(struct vmbus_channel *channel,
2061 int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel,
2062 struct vm_area_desc *desc))
2063 {
2064 struct kobject *kobj = &channel->kobj;
2065
2066 channel->mmap_prepare_ring_buffer = hv_mmap_prepare_ring_buffer;
2067 channel->ring_sysfs_visible = true;
2068
2069 return sysfs_update_group(kobj, &vmbus_chan_group);
2070 }
2071 EXPORT_SYMBOL_GPL(hv_create_ring_sysfs);
2072
2073 /**
2074 * hv_remove_ring_sysfs() - remove ring sysfs entry corresponding to ring buffers for a channel.
2075 * @channel: Pointer to vmbus_channel structure
2076 *
2077 * Hide "ring" sysfs for a channel by changing its is_visible attribute and updating sysfs group.
2078 *
2079 * Returns 0 on success or error code on failure.
2080 */
hv_remove_ring_sysfs(struct vmbus_channel * channel)2081 int hv_remove_ring_sysfs(struct vmbus_channel *channel)
2082 {
2083 struct kobject *kobj = &channel->kobj;
2084 int ret;
2085
2086 channel->ring_sysfs_visible = false;
2087 ret = sysfs_update_group(kobj, &vmbus_chan_group);
2088 channel->mmap_prepare_ring_buffer = NULL;
2089 return ret;
2090 }
2091 EXPORT_SYMBOL_GPL(hv_remove_ring_sysfs);
2092
2093 /*
2094 * vmbus_add_channel_kobj - setup a sub-directory under device/channels
2095 */
vmbus_add_channel_kobj(struct hv_device * dev,struct vmbus_channel * channel)2096 int vmbus_add_channel_kobj(struct hv_device *dev, struct vmbus_channel *channel)
2097 {
2098 const struct device *device = &dev->device;
2099 struct kobject *kobj = &channel->kobj;
2100 u32 relid = channel->offermsg.child_relid;
2101 int ret;
2102
2103 kobj->kset = dev->channels_kset;
2104 ret = kobject_init_and_add(kobj, &vmbus_chan_ktype, NULL,
2105 "%u", relid);
2106 if (ret) {
2107 kobject_put(kobj);
2108 return ret;
2109 }
2110
2111 ret = sysfs_create_group(kobj, &vmbus_chan_group);
2112
2113 if (ret) {
2114 /*
2115 * The calling functions' error handling paths will cleanup the
2116 * empty channel directory.
2117 */
2118 kobject_put(kobj);
2119 dev_err(device, "Unable to set up channel sysfs files\n");
2120 return ret;
2121 }
2122
2123 kobject_uevent(kobj, KOBJ_ADD);
2124
2125 return 0;
2126 }
2127
2128 /*
2129 * vmbus_remove_channel_attr_group - remove the channel's attribute group
2130 */
vmbus_remove_channel_attr_group(struct vmbus_channel * channel)2131 void vmbus_remove_channel_attr_group(struct vmbus_channel *channel)
2132 {
2133 sysfs_remove_group(&channel->kobj, &vmbus_chan_group);
2134 }
2135
2136 /*
2137 * vmbus_device_create - Creates and registers a new child device
2138 * on the vmbus.
2139 */
vmbus_device_create(const guid_t * type,const guid_t * instance,struct vmbus_channel * channel)2140 struct hv_device *vmbus_device_create(const guid_t *type,
2141 const guid_t *instance,
2142 struct vmbus_channel *channel)
2143 {
2144 struct hv_device *child_device_obj;
2145
2146 child_device_obj = kzalloc_obj(struct hv_device);
2147 if (!child_device_obj) {
2148 pr_err("Unable to allocate device object for child device\n");
2149 return NULL;
2150 }
2151
2152 child_device_obj->channel = channel;
2153 guid_copy(&child_device_obj->dev_type, type);
2154 guid_copy(&child_device_obj->dev_instance, instance);
2155 child_device_obj->vendor_id = PCI_VENDOR_ID_MICROSOFT;
2156
2157 return child_device_obj;
2158 }
2159
2160 /*
2161 * vmbus_device_register - Register the child device
2162 */
vmbus_device_register(struct hv_device * child_device_obj)2163 int vmbus_device_register(struct hv_device *child_device_obj)
2164 {
2165 struct kobject *kobj = &child_device_obj->device.kobj;
2166 int ret;
2167
2168 dev_set_name(&child_device_obj->device, "%pUl",
2169 &child_device_obj->channel->offermsg.offer.if_instance);
2170
2171 child_device_obj->device.bus = &hv_bus;
2172 child_device_obj->device.parent = vmbus_root_device;
2173 child_device_obj->device.release = vmbus_device_release;
2174
2175 child_device_obj->device.dma_parms = &child_device_obj->dma_parms;
2176 child_device_obj->device.dma_mask = &child_device_obj->dma_mask;
2177 dma_set_mask(&child_device_obj->device, DMA_BIT_MASK(64));
2178 dma_set_coherent_mask(&child_device_obj->device, DMA_BIT_MASK(64));
2179
2180 /*
2181 * Register with the LDM. This will kick off the driver/device
2182 * binding...which will eventually call vmbus_match() and vmbus_probe()
2183 */
2184 ret = device_register(&child_device_obj->device);
2185 if (ret) {
2186 pr_err("Unable to register child device\n");
2187 put_device(&child_device_obj->device);
2188 return ret;
2189 }
2190
2191 /*
2192 * If device_register() found a driver to assign to the device, the
2193 * driver's probe function has already run at this point. If that
2194 * probe function accesses or operates on the "channels" subdirectory
2195 * in sysfs, those operations will have failed because the "channels"
2196 * subdirectory doesn't exist until the code below runs. Or if the
2197 * probe function creates a /dev entry, a user space program could
2198 * find and open the /dev entry, and then create a race by accessing
2199 * the "channels" subdirectory while the creation steps are in progress
2200 * here. The race can't result in a kernel failure, but the user space
2201 * program may get an error in accessing "channels" or its
2202 * subdirectories. See also comments with vmbus_add_dynid() about a
2203 * related race condition.
2204 */
2205 child_device_obj->channels_kset = kset_create_and_add("channels",
2206 NULL, kobj);
2207 if (!child_device_obj->channels_kset) {
2208 ret = -ENOMEM;
2209 goto err_dev_unregister;
2210 }
2211
2212 ret = vmbus_add_channel_kobj(child_device_obj,
2213 child_device_obj->channel);
2214 if (ret) {
2215 pr_err("Unable to register primary channel\n");
2216 goto err_kset_unregister;
2217 }
2218 hv_debug_add_dev_dir(child_device_obj);
2219
2220 return 0;
2221
2222 err_kset_unregister:
2223 kset_unregister(child_device_obj->channels_kset);
2224
2225 err_dev_unregister:
2226 device_unregister(&child_device_obj->device);
2227 return ret;
2228 }
2229
2230 /*
2231 * vmbus_device_unregister - Remove the specified child device
2232 * from the vmbus.
2233 */
vmbus_device_unregister(struct hv_device * device_obj)2234 void vmbus_device_unregister(struct hv_device *device_obj)
2235 {
2236 pr_debug("child device %s unregistered\n",
2237 dev_name(&device_obj->device));
2238
2239 kset_unregister(device_obj->channels_kset);
2240
2241 /*
2242 * Kick off the process of unregistering the device.
2243 * This will call vmbus_remove() and eventually vmbus_device_release()
2244 */
2245 device_unregister(&device_obj->device);
2246 }
2247 EXPORT_SYMBOL_GPL(vmbus_device_unregister);
2248
2249 #ifdef CONFIG_ACPI
2250 /*
2251 * VMBUS is an acpi enumerated device. Get the information we
2252 * need from DSDT.
2253 */
vmbus_walk_resources(struct acpi_resource * res,void * ctx)2254 static acpi_status vmbus_walk_resources(struct acpi_resource *res, void *ctx)
2255 {
2256 resource_size_t start = 0;
2257 resource_size_t end = 0;
2258 struct resource *new_res;
2259 struct resource **old_res = &hyperv_mmio;
2260 struct resource **prev_res = NULL;
2261 struct resource r;
2262
2263 switch (res->type) {
2264
2265 /*
2266 * "Address" descriptors are for bus windows. Ignore
2267 * "memory" descriptors, which are for registers on
2268 * devices.
2269 */
2270 case ACPI_RESOURCE_TYPE_ADDRESS32:
2271 start = res->data.address32.address.minimum;
2272 end = res->data.address32.address.maximum;
2273 break;
2274
2275 case ACPI_RESOURCE_TYPE_ADDRESS64:
2276 start = res->data.address64.address.minimum;
2277 end = res->data.address64.address.maximum;
2278 break;
2279
2280 /*
2281 * The IRQ information is needed only on ARM64, which Hyper-V
2282 * sets up in the extended format. IRQ information is present
2283 * on x86/x64 in the non-extended format but it is not used by
2284 * Linux. So don't bother checking for the non-extended format.
2285 */
2286 case ACPI_RESOURCE_TYPE_EXTENDED_IRQ:
2287 if (!acpi_dev_resource_interrupt(res, 0, &r)) {
2288 pr_err("Unable to parse Hyper-V ACPI interrupt\n");
2289 return AE_ERROR;
2290 }
2291 /* ARM64 INTID for VMbus */
2292 vmbus_interrupt = res->data.extended_irq.interrupts[0];
2293 /* Linux IRQ number */
2294 vmbus_irq = r.start;
2295 return AE_OK;
2296
2297 default:
2298 /* Unused resource type */
2299 return AE_OK;
2300
2301 }
2302 /*
2303 * Ignore ranges that are below 1MB, as they're not
2304 * necessary or useful here.
2305 */
2306 if (end < 0x100000)
2307 return AE_OK;
2308
2309 new_res = kzalloc_obj(*new_res, GFP_ATOMIC);
2310 if (!new_res)
2311 return AE_NO_MEMORY;
2312
2313 /* If this range overlaps the virtual TPM, truncate it. */
2314 if (end >= VTPM_BASE_ADDRESS && start < VTPM_BASE_ADDRESS)
2315 end = VTPM_BASE_ADDRESS - 1;
2316
2317 new_res->name = "hyperv mmio";
2318 new_res->flags = IORESOURCE_MEM;
2319 new_res->start = start;
2320 new_res->end = end;
2321
2322 /*
2323 * If two ranges are adjacent, merge them.
2324 */
2325 do {
2326 if (!*old_res) {
2327 *old_res = new_res;
2328 break;
2329 }
2330
2331 if (((*old_res)->end + 1) == new_res->start) {
2332 (*old_res)->end = new_res->end;
2333 kfree(new_res);
2334 break;
2335 }
2336
2337 if ((*old_res)->start == new_res->end + 1) {
2338 (*old_res)->start = new_res->start;
2339 kfree(new_res);
2340 break;
2341 }
2342
2343 if ((*old_res)->start > new_res->end) {
2344 new_res->sibling = *old_res;
2345 if (prev_res)
2346 (*prev_res)->sibling = new_res;
2347 *old_res = new_res;
2348 break;
2349 }
2350
2351 prev_res = old_res;
2352 old_res = &(*old_res)->sibling;
2353
2354 } while (1);
2355
2356 return AE_OK;
2357 }
2358 #endif
2359
vmbus_mmio_remove(void)2360 static void vmbus_mmio_remove(void)
2361 {
2362 struct resource *cur_res;
2363 struct resource *next_res;
2364
2365 if (hyperv_mmio) {
2366 if (fb_mmio) {
2367 __release_region(hyperv_mmio, fb_mmio->start,
2368 resource_size(fb_mmio));
2369 fb_mmio = NULL;
2370 }
2371
2372 for (cur_res = hyperv_mmio; cur_res; cur_res = next_res) {
2373 next_res = cur_res->sibling;
2374 kfree(cur_res);
2375 }
2376 }
2377 }
2378
vmbus_reserve_fb(void)2379 static void __maybe_unused vmbus_reserve_fb(void)
2380 {
2381 resource_size_t start = 0, size;
2382 resource_size_t low_mmio_base;
2383 struct pci_dev *pdev;
2384
2385 if (efi_enabled(EFI_BOOT)) {
2386 /* Gen2 VM: get FB base from EFI framebuffer */
2387 if (IS_ENABLED(CONFIG_SYSFB)) {
2388 start = sysfb_primary_display.screen.lfb_base;
2389 size = max_t(__u32, sysfb_primary_display.screen.lfb_size, 0x800000);
2390
2391 low_mmio_base = hyperv_mmio->start;
2392 if (!low_mmio_base || upper_32_bits(low_mmio_base) ||
2393 (start && start < low_mmio_base)) {
2394 pr_warn("Unexpected low mmio base %pa\n", &low_mmio_base);
2395 } else {
2396 /*
2397 * If the kdump/kexec or CVM kernel's lfb_base
2398 * is 0, fall back to the low mmio base.
2399 */
2400 if (!start)
2401 start = low_mmio_base;
2402 /*
2403 * Reserve half of the space below 4GB for high
2404 * resolutions, but cap the reservation to 128MB.
2405 */
2406 size = min((SZ_4G - start) / 2, SZ_128M);
2407 }
2408 }
2409 } else {
2410 /* Gen1 VM: get FB base from PCI */
2411 pdev = pci_get_device(PCI_VENDOR_ID_MICROSOFT,
2412 PCI_DEVICE_ID_HYPERV_VIDEO, NULL);
2413 if (!pdev)
2414 return;
2415
2416 if (pdev->resource[0].flags & IORESOURCE_MEM) {
2417 start = pci_resource_start(pdev, 0);
2418 size = pci_resource_len(pdev, 0);
2419 }
2420
2421 /*
2422 * Release the PCI device so hyperv_drm driver can grab it
2423 * later.
2424 */
2425 pci_dev_put(pdev);
2426 }
2427
2428 if (!start) {
2429 pr_warn("Unexpected framebuffer mmio base of zero\n");
2430 return;
2431 }
2432
2433 /*
2434 * Make a claim for the frame buffer in the resource tree under the
2435 * first node, which will be the one below 4GB. The length seems to
2436 * be underreported, particularly in a Generation 1 VM. So start out
2437 * reserving a larger area and make it smaller until it succeeds.
2438 */
2439 for (; !fb_mmio && (size >= 0x100000); size >>= 1)
2440 fb_mmio = __request_region(hyperv_mmio, start, size, fb_mmio_name, 0);
2441
2442 pr_info("hv_mmio=%pR,%pR fb=%pR\n", hyperv_mmio, hyperv_mmio->sibling, fb_mmio);
2443 }
2444
2445 /**
2446 * vmbus_allocate_mmio() - Pick a memory-mapped I/O range.
2447 * @new: If successful, supplied a pointer to the
2448 * allocated MMIO space.
2449 * @device_obj: Identifies the caller
2450 * @min: Minimum guest physical address of the
2451 * allocation
2452 * @max: Maximum guest physical address
2453 * @size: Size of the range to be allocated
2454 * @align: Alignment of the range to be allocated
2455 * @fb_overlap_ok: Whether this allocation can be allowed
2456 * to overlap the video frame buffer.
2457 *
2458 * This function walks the resources granted to VMBus by the
2459 * _CRS object in the ACPI namespace underneath the parent
2460 * "bridge" whether that's a root PCI bus in the Generation 1
2461 * case or a Module Device in the Generation 2 case. It then
2462 * attempts to allocate from the global MMIO pool in a way that
2463 * matches the constraints supplied in these parameters and by
2464 * that _CRS.
2465 *
2466 * Return: 0 on success, -errno on failure
2467 */
vmbus_allocate_mmio(struct resource ** new,struct hv_device * device_obj,resource_size_t min,resource_size_t max,resource_size_t size,resource_size_t align,bool fb_overlap_ok)2468 int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj,
2469 resource_size_t min, resource_size_t max,
2470 resource_size_t size, resource_size_t align,
2471 bool fb_overlap_ok)
2472 {
2473 struct resource *iter, *shadow;
2474 resource_size_t range_min, range_max, start, end;
2475 const char *dev_n = dev_name(&device_obj->device);
2476 int retval;
2477
2478 retval = -ENXIO;
2479 mutex_lock(&hyperv_mmio_lock);
2480
2481 /*
2482 * If overlaps with frame buffers are allowed, then first attempt to
2483 * make the allocation from within the reserved region. Because it
2484 * is already reserved, no shadow allocation is necessary.
2485 */
2486 if (fb_overlap_ok && fb_mmio && !(min > fb_mmio->end) &&
2487 !(max < fb_mmio->start)) {
2488
2489 range_min = fb_mmio->start;
2490 range_max = fb_mmio->end;
2491 start = (range_min + align - 1) & ~(align - 1);
2492 for (; start + size - 1 <= range_max; start += align) {
2493 *new = request_mem_region_exclusive(start, size, dev_n);
2494 if (*new) {
2495 retval = 0;
2496 goto exit;
2497 }
2498 }
2499 }
2500
2501 for (iter = hyperv_mmio; iter; iter = iter->sibling) {
2502 if ((iter->start >= max) || (iter->end <= min))
2503 continue;
2504
2505 range_min = iter->start;
2506 range_max = iter->end;
2507 start = (range_min + align - 1) & ~(align - 1);
2508 for (; start + size - 1 <= range_max; start += align) {
2509 end = start + size - 1;
2510
2511 /* Skip the whole fb_mmio region if not fb_overlap_ok */
2512 if (!fb_overlap_ok && fb_mmio &&
2513 (((start >= fb_mmio->start) && (start <= fb_mmio->end)) ||
2514 ((end >= fb_mmio->start) && (end <= fb_mmio->end))))
2515 continue;
2516
2517 shadow = __request_region(iter, start, size, NULL,
2518 IORESOURCE_BUSY);
2519 if (!shadow)
2520 continue;
2521
2522 *new = request_mem_region_exclusive(start, size, dev_n);
2523 if (*new) {
2524 shadow->name = (char *)*new;
2525 retval = 0;
2526 goto exit;
2527 }
2528
2529 __release_region(iter, start, size);
2530 }
2531 }
2532
2533 exit:
2534 mutex_unlock(&hyperv_mmio_lock);
2535 return retval;
2536 }
2537 EXPORT_SYMBOL_GPL(vmbus_allocate_mmio);
2538
2539 /**
2540 * vmbus_free_mmio() - Free a memory-mapped I/O range.
2541 * @start: Base address of region to release.
2542 * @size: Size of the range to be allocated
2543 *
2544 * This function releases anything requested by
2545 * vmbus_mmio_allocate().
2546 */
vmbus_free_mmio(resource_size_t start,resource_size_t size)2547 void vmbus_free_mmio(resource_size_t start, resource_size_t size)
2548 {
2549 struct resource *iter;
2550
2551 mutex_lock(&hyperv_mmio_lock);
2552
2553 /*
2554 * If all bytes of the MMIO range to be released are within the
2555 * special case fb_mmio shadow region, skip releasing the shadow
2556 * region since no corresponding __request_region() was done
2557 * in vmbus_allocate_mmio().
2558 */
2559 if (fb_mmio && start >= fb_mmio->start &&
2560 (start + size - 1 <= fb_mmio->end))
2561 goto skip_shadow_release;
2562
2563 for (iter = hyperv_mmio; iter; iter = iter->sibling) {
2564 if ((iter->start >= start + size) || (iter->end <= start))
2565 continue;
2566
2567 __release_region(iter, start, size);
2568 }
2569
2570 skip_shadow_release:
2571 release_mem_region(start, size);
2572 mutex_unlock(&hyperv_mmio_lock);
2573
2574 }
2575 EXPORT_SYMBOL_GPL(vmbus_free_mmio);
2576
2577 #ifdef CONFIG_ACPI
vmbus_acpi_add(struct platform_device * pdev)2578 static int vmbus_acpi_add(struct platform_device *pdev)
2579 {
2580 acpi_status result;
2581 int ret_val = -ENODEV;
2582 struct acpi_device *ancestor;
2583 struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
2584
2585 vmbus_root_device = &device->dev;
2586
2587 /*
2588 * Older versions of Hyper-V for ARM64 fail to include the _CCA
2589 * method on the top level VMbus device in the DSDT. But devices
2590 * are hardware coherent in all current Hyper-V use cases, so fix
2591 * up the ACPI device to behave as if _CCA is present and indicates
2592 * hardware coherence.
2593 */
2594 ACPI_COMPANION_SET(&device->dev, device);
2595 if (IS_ENABLED(CONFIG_ACPI_CCA_REQUIRED) &&
2596 device_get_dma_attr(&device->dev) == DEV_DMA_NOT_SUPPORTED) {
2597 pr_info("No ACPI _CCA found; assuming coherent device I/O\n");
2598 device->flags.cca_seen = true;
2599 device->flags.coherent_dma = true;
2600 }
2601
2602 result = acpi_walk_resources(device->handle, METHOD_NAME__CRS,
2603 vmbus_walk_resources, NULL);
2604
2605 if (ACPI_FAILURE(result))
2606 goto acpi_walk_err;
2607 /*
2608 * Some ancestor of the vmbus acpi device (Gen1 or Gen2
2609 * firmware) is the VMOD that has the mmio ranges. Get that.
2610 */
2611 for (ancestor = acpi_dev_parent(device);
2612 ancestor && ancestor->handle != ACPI_ROOT_OBJECT;
2613 ancestor = acpi_dev_parent(ancestor)) {
2614 result = acpi_walk_resources(ancestor->handle, METHOD_NAME__CRS,
2615 vmbus_walk_resources, NULL);
2616
2617 if (ACPI_FAILURE(result))
2618 continue;
2619 if (hyperv_mmio) {
2620 vmbus_reserve_fb();
2621 break;
2622 }
2623 }
2624 ret_val = 0;
2625
2626 acpi_walk_err:
2627 if (ret_val)
2628 vmbus_mmio_remove();
2629 return ret_val;
2630 }
2631 #else
vmbus_acpi_add(struct platform_device * pdev)2632 static int vmbus_acpi_add(struct platform_device *pdev)
2633 {
2634 return 0;
2635 }
2636 #endif
2637 #ifndef HYPERVISOR_CALLBACK_VECTOR
vmbus_set_irq(struct platform_device * pdev)2638 static int vmbus_set_irq(struct platform_device *pdev)
2639 {
2640 struct irq_data *data;
2641 int irq;
2642 irq_hw_number_t hwirq;
2643
2644 irq = platform_get_irq(pdev, 0);
2645 /* platform_get_irq() may not return 0. */
2646 if (irq < 0)
2647 return irq;
2648
2649 data = irq_get_irq_data(irq);
2650 if (!data) {
2651 pr_err("No interrupt data for VMBus virq %d\n", irq);
2652 return -ENODEV;
2653 }
2654 hwirq = irqd_to_hwirq(data);
2655
2656 vmbus_irq = irq;
2657 vmbus_interrupt = hwirq;
2658 pr_debug("VMBus virq %d, hwirq %d\n", vmbus_irq, vmbus_interrupt);
2659
2660 return 0;
2661 }
2662 #endif
2663
vmbus_device_add(struct platform_device * pdev)2664 static int vmbus_device_add(struct platform_device *pdev)
2665 {
2666 struct resource **cur_res = &hyperv_mmio;
2667 struct of_range range;
2668 struct of_range_parser parser;
2669 struct device_node *np = pdev->dev.of_node;
2670 int ret;
2671
2672 vmbus_root_device = &pdev->dev;
2673
2674 ret = of_range_parser_init(&parser, np);
2675 if (ret)
2676 return ret;
2677
2678 #ifndef HYPERVISOR_CALLBACK_VECTOR
2679 ret = vmbus_set_irq(pdev);
2680 if (ret)
2681 return ret;
2682 #endif
2683 for_each_of_range(&parser, &range) {
2684 struct resource *res;
2685
2686 res = kzalloc_obj(*res);
2687 if (!res) {
2688 vmbus_mmio_remove();
2689 return -ENOMEM;
2690 }
2691
2692 res->name = "hyperv mmio";
2693 res->flags = range.flags;
2694 res->start = range.cpu_addr;
2695 res->end = range.cpu_addr + range.size;
2696
2697 *cur_res = res;
2698 cur_res = &res->sibling;
2699 }
2700
2701 return ret;
2702 }
2703
vmbus_platform_driver_probe(struct platform_device * pdev)2704 static int vmbus_platform_driver_probe(struct platform_device *pdev)
2705 {
2706 if (acpi_disabled)
2707 return vmbus_device_add(pdev);
2708 else
2709 return vmbus_acpi_add(pdev);
2710 }
2711
vmbus_platform_driver_remove(struct platform_device * pdev)2712 static void vmbus_platform_driver_remove(struct platform_device *pdev)
2713 {
2714 vmbus_mmio_remove();
2715 }
2716
2717 #ifdef CONFIG_PM_SLEEP
vmbus_bus_suspend(struct device * dev)2718 static int vmbus_bus_suspend(struct device *dev)
2719 {
2720 struct hv_per_cpu_context *hv_cpu = per_cpu_ptr(
2721 hv_context.cpu_context, VMBUS_CONNECT_CPU);
2722 struct vmbus_channel *channel, *sc;
2723
2724 tasklet_disable(&hv_cpu->msg_dpc);
2725 vmbus_connection.ignore_any_offer_msg = true;
2726 /* The tasklet_enable() takes care of providing a memory barrier */
2727 tasklet_enable(&hv_cpu->msg_dpc);
2728
2729 /* Drain all the workqueues as we are in suspend */
2730 drain_workqueue(vmbus_connection.rescind_work_queue);
2731 drain_workqueue(vmbus_connection.work_queue);
2732 drain_workqueue(vmbus_connection.handle_primary_chan_wq);
2733 drain_workqueue(vmbus_connection.handle_sub_chan_wq);
2734
2735 mutex_lock(&vmbus_connection.channel_mutex);
2736 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
2737 if (!is_hvsock_channel(channel))
2738 continue;
2739
2740 vmbus_force_channel_rescinded(channel);
2741 }
2742 mutex_unlock(&vmbus_connection.channel_mutex);
2743
2744 /*
2745 * Wait until all the sub-channels and hv_sock channels have been
2746 * cleaned up. Sub-channels should be destroyed upon suspend, otherwise
2747 * they would conflict with the new sub-channels that will be created
2748 * in the resume path. hv_sock channels should also be destroyed, but
2749 * a hv_sock channel of an established hv_sock connection can not be
2750 * really destroyed since it may still be referenced by the userspace
2751 * application, so we just force the hv_sock channel to be rescinded
2752 * by vmbus_force_channel_rescinded(), and the userspace application
2753 * will thoroughly destroy the channel after hibernation.
2754 *
2755 * Note: the counter nr_chan_close_on_suspend may never go above 0 if
2756 * the VM has no sub-channel and hv_sock channel, e.g. a 1-vCPU VM.
2757 */
2758 if (atomic_read(&vmbus_connection.nr_chan_close_on_suspend) > 0)
2759 wait_for_completion(&vmbus_connection.ready_for_suspend_event);
2760
2761 mutex_lock(&vmbus_connection.channel_mutex);
2762
2763 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
2764 /*
2765 * Remove the channel from the array of channels and invalidate
2766 * the channel's relid. Upon resume, vmbus_onoffer() will fix
2767 * up the relid (and other fields, if necessary) and add the
2768 * channel back to the array.
2769 */
2770 vmbus_channel_unmap_relid(channel);
2771 channel->offermsg.child_relid = INVALID_RELID;
2772
2773 if (is_hvsock_channel(channel)) {
2774 if (!channel->rescind) {
2775 pr_err("hv_sock channel not rescinded!\n");
2776 WARN_ON_ONCE(1);
2777 }
2778 continue;
2779 }
2780
2781 list_for_each_entry(sc, &channel->sc_list, sc_list) {
2782 pr_err("Sub-channel not deleted!\n");
2783 WARN_ON_ONCE(1);
2784 }
2785 }
2786
2787 mutex_unlock(&vmbus_connection.channel_mutex);
2788
2789 vmbus_initiate_unload(false);
2790
2791 return 0;
2792 }
2793
vmbus_bus_resume(struct device * dev)2794 static int vmbus_bus_resume(struct device *dev)
2795 {
2796 struct vmbus_channel *channel;
2797 struct vmbus_channel_msginfo *msginfo;
2798 size_t msgsize;
2799 int ret;
2800
2801 vmbus_connection.ignore_any_offer_msg = false;
2802
2803 /*
2804 * We only use the 'vmbus_proto_version', which was in use before
2805 * hibernation, to re-negotiate with the host.
2806 */
2807 if (!vmbus_proto_version) {
2808 pr_err("Invalid proto version = 0x%x\n", vmbus_proto_version);
2809 return -EINVAL;
2810 }
2811
2812 msgsize = sizeof(*msginfo) +
2813 sizeof(struct vmbus_channel_initiate_contact);
2814
2815 msginfo = kzalloc(msgsize, GFP_KERNEL);
2816
2817 if (msginfo == NULL)
2818 return -ENOMEM;
2819
2820 ret = vmbus_negotiate_version(msginfo, vmbus_proto_version);
2821
2822 kfree(msginfo);
2823
2824 if (ret != 0)
2825 return ret;
2826
2827 vmbus_request_offers();
2828
2829 mutex_lock(&vmbus_connection.channel_mutex);
2830 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
2831 if (channel->offermsg.child_relid != INVALID_RELID)
2832 continue;
2833
2834 /* hvsock channels are not expected to be present. */
2835 if (is_hvsock_channel(channel))
2836 continue;
2837
2838 pr_err("channel %pUl/%pUl not present after resume.\n",
2839 &channel->offermsg.offer.if_type,
2840 &channel->offermsg.offer.if_instance);
2841 /* ToDo: Cleanup these channels here */
2842 }
2843 mutex_unlock(&vmbus_connection.channel_mutex);
2844
2845 /* Reset the event for the next suspend. */
2846 reinit_completion(&vmbus_connection.ready_for_suspend_event);
2847
2848 return 0;
2849 }
2850 #else
2851 #define vmbus_bus_suspend NULL
2852 #define vmbus_bus_resume NULL
2853 #endif /* CONFIG_PM_SLEEP */
2854
2855 static const __maybe_unused struct of_device_id vmbus_of_match[] = {
2856 {
2857 .compatible = "microsoft,vmbus",
2858 },
2859 {
2860 /* sentinel */
2861 },
2862 };
2863 MODULE_DEVICE_TABLE(of, vmbus_of_match);
2864
2865 static const __maybe_unused struct acpi_device_id vmbus_acpi_device_ids[] = {
2866 {"VMBUS", 0},
2867 {"VMBus", 0},
2868 {"", 0},
2869 };
2870 MODULE_DEVICE_TABLE(acpi, vmbus_acpi_device_ids);
2871
2872 /*
2873 * Note: we must use the "no_irq" ops, otherwise hibernation can not work with
2874 * PCI device assignment, because "pci_dev_pm_ops" uses the "noirq" ops: in
2875 * the resume path, the pci "noirq" restore op runs before "non-noirq" op (see
2876 * resume_target_kernel() -> dpm_resume_start(), and hibernation_restore() ->
2877 * dpm_resume_end()). This means vmbus_bus_resume() and the pci-hyperv's
2878 * resume callback must also run via the "noirq" ops.
2879 *
2880 * Set suspend_noirq/resume_noirq to NULL for Suspend-to-Idle: see the comment
2881 * earlier in this file before vmbus_pm.
2882 */
2883
2884 static const struct dev_pm_ops vmbus_bus_pm = {
2885 .suspend_noirq = NULL,
2886 .resume_noirq = NULL,
2887 .freeze_noirq = vmbus_bus_suspend,
2888 .thaw_noirq = vmbus_bus_resume,
2889 .poweroff_noirq = vmbus_bus_suspend,
2890 .restore_noirq = vmbus_bus_resume
2891 };
2892
2893 static struct platform_driver vmbus_platform_driver = {
2894 .probe = vmbus_platform_driver_probe,
2895 .remove = vmbus_platform_driver_remove,
2896 .driver = {
2897 .name = "vmbus",
2898 .acpi_match_table = ACPI_PTR(vmbus_acpi_device_ids),
2899 .of_match_table = of_match_ptr(vmbus_of_match),
2900 .pm = &vmbus_bus_pm,
2901 .probe_type = PROBE_FORCE_SYNCHRONOUS,
2902 }
2903 };
2904
hv_kexec_handler(void)2905 static void hv_kexec_handler(void)
2906 {
2907 vmbus_initiate_unload(false);
2908 /* Make sure conn_state is set as hv_synic_cleanup checks for it */
2909 mb();
2910 cpuhp_remove_state(hyperv_cpuhp_online);
2911 };
2912
hv_crash_handler(struct pt_regs * regs)2913 static void hv_crash_handler(struct pt_regs *regs)
2914 {
2915 int cpu;
2916
2917 if (!skip_vmbus_unload)
2918 vmbus_initiate_unload(true);
2919 /*
2920 * In crash handler we can't schedule synic cleanup for all CPUs,
2921 * doing the cleanup for current CPU only. This should be sufficient
2922 * for kdump.
2923 */
2924 cpu = smp_processor_id();
2925 hv_stimer_cleanup(cpu);
2926 hv_hyp_synic_disable_regs(cpu);
2927 };
2928
hv_synic_suspend(void * data)2929 static int hv_synic_suspend(void *data)
2930 {
2931 /*
2932 * When we reach here, all the non-boot CPUs have been offlined.
2933 * If we're in a legacy configuration where stimer Direct Mode is
2934 * not enabled, the stimers on the non-boot CPUs have been unbound
2935 * in hv_synic_cleanup() -> hv_stimer_legacy_cleanup() ->
2936 * hv_stimer_cleanup() -> clockevents_unbind_device().
2937 *
2938 * hv_synic_suspend() only runs on CPU0 with interrupts disabled.
2939 * Here we do not call hv_stimer_legacy_cleanup() on CPU0 because:
2940 * 1) it's unnecessary as interrupts remain disabled between
2941 * syscore_suspend() and syscore_resume(): see create_image() and
2942 * resume_target_kernel()
2943 * 2) the stimer on CPU0 is automatically disabled later by
2944 * syscore_suspend() -> timekeeping_suspend() -> tick_suspend() -> ...
2945 * -> clockevents_shutdown() -> ... -> hv_ce_shutdown()
2946 * 3) a warning would be triggered if we call
2947 * clockevents_unbind_device(), which may sleep, in an
2948 * interrupts-disabled context.
2949 */
2950
2951 hv_hyp_synic_disable_regs(0);
2952
2953 return 0;
2954 }
2955
hv_synic_resume(void * data)2956 static void hv_synic_resume(void *data)
2957 {
2958 hv_hyp_synic_enable_regs(0);
2959
2960 /*
2961 * Note: we don't need to call hv_stimer_init(0), because the timer
2962 * on CPU0 is not unbound in hv_synic_suspend(), and the timer is
2963 * automatically re-enabled in timekeeping_resume().
2964 */
2965 }
2966
2967 /* The callbacks run only on CPU0, with irqs_disabled. */
2968 static const struct syscore_ops hv_synic_syscore_ops = {
2969 .suspend = hv_synic_suspend,
2970 .resume = hv_synic_resume,
2971 };
2972
2973 static struct syscore hv_synic_syscore = {
2974 .ops = &hv_synic_syscore_ops,
2975 };
2976
hv_acpi_init(void)2977 static int __init hv_acpi_init(void)
2978 {
2979 int ret;
2980
2981 if (!hv_is_hyperv_initialized())
2982 return -ENODEV;
2983
2984 if (hv_root_partition() && !hv_nested)
2985 return 0;
2986
2987 /*
2988 * Get ACPI resources first.
2989 */
2990 ret = platform_driver_register(&vmbus_platform_driver);
2991 if (ret)
2992 return ret;
2993
2994 if (!vmbus_root_device) {
2995 ret = -ENODEV;
2996 goto cleanup;
2997 }
2998
2999 /*
3000 * If we're on an architecture with a hardcoded hypervisor
3001 * vector (i.e. x86/x64), override the VMbus interrupt found
3002 * in the ACPI tables. Ensure vmbus_irq is not set since the
3003 * normal Linux IRQ mechanism is not used in this case.
3004 */
3005 #ifdef HYPERVISOR_CALLBACK_VECTOR
3006 vmbus_interrupt = HYPERVISOR_CALLBACK_VECTOR;
3007 vmbus_irq = -1;
3008 #endif
3009
3010 hv_debug_init();
3011
3012 ret = vmbus_bus_init();
3013 if (ret)
3014 goto cleanup;
3015
3016 hv_setup_kexec_handler(hv_kexec_handler);
3017 hv_setup_crash_handler(hv_crash_handler);
3018
3019 register_syscore(&hv_synic_syscore);
3020
3021 return 0;
3022
3023 cleanup:
3024 platform_driver_unregister(&vmbus_platform_driver);
3025 vmbus_root_device = NULL;
3026 return ret;
3027 }
3028
vmbus_exit(void)3029 static void __exit vmbus_exit(void)
3030 {
3031 int cpu;
3032
3033 unregister_syscore(&hv_synic_syscore);
3034
3035 hv_remove_kexec_handler();
3036 hv_remove_crash_handler();
3037 vmbus_connection.conn_state = DISCONNECTED;
3038 hv_stimer_global_cleanup();
3039 vmbus_disconnect();
3040 if (vmbus_irq == -1)
3041 hv_remove_vmbus_handler();
3042 else
3043 free_percpu_irq(vmbus_irq, &vmbus_evt);
3044 if (IS_ENABLED(CONFIG_PREEMPT_RT))
3045 smpboot_unregister_percpu_thread(&vmbus_irq_threads);
3046
3047 for_each_online_cpu(cpu) {
3048 struct hv_per_cpu_context *hv_cpu
3049 = per_cpu_ptr(hv_context.cpu_context, cpu);
3050
3051 tasklet_kill(&hv_cpu->msg_dpc);
3052 }
3053 hv_debug_rm_all_dir();
3054
3055 vmbus_free_channels();
3056 kfree(vmbus_connection.channels);
3057
3058 /*
3059 * The vmbus panic notifier is always registered, hence we should
3060 * also unconditionally unregister it here as well.
3061 */
3062 atomic_notifier_chain_unregister(&panic_notifier_list,
3063 &hyperv_panic_vmbus_unload_block);
3064
3065 bus_unregister(&hv_bus);
3066
3067 cpuhp_remove_state(hyperv_cpuhp_online);
3068 hv_synic_free();
3069 platform_driver_unregister(&vmbus_platform_driver);
3070 }
3071
3072
3073 MODULE_LICENSE("GPL");
3074 MODULE_DESCRIPTION("Microsoft Hyper-V VMBus Driver");
3075
3076 subsys_initcall(hv_acpi_init);
3077 module_exit(vmbus_exit);
3078