1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2009, Microsoft Corporation. 4 * 5 * Authors: 6 * Haiyang Zhang <haiyangz@microsoft.com> 7 * Hank Janssen <hjanssen@microsoft.com> 8 * K. Y. Srinivasan <kys@microsoft.com> 9 */ 10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 11 12 #include <linux/init.h> 13 #include <linux/module.h> 14 #include <linux/device.h> 15 #include <linux/platform_device.h> 16 #include <linux/interrupt.h> 17 #include <linux/sysctl.h> 18 #include <linux/slab.h> 19 #include <linux/acpi.h> 20 #include <linux/completion.h> 21 #include <linux/hyperv.h> 22 #include <linux/kernel_stat.h> 23 #include <linux/of_address.h> 24 #include <linux/clockchips.h> 25 #include <linux/cpu.h> 26 #include <linux/sched/isolation.h> 27 #include <linux/sched/task_stack.h> 28 #include <linux/smpboot.h> 29 30 #include <linux/delay.h> 31 #include <linux/panic_notifier.h> 32 #include <linux/ptrace.h> 33 #include <linux/sysfb.h> 34 #include <linux/efi.h> 35 #include <linux/kernel.h> 36 #include <linux/kexec.h> 37 #include <linux/syscore_ops.h> 38 #include <linux/dma-map-ops.h> 39 #include <linux/pci.h> 40 #include <linux/export.h> 41 #include <clocksource/hyperv_timer.h> 42 #include <asm/mshyperv.h> 43 #include "hyperv_vmbus.h" 44 45 struct vmbus_dynid { 46 struct list_head node; 47 struct hv_vmbus_device_id id; 48 }; 49 50 /* VMBus Root Device */ 51 static struct device *vmbus_root_device; 52 53 static int hyperv_cpuhp_online; 54 55 static DEFINE_PER_CPU(long, vmbus_evt); 56 57 /* Values parsed from ACPI DSDT */ 58 int vmbus_irq; 59 int vmbus_interrupt; 60 61 /* 62 * If the Confidential VMBus is used, the data on the "wire" is not 63 * visible to either the host or the hypervisor. 64 */ 65 static bool is_confidential; 66 67 bool vmbus_is_confidential(void) 68 { 69 return is_confidential; 70 } 71 EXPORT_SYMBOL_GPL(vmbus_is_confidential); 72 73 static bool skip_vmbus_unload; 74 75 /* 76 * Allow a VMBus framebuffer driver to specify that in the case of a panic, 77 * it will do the VMbus unload operation once it has flushed any dirty 78 * portions of the framebuffer to the Hyper-V host. 79 */ 80 void vmbus_set_skip_unload(bool skip) 81 { 82 skip_vmbus_unload = skip; 83 } 84 EXPORT_SYMBOL_GPL(vmbus_set_skip_unload); 85 86 /* 87 * The panic notifier below is responsible solely for unloading the 88 * vmbus connection, which is necessary in a panic event. 89 */ 90 static int hv_panic_vmbus_unload(struct notifier_block *nb, unsigned long val, 91 void *args) 92 { 93 if (!skip_vmbus_unload) 94 vmbus_initiate_unload(true); 95 96 return NOTIFY_DONE; 97 } 98 static struct notifier_block hyperv_panic_vmbus_unload_block = { 99 .notifier_call = hv_panic_vmbus_unload, 100 .priority = INT_MIN + 1, /* almost the latest one to execute */ 101 }; 102 103 static const char *fb_mmio_name = "fb_range"; 104 static struct resource *fb_mmio; 105 static struct resource *hyperv_mmio; 106 static DEFINE_MUTEX(hyperv_mmio_lock); 107 108 struct device *hv_get_vmbus_root_device(void) 109 { 110 return vmbus_root_device; 111 } 112 EXPORT_SYMBOL_GPL(hv_get_vmbus_root_device); 113 114 bool hv_vmbus_exists(void) 115 { 116 return vmbus_root_device != NULL; 117 } 118 EXPORT_SYMBOL_GPL(hv_vmbus_exists); 119 120 static u8 channel_monitor_group(const struct vmbus_channel *channel) 121 { 122 return (u8)channel->offermsg.monitorid / 32; 123 } 124 125 static u8 channel_monitor_offset(const struct vmbus_channel *channel) 126 { 127 return (u8)channel->offermsg.monitorid % 32; 128 } 129 130 static u32 channel_pending(const struct vmbus_channel *channel, 131 const struct hv_monitor_page *monitor_page) 132 { 133 u8 monitor_group = channel_monitor_group(channel); 134 135 return monitor_page->trigger_group[monitor_group].pending; 136 } 137 138 static u32 channel_latency(const struct vmbus_channel *channel, 139 const struct hv_monitor_page *monitor_page) 140 { 141 u8 monitor_group = channel_monitor_group(channel); 142 u8 monitor_offset = channel_monitor_offset(channel); 143 144 return monitor_page->latency[monitor_group][monitor_offset]; 145 } 146 147 static u32 channel_conn_id(struct vmbus_channel *channel, 148 struct hv_monitor_page *monitor_page) 149 { 150 u8 monitor_group = channel_monitor_group(channel); 151 u8 monitor_offset = channel_monitor_offset(channel); 152 153 return monitor_page->parameter[monitor_group][monitor_offset].connectionid.u.id; 154 } 155 156 static ssize_t id_show(struct device *dev, struct device_attribute *dev_attr, 157 char *buf) 158 { 159 struct hv_device *hv_dev = device_to_hv_device(dev); 160 161 if (!hv_dev->channel) 162 return -ENODEV; 163 return sysfs_emit(buf, "%d\n", hv_dev->channel->offermsg.child_relid); 164 } 165 static DEVICE_ATTR_RO(id); 166 167 static ssize_t state_show(struct device *dev, struct device_attribute *dev_attr, 168 char *buf) 169 { 170 struct hv_device *hv_dev = device_to_hv_device(dev); 171 172 if (!hv_dev->channel) 173 return -ENODEV; 174 return sysfs_emit(buf, "%d\n", hv_dev->channel->state); 175 } 176 static DEVICE_ATTR_RO(state); 177 178 static ssize_t monitor_id_show(struct device *dev, 179 struct device_attribute *dev_attr, char *buf) 180 { 181 struct hv_device *hv_dev = device_to_hv_device(dev); 182 183 if (!hv_dev->channel) 184 return -ENODEV; 185 return sysfs_emit(buf, "%d\n", hv_dev->channel->offermsg.monitorid); 186 } 187 static DEVICE_ATTR_RO(monitor_id); 188 189 static ssize_t class_id_show(struct device *dev, 190 struct device_attribute *dev_attr, char *buf) 191 { 192 struct hv_device *hv_dev = device_to_hv_device(dev); 193 194 if (!hv_dev->channel) 195 return -ENODEV; 196 return sysfs_emit(buf, "{%pUl}\n", 197 &hv_dev->channel->offermsg.offer.if_type); 198 } 199 static DEVICE_ATTR_RO(class_id); 200 201 static ssize_t device_id_show(struct device *dev, 202 struct device_attribute *dev_attr, char *buf) 203 { 204 struct hv_device *hv_dev = device_to_hv_device(dev); 205 206 if (!hv_dev->channel) 207 return -ENODEV; 208 return sysfs_emit(buf, "{%pUl}\n", 209 &hv_dev->channel->offermsg.offer.if_instance); 210 } 211 static DEVICE_ATTR_RO(device_id); 212 213 static ssize_t modalias_show(struct device *dev, 214 struct device_attribute *dev_attr, char *buf) 215 { 216 struct hv_device *hv_dev = device_to_hv_device(dev); 217 218 return sysfs_emit(buf, "vmbus:%*phN\n", UUID_SIZE, &hv_dev->dev_type); 219 } 220 static DEVICE_ATTR_RO(modalias); 221 222 #ifdef CONFIG_NUMA 223 static ssize_t numa_node_show(struct device *dev, 224 struct device_attribute *attr, char *buf) 225 { 226 struct hv_device *hv_dev = device_to_hv_device(dev); 227 228 if (!hv_dev->channel) 229 return -ENODEV; 230 231 return sysfs_emit(buf, "%d\n", cpu_to_node(hv_dev->channel->target_cpu)); 232 } 233 static DEVICE_ATTR_RO(numa_node); 234 #endif 235 236 static ssize_t server_monitor_pending_show(struct device *dev, 237 struct device_attribute *dev_attr, 238 char *buf) 239 { 240 struct hv_device *hv_dev = device_to_hv_device(dev); 241 242 if (!hv_dev->channel) 243 return -ENODEV; 244 return sysfs_emit(buf, "%d\n", channel_pending(hv_dev->channel, 245 vmbus_connection.monitor_pages[0])); 246 } 247 static DEVICE_ATTR_RO(server_monitor_pending); 248 249 static ssize_t client_monitor_pending_show(struct device *dev, 250 struct device_attribute *dev_attr, 251 char *buf) 252 { 253 struct hv_device *hv_dev = device_to_hv_device(dev); 254 255 if (!hv_dev->channel) 256 return -ENODEV; 257 return sysfs_emit(buf, "%d\n", channel_pending(hv_dev->channel, 258 vmbus_connection.monitor_pages[1])); 259 } 260 static DEVICE_ATTR_RO(client_monitor_pending); 261 262 static ssize_t server_monitor_latency_show(struct device *dev, 263 struct device_attribute *dev_attr, 264 char *buf) 265 { 266 struct hv_device *hv_dev = device_to_hv_device(dev); 267 268 if (!hv_dev->channel) 269 return -ENODEV; 270 return sysfs_emit(buf, "%d\n", channel_latency(hv_dev->channel, 271 vmbus_connection.monitor_pages[0])); 272 } 273 static DEVICE_ATTR_RO(server_monitor_latency); 274 275 static ssize_t client_monitor_latency_show(struct device *dev, 276 struct device_attribute *dev_attr, 277 char *buf) 278 { 279 struct hv_device *hv_dev = device_to_hv_device(dev); 280 281 if (!hv_dev->channel) 282 return -ENODEV; 283 return sysfs_emit(buf, "%d\n", channel_latency(hv_dev->channel, 284 vmbus_connection.monitor_pages[1])); 285 } 286 static DEVICE_ATTR_RO(client_monitor_latency); 287 288 static ssize_t server_monitor_conn_id_show(struct device *dev, 289 struct device_attribute *dev_attr, 290 char *buf) 291 { 292 struct hv_device *hv_dev = device_to_hv_device(dev); 293 294 if (!hv_dev->channel) 295 return -ENODEV; 296 return sysfs_emit(buf, "%d\n", channel_conn_id(hv_dev->channel, 297 vmbus_connection.monitor_pages[0])); 298 } 299 static DEVICE_ATTR_RO(server_monitor_conn_id); 300 301 static ssize_t client_monitor_conn_id_show(struct device *dev, 302 struct device_attribute *dev_attr, 303 char *buf) 304 { 305 struct hv_device *hv_dev = device_to_hv_device(dev); 306 307 if (!hv_dev->channel) 308 return -ENODEV; 309 return sysfs_emit(buf, "%d\n", channel_conn_id(hv_dev->channel, 310 vmbus_connection.monitor_pages[1])); 311 } 312 static DEVICE_ATTR_RO(client_monitor_conn_id); 313 314 static ssize_t out_intr_mask_show(struct device *dev, 315 struct device_attribute *dev_attr, char *buf) 316 { 317 struct hv_device *hv_dev = device_to_hv_device(dev); 318 struct hv_ring_buffer_debug_info outbound; 319 int ret; 320 321 if (!hv_dev->channel) 322 return -ENODEV; 323 324 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, 325 &outbound); 326 if (ret < 0) 327 return ret; 328 329 return sysfs_emit(buf, "%d\n", outbound.current_interrupt_mask); 330 } 331 static DEVICE_ATTR_RO(out_intr_mask); 332 333 static ssize_t out_read_index_show(struct device *dev, 334 struct device_attribute *dev_attr, char *buf) 335 { 336 struct hv_device *hv_dev = device_to_hv_device(dev); 337 struct hv_ring_buffer_debug_info outbound; 338 int ret; 339 340 if (!hv_dev->channel) 341 return -ENODEV; 342 343 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, 344 &outbound); 345 if (ret < 0) 346 return ret; 347 return sysfs_emit(buf, "%u\n", outbound.current_read_index); 348 } 349 static DEVICE_ATTR_RO(out_read_index); 350 351 static ssize_t out_write_index_show(struct device *dev, 352 struct device_attribute *dev_attr, 353 char *buf) 354 { 355 struct hv_device *hv_dev = device_to_hv_device(dev); 356 struct hv_ring_buffer_debug_info outbound; 357 int ret; 358 359 if (!hv_dev->channel) 360 return -ENODEV; 361 362 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, 363 &outbound); 364 if (ret < 0) 365 return ret; 366 return sysfs_emit(buf, "%u\n", outbound.current_write_index); 367 } 368 static DEVICE_ATTR_RO(out_write_index); 369 370 static ssize_t out_read_bytes_avail_show(struct device *dev, 371 struct device_attribute *dev_attr, 372 char *buf) 373 { 374 struct hv_device *hv_dev = device_to_hv_device(dev); 375 struct hv_ring_buffer_debug_info outbound; 376 int ret; 377 378 if (!hv_dev->channel) 379 return -ENODEV; 380 381 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, 382 &outbound); 383 if (ret < 0) 384 return ret; 385 return sysfs_emit(buf, "%d\n", outbound.bytes_avail_toread); 386 } 387 static DEVICE_ATTR_RO(out_read_bytes_avail); 388 389 static ssize_t out_write_bytes_avail_show(struct device *dev, 390 struct device_attribute *dev_attr, 391 char *buf) 392 { 393 struct hv_device *hv_dev = device_to_hv_device(dev); 394 struct hv_ring_buffer_debug_info outbound; 395 int ret; 396 397 if (!hv_dev->channel) 398 return -ENODEV; 399 400 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, 401 &outbound); 402 if (ret < 0) 403 return ret; 404 return sysfs_emit(buf, "%d\n", outbound.bytes_avail_towrite); 405 } 406 static DEVICE_ATTR_RO(out_write_bytes_avail); 407 408 static ssize_t in_intr_mask_show(struct device *dev, 409 struct device_attribute *dev_attr, char *buf) 410 { 411 struct hv_device *hv_dev = device_to_hv_device(dev); 412 struct hv_ring_buffer_debug_info inbound; 413 int ret; 414 415 if (!hv_dev->channel) 416 return -ENODEV; 417 418 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); 419 if (ret < 0) 420 return ret; 421 422 return sysfs_emit(buf, "%d\n", inbound.current_interrupt_mask); 423 } 424 static DEVICE_ATTR_RO(in_intr_mask); 425 426 static ssize_t in_read_index_show(struct device *dev, 427 struct device_attribute *dev_attr, char *buf) 428 { 429 struct hv_device *hv_dev = device_to_hv_device(dev); 430 struct hv_ring_buffer_debug_info inbound; 431 int ret; 432 433 if (!hv_dev->channel) 434 return -ENODEV; 435 436 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); 437 if (ret < 0) 438 return ret; 439 440 return sysfs_emit(buf, "%d\n", inbound.current_read_index); 441 } 442 static DEVICE_ATTR_RO(in_read_index); 443 444 static ssize_t in_write_index_show(struct device *dev, 445 struct device_attribute *dev_attr, char *buf) 446 { 447 struct hv_device *hv_dev = device_to_hv_device(dev); 448 struct hv_ring_buffer_debug_info inbound; 449 int ret; 450 451 if (!hv_dev->channel) 452 return -ENODEV; 453 454 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); 455 if (ret < 0) 456 return ret; 457 458 return sysfs_emit(buf, "%d\n", inbound.current_write_index); 459 } 460 static DEVICE_ATTR_RO(in_write_index); 461 462 static ssize_t in_read_bytes_avail_show(struct device *dev, 463 struct device_attribute *dev_attr, 464 char *buf) 465 { 466 struct hv_device *hv_dev = device_to_hv_device(dev); 467 struct hv_ring_buffer_debug_info inbound; 468 int ret; 469 470 if (!hv_dev->channel) 471 return -ENODEV; 472 473 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); 474 if (ret < 0) 475 return ret; 476 477 return sysfs_emit(buf, "%d\n", inbound.bytes_avail_toread); 478 } 479 static DEVICE_ATTR_RO(in_read_bytes_avail); 480 481 static ssize_t in_write_bytes_avail_show(struct device *dev, 482 struct device_attribute *dev_attr, 483 char *buf) 484 { 485 struct hv_device *hv_dev = device_to_hv_device(dev); 486 struct hv_ring_buffer_debug_info inbound; 487 int ret; 488 489 if (!hv_dev->channel) 490 return -ENODEV; 491 492 ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); 493 if (ret < 0) 494 return ret; 495 496 return sysfs_emit(buf, "%d\n", inbound.bytes_avail_towrite); 497 } 498 static DEVICE_ATTR_RO(in_write_bytes_avail); 499 500 static ssize_t channel_vp_mapping_show(struct device *dev, 501 struct device_attribute *dev_attr, 502 char *buf) 503 { 504 struct hv_device *hv_dev = device_to_hv_device(dev); 505 struct vmbus_channel *channel = hv_dev->channel, *cur_sc; 506 int n_written; 507 struct list_head *cur; 508 509 if (!channel) 510 return -ENODEV; 511 512 mutex_lock(&vmbus_connection.channel_mutex); 513 514 n_written = sysfs_emit(buf, "%u:%u\n", 515 channel->offermsg.child_relid, 516 channel->target_cpu); 517 518 list_for_each(cur, &channel->sc_list) { 519 520 cur_sc = list_entry(cur, struct vmbus_channel, sc_list); 521 n_written += sysfs_emit_at(buf, n_written, "%u:%u\n", 522 cur_sc->offermsg.child_relid, 523 cur_sc->target_cpu); 524 } 525 526 mutex_unlock(&vmbus_connection.channel_mutex); 527 528 return n_written; 529 } 530 static DEVICE_ATTR_RO(channel_vp_mapping); 531 532 static ssize_t vendor_show(struct device *dev, 533 struct device_attribute *dev_attr, 534 char *buf) 535 { 536 struct hv_device *hv_dev = device_to_hv_device(dev); 537 538 return sysfs_emit(buf, "0x%x\n", hv_dev->vendor_id); 539 } 540 static DEVICE_ATTR_RO(vendor); 541 542 static ssize_t device_show(struct device *dev, 543 struct device_attribute *dev_attr, 544 char *buf) 545 { 546 struct hv_device *hv_dev = device_to_hv_device(dev); 547 548 return sysfs_emit(buf, "0x%x\n", hv_dev->device_id); 549 } 550 static DEVICE_ATTR_RO(device); 551 552 /* Set up per device attributes in /sys/bus/vmbus/devices/<bus device> */ 553 static struct attribute *vmbus_dev_attrs[] = { 554 &dev_attr_id.attr, 555 &dev_attr_state.attr, 556 &dev_attr_monitor_id.attr, 557 &dev_attr_class_id.attr, 558 &dev_attr_device_id.attr, 559 &dev_attr_modalias.attr, 560 #ifdef CONFIG_NUMA 561 &dev_attr_numa_node.attr, 562 #endif 563 &dev_attr_server_monitor_pending.attr, 564 &dev_attr_client_monitor_pending.attr, 565 &dev_attr_server_monitor_latency.attr, 566 &dev_attr_client_monitor_latency.attr, 567 &dev_attr_server_monitor_conn_id.attr, 568 &dev_attr_client_monitor_conn_id.attr, 569 &dev_attr_out_intr_mask.attr, 570 &dev_attr_out_read_index.attr, 571 &dev_attr_out_write_index.attr, 572 &dev_attr_out_read_bytes_avail.attr, 573 &dev_attr_out_write_bytes_avail.attr, 574 &dev_attr_in_intr_mask.attr, 575 &dev_attr_in_read_index.attr, 576 &dev_attr_in_write_index.attr, 577 &dev_attr_in_read_bytes_avail.attr, 578 &dev_attr_in_write_bytes_avail.attr, 579 &dev_attr_channel_vp_mapping.attr, 580 &dev_attr_vendor.attr, 581 &dev_attr_device.attr, 582 NULL, 583 }; 584 585 /* 586 * Device-level attribute_group callback function. Returns the permission for 587 * each attribute, and returns 0 if an attribute is not visible. 588 */ 589 static umode_t vmbus_dev_attr_is_visible(struct kobject *kobj, 590 struct attribute *attr, int idx) 591 { 592 struct device *dev = kobj_to_dev(kobj); 593 const struct hv_device *hv_dev = device_to_hv_device(dev); 594 595 /* Hide the monitor attributes if the monitor mechanism is not used. */ 596 if (!hv_dev->channel->offermsg.monitor_allocated && 597 (attr == &dev_attr_monitor_id.attr || 598 attr == &dev_attr_server_monitor_pending.attr || 599 attr == &dev_attr_client_monitor_pending.attr || 600 attr == &dev_attr_server_monitor_latency.attr || 601 attr == &dev_attr_client_monitor_latency.attr || 602 attr == &dev_attr_server_monitor_conn_id.attr || 603 attr == &dev_attr_client_monitor_conn_id.attr)) 604 return 0; 605 606 return attr->mode; 607 } 608 609 static const struct attribute_group vmbus_dev_group = { 610 .attrs = vmbus_dev_attrs, 611 .is_visible = vmbus_dev_attr_is_visible 612 }; 613 __ATTRIBUTE_GROUPS(vmbus_dev); 614 615 /* Set up the attribute for /sys/bus/vmbus/hibernation */ 616 static ssize_t hibernation_show(const struct bus_type *bus, char *buf) 617 { 618 return sprintf(buf, "%d\n", !!hv_is_hibernation_supported()); 619 } 620 621 static BUS_ATTR_RO(hibernation); 622 623 static struct attribute *vmbus_bus_attrs[] = { 624 &bus_attr_hibernation.attr, 625 NULL, 626 }; 627 static const struct attribute_group vmbus_bus_group = { 628 .attrs = vmbus_bus_attrs, 629 }; 630 __ATTRIBUTE_GROUPS(vmbus_bus); 631 632 /* 633 * vmbus_uevent - add uevent for our device 634 * 635 * This routine is invoked when a device is added or removed on the vmbus to 636 * generate a uevent to udev in the userspace. The udev will then look at its 637 * rule and the uevent generated here to load the appropriate driver 638 * 639 * The alias string will be of the form vmbus:guid where guid is the string 640 * representation of the device guid (each byte of the guid will be 641 * represented with two hex characters. 642 */ 643 static int vmbus_uevent(const struct device *device, struct kobj_uevent_env *env) 644 { 645 const struct hv_device *dev = device_to_hv_device(device); 646 const char *format = "MODALIAS=vmbus:%*phN"; 647 648 return add_uevent_var(env, format, UUID_SIZE, &dev->dev_type); 649 } 650 651 static const struct hv_vmbus_device_id * 652 hv_vmbus_dev_match(const struct hv_vmbus_device_id *id, const guid_t *guid) 653 { 654 if (id == NULL) 655 return NULL; /* empty device table */ 656 657 for (; !guid_is_null(&id->guid); id++) 658 if (guid_equal(&id->guid, guid)) 659 return id; 660 661 return NULL; 662 } 663 664 static const struct hv_vmbus_device_id * 665 hv_vmbus_dynid_match(struct hv_driver *drv, const guid_t *guid) 666 { 667 const struct hv_vmbus_device_id *id = NULL; 668 struct vmbus_dynid *dynid; 669 670 spin_lock(&drv->dynids.lock); 671 list_for_each_entry(dynid, &drv->dynids.list, node) { 672 if (guid_equal(&dynid->id.guid, guid)) { 673 id = &dynid->id; 674 break; 675 } 676 } 677 spin_unlock(&drv->dynids.lock); 678 679 return id; 680 } 681 682 static const struct hv_vmbus_device_id vmbus_device_null; 683 684 /* 685 * Return a matching hv_vmbus_device_id pointer. 686 * If there is no match, return NULL. 687 */ 688 static const struct hv_vmbus_device_id *hv_vmbus_get_id(const struct hv_driver *drv, 689 struct hv_device *dev) 690 { 691 const guid_t *guid = &dev->dev_type; 692 const struct hv_vmbus_device_id *id; 693 int ret; 694 695 /* If a driver override is set, only bind to the matching driver */ 696 ret = device_match_driver_override(&dev->device, &drv->driver); 697 if (ret == 0) 698 return NULL; 699 700 /* Look at the dynamic ids first, before the static ones */ 701 id = hv_vmbus_dynid_match((struct hv_driver *)drv, guid); 702 if (!id) 703 id = hv_vmbus_dev_match(drv->id_table, guid); 704 705 /* 706 * If there's a matching driver override, this function should succeed, 707 * thus return a dummy device ID if no matching ID is found. 708 */ 709 if (!id && ret > 0) 710 id = &vmbus_device_null; 711 712 return id; 713 } 714 715 /* vmbus_add_dynid - add a new device ID to this driver and re-probe devices 716 * 717 * This function can race with vmbus_device_register(). This function is 718 * typically running on a user thread in response to writing to the "new_id" 719 * sysfs entry for a driver. vmbus_device_register() is running on a 720 * workqueue thread in response to the Hyper-V host offering a device to the 721 * guest. This function calls driver_attach(), which looks for an existing 722 * device matching the new id, and attaches the driver to which the new id 723 * has been assigned. vmbus_device_register() calls device_register(), which 724 * looks for a driver that matches the device being registered. If both 725 * operations are running simultaneously, the device driver probe function runs 726 * on whichever thread establishes the linkage between the driver and device. 727 * 728 * In most cases, it doesn't matter which thread runs the driver probe 729 * function. But if vmbus_device_register() does not find a matching driver, 730 * it proceeds to create the "channels" subdirectory and numbered per-channel 731 * subdirectory in sysfs. While that multi-step creation is in progress, this 732 * function could run the driver probe function. If the probe function checks 733 * for, or operates on, entries in the "channels" subdirectory, including by 734 * calling hv_create_ring_sysfs(), the operation may or may not succeed 735 * depending on the race. The race can't create a kernel failure in VMBus 736 * or device subsystem code, but probe functions in VMBus drivers doing such 737 * operations must be prepared for the failure case. 738 */ 739 static int vmbus_add_dynid(struct hv_driver *drv, guid_t *guid) 740 { 741 struct vmbus_dynid *dynid; 742 743 dynid = kzalloc_obj(*dynid); 744 if (!dynid) 745 return -ENOMEM; 746 747 dynid->id.guid = *guid; 748 749 spin_lock(&drv->dynids.lock); 750 list_add_tail(&dynid->node, &drv->dynids.list); 751 spin_unlock(&drv->dynids.lock); 752 753 return driver_attach(&drv->driver); 754 } 755 756 static void vmbus_free_dynids(struct hv_driver *drv) 757 { 758 struct vmbus_dynid *dynid, *n; 759 760 spin_lock(&drv->dynids.lock); 761 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) { 762 list_del(&dynid->node); 763 kfree(dynid); 764 } 765 spin_unlock(&drv->dynids.lock); 766 } 767 768 /* 769 * store_new_id - sysfs frontend to vmbus_add_dynid() 770 * 771 * Allow GUIDs to be added to an existing driver via sysfs. 772 */ 773 static ssize_t new_id_store(struct device_driver *driver, const char *buf, 774 size_t count) 775 { 776 struct hv_driver *drv = drv_to_hv_drv(driver); 777 guid_t guid; 778 ssize_t retval; 779 780 retval = guid_parse(buf, &guid); 781 if (retval) 782 return retval; 783 784 if (hv_vmbus_dynid_match(drv, &guid)) 785 return -EEXIST; 786 787 retval = vmbus_add_dynid(drv, &guid); 788 if (retval) 789 return retval; 790 return count; 791 } 792 static DRIVER_ATTR_WO(new_id); 793 794 /* 795 * store_remove_id - remove a PCI device ID from this driver 796 * 797 * Removes a dynamic pci device ID to this driver. 798 */ 799 static ssize_t remove_id_store(struct device_driver *driver, const char *buf, 800 size_t count) 801 { 802 struct hv_driver *drv = drv_to_hv_drv(driver); 803 struct vmbus_dynid *dynid, *n; 804 guid_t guid; 805 ssize_t retval; 806 807 retval = guid_parse(buf, &guid); 808 if (retval) 809 return retval; 810 811 retval = -ENODEV; 812 spin_lock(&drv->dynids.lock); 813 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) { 814 struct hv_vmbus_device_id *id = &dynid->id; 815 816 if (guid_equal(&id->guid, &guid)) { 817 list_del(&dynid->node); 818 kfree(dynid); 819 retval = count; 820 break; 821 } 822 } 823 spin_unlock(&drv->dynids.lock); 824 825 return retval; 826 } 827 static DRIVER_ATTR_WO(remove_id); 828 829 static struct attribute *vmbus_drv_attrs[] = { 830 &driver_attr_new_id.attr, 831 &driver_attr_remove_id.attr, 832 NULL, 833 }; 834 ATTRIBUTE_GROUPS(vmbus_drv); 835 836 837 /* 838 * vmbus_match - Attempt to match the specified device to the specified driver 839 */ 840 static int vmbus_match(struct device *device, const struct device_driver *driver) 841 { 842 const struct hv_driver *drv = drv_to_hv_drv(driver); 843 struct hv_device *hv_dev = device_to_hv_device(device); 844 845 /* The hv_sock driver handles all hv_sock offers. */ 846 if (is_hvsock_channel(hv_dev->channel)) 847 return drv->hvsock; 848 849 if (hv_vmbus_get_id(drv, hv_dev)) 850 return 1; 851 852 return 0; 853 } 854 855 /* 856 * vmbus_probe - Add the new vmbus's child device 857 */ 858 static int vmbus_probe(struct device *child_device) 859 { 860 int ret = 0; 861 struct hv_driver *drv = 862 drv_to_hv_drv(child_device->driver); 863 struct hv_device *dev = device_to_hv_device(child_device); 864 const struct hv_vmbus_device_id *dev_id; 865 866 dev_id = hv_vmbus_get_id(drv, dev); 867 if (drv->probe) { 868 ret = drv->probe(dev, dev_id); 869 if (ret != 0) 870 pr_err("probe failed for device %s (%d)\n", 871 dev_name(child_device), ret); 872 873 } else { 874 pr_err("probe not set for driver %s\n", 875 dev_name(child_device)); 876 ret = -ENODEV; 877 } 878 return ret; 879 } 880 881 /* 882 * vmbus_dma_configure -- Configure DMA coherence for VMbus device 883 */ 884 static int vmbus_dma_configure(struct device *child_device) 885 { 886 /* 887 * On ARM64, propagate the DMA coherence setting from the top level 888 * VMbus ACPI device to the child VMbus device being added here. 889 * On x86/x64 coherence is assumed and these calls have no effect. 890 */ 891 hv_setup_dma_ops(child_device, 892 device_get_dma_attr(vmbus_root_device) == DEV_DMA_COHERENT); 893 return 0; 894 } 895 896 /* 897 * vmbus_remove - Remove a vmbus device 898 */ 899 static void vmbus_remove(struct device *child_device) 900 { 901 struct hv_driver *drv; 902 struct hv_device *dev = device_to_hv_device(child_device); 903 904 if (child_device->driver) { 905 drv = drv_to_hv_drv(child_device->driver); 906 if (drv->remove) 907 drv->remove(dev); 908 } 909 } 910 911 /* 912 * vmbus_shutdown - Shutdown a vmbus device 913 */ 914 static void vmbus_shutdown(struct device *child_device) 915 { 916 struct hv_driver *drv; 917 struct hv_device *dev = device_to_hv_device(child_device); 918 919 920 /* The device may not be attached yet */ 921 if (!child_device->driver) 922 return; 923 924 drv = drv_to_hv_drv(child_device->driver); 925 926 if (drv->shutdown) 927 drv->shutdown(dev); 928 } 929 930 #ifdef CONFIG_PM_SLEEP 931 /* 932 * vmbus_suspend - Suspend a vmbus device 933 */ 934 static int vmbus_suspend(struct device *child_device) 935 { 936 struct hv_driver *drv; 937 struct hv_device *dev = device_to_hv_device(child_device); 938 939 /* The device may not be attached yet */ 940 if (!child_device->driver) 941 return 0; 942 943 drv = drv_to_hv_drv(child_device->driver); 944 if (!drv->suspend) 945 return -EOPNOTSUPP; 946 947 return drv->suspend(dev); 948 } 949 950 /* 951 * vmbus_resume - Resume a vmbus device 952 */ 953 static int vmbus_resume(struct device *child_device) 954 { 955 struct hv_driver *drv; 956 struct hv_device *dev = device_to_hv_device(child_device); 957 958 /* The device may not be attached yet */ 959 if (!child_device->driver) 960 return 0; 961 962 drv = drv_to_hv_drv(child_device->driver); 963 if (!drv->resume) 964 return -EOPNOTSUPP; 965 966 return drv->resume(dev); 967 } 968 #else 969 #define vmbus_suspend NULL 970 #define vmbus_resume NULL 971 #endif /* CONFIG_PM_SLEEP */ 972 973 /* 974 * vmbus_device_release - Final callback release of the vmbus child device 975 */ 976 static void vmbus_device_release(struct device *device) 977 { 978 struct hv_device *hv_dev = device_to_hv_device(device); 979 struct vmbus_channel *channel = hv_dev->channel; 980 981 hv_debug_rm_dev_dir(hv_dev); 982 983 mutex_lock(&vmbus_connection.channel_mutex); 984 hv_process_channel_removal(channel); 985 mutex_unlock(&vmbus_connection.channel_mutex); 986 kfree(hv_dev); 987 } 988 989 /* 990 * Note: we must use the "noirq" ops: see the comment before vmbus_bus_pm. 991 * 992 * suspend_noirq/resume_noirq are set to NULL to support Suspend-to-Idle: we 993 * shouldn't suspend the vmbus devices upon Suspend-to-Idle, otherwise there 994 * is no way to wake up a Generation-2 VM. 995 * 996 * The other 4 ops are for hibernation. 997 */ 998 999 static const struct dev_pm_ops vmbus_pm = { 1000 .suspend_noirq = NULL, 1001 .resume_noirq = NULL, 1002 .freeze_noirq = vmbus_suspend, 1003 .thaw_noirq = vmbus_resume, 1004 .poweroff_noirq = vmbus_suspend, 1005 .restore_noirq = vmbus_resume, 1006 }; 1007 1008 /* The one and only one */ 1009 static const struct bus_type hv_bus = { 1010 .name = "vmbus", 1011 .driver_override = true, 1012 .match = vmbus_match, 1013 .shutdown = vmbus_shutdown, 1014 .remove = vmbus_remove, 1015 .probe = vmbus_probe, 1016 .uevent = vmbus_uevent, 1017 .dma_configure = vmbus_dma_configure, 1018 .dev_groups = vmbus_dev_groups, 1019 .drv_groups = vmbus_drv_groups, 1020 .bus_groups = vmbus_bus_groups, 1021 .pm = &vmbus_pm, 1022 }; 1023 1024 struct onmessage_work_context { 1025 struct work_struct work; 1026 struct { 1027 struct hv_message_header header; 1028 u8 payload[]; 1029 } msg; 1030 }; 1031 1032 static void vmbus_onmessage_work(struct work_struct *work) 1033 { 1034 struct onmessage_work_context *ctx; 1035 1036 /* Do not process messages if we're in DISCONNECTED state */ 1037 if (vmbus_connection.conn_state == DISCONNECTED) 1038 return; 1039 1040 ctx = container_of(work, struct onmessage_work_context, 1041 work); 1042 vmbus_onmessage((struct vmbus_channel_message_header *) 1043 &ctx->msg.payload); 1044 kfree(ctx); 1045 } 1046 1047 static void __vmbus_on_msg_dpc(void *message_page_addr) 1048 { 1049 struct hv_message msg_copy, *msg; 1050 struct vmbus_channel_message_header *hdr; 1051 enum vmbus_channel_message_type msgtype; 1052 const struct vmbus_channel_message_table_entry *entry; 1053 struct onmessage_work_context *ctx; 1054 __u8 payload_size; 1055 u32 message_type; 1056 1057 if (!message_page_addr) 1058 return; 1059 msg = (struct hv_message *)message_page_addr + VMBUS_MESSAGE_SINT; 1060 1061 /* 1062 * 'enum vmbus_channel_message_type' is supposed to always be 'u32' as 1063 * it is being used in 'struct vmbus_channel_message_header' definition 1064 * which is supposed to match hypervisor ABI. 1065 */ 1066 BUILD_BUG_ON(sizeof(enum vmbus_channel_message_type) != sizeof(u32)); 1067 1068 /* 1069 * Since the message is in memory shared with the host, an erroneous or 1070 * malicious Hyper-V could modify the message while vmbus_on_msg_dpc() 1071 * or individual message handlers are executing; to prevent this, copy 1072 * the message into private memory. 1073 */ 1074 memcpy(&msg_copy, msg, sizeof(struct hv_message)); 1075 1076 message_type = msg_copy.header.message_type; 1077 if (message_type == HVMSG_NONE) 1078 /* no msg */ 1079 return; 1080 1081 hdr = (struct vmbus_channel_message_header *)msg_copy.u.payload; 1082 msgtype = hdr->msgtype; 1083 1084 trace_vmbus_on_msg_dpc(hdr); 1085 1086 if (msgtype >= CHANNELMSG_COUNT) { 1087 WARN_ONCE(1, "unknown msgtype=%d\n", msgtype); 1088 goto msg_handled; 1089 } 1090 1091 payload_size = msg_copy.header.payload_size; 1092 if (payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) { 1093 WARN_ONCE(1, "payload size is too large (%d)\n", payload_size); 1094 goto msg_handled; 1095 } 1096 1097 entry = &channel_message_table[msgtype]; 1098 1099 if (!entry->message_handler) 1100 goto msg_handled; 1101 1102 if (payload_size < entry->min_payload_len) { 1103 WARN_ONCE(1, "message too short: msgtype=%d len=%d\n", msgtype, payload_size); 1104 goto msg_handled; 1105 } 1106 1107 if (entry->handler_type == VMHT_BLOCKING) { 1108 ctx = kmalloc_flex(*ctx, msg.payload, payload_size, GFP_ATOMIC); 1109 if (ctx == NULL) 1110 return; 1111 1112 INIT_WORK(&ctx->work, vmbus_onmessage_work); 1113 ctx->msg.header = msg_copy.header; 1114 memcpy(&ctx->msg.payload, msg_copy.u.payload, payload_size); 1115 1116 /* 1117 * The host can generate a rescind message while we 1118 * may still be handling the original offer. We deal with 1119 * this condition by relying on the synchronization provided 1120 * by offer_in_progress and by channel_mutex. See also the 1121 * inline comments in vmbus_onoffer_rescind(). 1122 */ 1123 switch (msgtype) { 1124 case CHANNELMSG_RESCIND_CHANNELOFFER: 1125 /* 1126 * If we are handling the rescind message; 1127 * schedule the work on the global work queue. 1128 * 1129 * The OFFER message and the RESCIND message should 1130 * not be handled by the same serialized work queue, 1131 * because the OFFER handler may call vmbus_open(), 1132 * which tries to open the channel by sending an 1133 * OPEN_CHANNEL message to the host and waits for 1134 * the host's response; however, if the host has 1135 * rescinded the channel before it receives the 1136 * OPEN_CHANNEL message, the host just silently 1137 * ignores the OPEN_CHANNEL message; as a result, 1138 * the guest's OFFER handler hangs for ever, if we 1139 * handle the RESCIND message in the same serialized 1140 * work queue: the RESCIND handler can not start to 1141 * run before the OFFER handler finishes. 1142 */ 1143 if (vmbus_connection.ignore_any_offer_msg) 1144 break; 1145 queue_work(vmbus_connection.rescind_work_queue, &ctx->work); 1146 break; 1147 1148 case CHANNELMSG_OFFERCHANNEL: 1149 /* 1150 * The host sends the offer message of a given channel 1151 * before sending the rescind message of the same 1152 * channel. These messages are sent to the guest's 1153 * connect CPU; the guest then starts processing them 1154 * in the tasklet handler on this CPU: 1155 * 1156 * VMBUS_CONNECT_CPU 1157 * 1158 * [vmbus_on_msg_dpc()] 1159 * atomic_inc() // CHANNELMSG_OFFERCHANNEL 1160 * queue_work() 1161 * ... 1162 * [vmbus_on_msg_dpc()] 1163 * schedule_work() // CHANNELMSG_RESCIND_CHANNELOFFER 1164 * 1165 * We rely on the memory-ordering properties of the 1166 * queue_work() and schedule_work() primitives, which 1167 * guarantee that the atomic increment will be visible 1168 * to the CPUs which will execute the offer & rescind 1169 * works by the time these works will start execution. 1170 */ 1171 if (vmbus_connection.ignore_any_offer_msg) 1172 break; 1173 atomic_inc(&vmbus_connection.offer_in_progress); 1174 fallthrough; 1175 1176 default: 1177 queue_work(vmbus_connection.work_queue, &ctx->work); 1178 } 1179 } else 1180 entry->message_handler(hdr); 1181 1182 msg_handled: 1183 vmbus_signal_eom(msg, message_type); 1184 } 1185 1186 void vmbus_on_msg_dpc(unsigned long data) 1187 { 1188 struct hv_per_cpu_context *hv_cpu = (void *)data; 1189 1190 __vmbus_on_msg_dpc(hv_cpu->hyp_synic_message_page); 1191 __vmbus_on_msg_dpc(hv_cpu->para_synic_message_page); 1192 } 1193 1194 #ifdef CONFIG_PM_SLEEP 1195 /* 1196 * Fake RESCIND_CHANNEL messages to clean up hv_sock channels by force for 1197 * hibernation, because hv_sock connections can not persist across hibernation. 1198 */ 1199 static void vmbus_force_channel_rescinded(struct vmbus_channel *channel) 1200 { 1201 struct onmessage_work_context *ctx; 1202 struct vmbus_channel_rescind_offer *rescind; 1203 1204 WARN_ON(!is_hvsock_channel(channel)); 1205 1206 /* 1207 * Allocation size is small and the allocation should really not fail, 1208 * otherwise the state of the hv_sock connections ends up in limbo. 1209 */ 1210 ctx = kzalloc(sizeof(*ctx) + sizeof(*rescind), 1211 GFP_KERNEL | __GFP_NOFAIL); 1212 1213 /* 1214 * So far, these are not really used by Linux. Just set them to the 1215 * reasonable values conforming to the definitions of the fields. 1216 */ 1217 ctx->msg.header.message_type = 1; 1218 ctx->msg.header.payload_size = sizeof(*rescind); 1219 1220 /* These values are actually used by Linux. */ 1221 rescind = (struct vmbus_channel_rescind_offer *)ctx->msg.payload; 1222 rescind->header.msgtype = CHANNELMSG_RESCIND_CHANNELOFFER; 1223 rescind->child_relid = channel->offermsg.child_relid; 1224 1225 INIT_WORK(&ctx->work, vmbus_onmessage_work); 1226 1227 queue_work(vmbus_connection.work_queue, &ctx->work); 1228 } 1229 #endif /* CONFIG_PM_SLEEP */ 1230 1231 /* 1232 * Schedule all channels with events pending. 1233 * The event page can be directly checked to get the id of 1234 * the channel that has the interrupt pending. 1235 */ 1236 static void vmbus_chan_sched(void *event_page_addr) 1237 { 1238 unsigned long *recv_int_page; 1239 u32 maxbits, relid; 1240 union hv_synic_event_flags *event; 1241 1242 if (!event_page_addr) 1243 return; 1244 event = (union hv_synic_event_flags *)event_page_addr + VMBUS_MESSAGE_SINT; 1245 1246 maxbits = READ_ONCE(vmbus_connection.relid_hiwater) + 1; 1247 recv_int_page = event->flags; 1248 1249 if (unlikely(!recv_int_page)) 1250 return; 1251 1252 for_each_set_bit(relid, recv_int_page, maxbits) { 1253 void (*callback_fn)(void *context); 1254 struct vmbus_channel *channel; 1255 1256 if (!sync_test_and_clear_bit(relid, recv_int_page)) 1257 continue; 1258 1259 /* Special case - vmbus channel protocol msg */ 1260 if (relid == 0) 1261 continue; 1262 1263 /* 1264 * Pairs with the kfree_rcu() in vmbus_chan_release(). 1265 * Guarantees that the channel data structure doesn't 1266 * get freed while the channel pointer below is being 1267 * dereferenced. 1268 */ 1269 rcu_read_lock(); 1270 1271 /* Find channel based on relid */ 1272 channel = relid2channel(relid); 1273 if (channel == NULL) 1274 goto sched_unlock_rcu; 1275 1276 if (channel->rescind) 1277 goto sched_unlock_rcu; 1278 1279 /* 1280 * Make sure that the ring buffer data structure doesn't get 1281 * freed while we dereference the ring buffer pointer. Test 1282 * for the channel's onchannel_callback being NULL within a 1283 * sched_lock critical section. See also the inline comments 1284 * in vmbus_reset_channel_cb(). 1285 */ 1286 spin_lock(&channel->sched_lock); 1287 1288 callback_fn = channel->onchannel_callback; 1289 if (unlikely(callback_fn == NULL)) 1290 goto sched_unlock; 1291 1292 trace_vmbus_chan_sched(channel); 1293 1294 ++channel->interrupts; 1295 1296 switch (channel->callback_mode) { 1297 case HV_CALL_ISR: 1298 (*callback_fn)(channel->channel_callback_context); 1299 break; 1300 1301 case HV_CALL_BATCHED: 1302 hv_begin_read(&channel->inbound); 1303 fallthrough; 1304 case HV_CALL_DIRECT: 1305 tasklet_schedule(&channel->callback_event); 1306 } 1307 1308 sched_unlock: 1309 spin_unlock(&channel->sched_lock); 1310 sched_unlock_rcu: 1311 rcu_read_unlock(); 1312 } 1313 } 1314 1315 static void vmbus_message_sched(struct hv_per_cpu_context *hv_cpu, void *message_page_addr) 1316 { 1317 struct hv_message *msg; 1318 1319 if (!message_page_addr) 1320 return; 1321 msg = (struct hv_message *)message_page_addr + VMBUS_MESSAGE_SINT; 1322 1323 /* Check if there are actual msgs to be processed */ 1324 if (msg->header.message_type != HVMSG_NONE) 1325 tasklet_schedule(&hv_cpu->msg_dpc); 1326 } 1327 1328 static void __vmbus_isr(void) 1329 { 1330 struct hv_per_cpu_context *hv_cpu 1331 = this_cpu_ptr(hv_context.cpu_context); 1332 1333 vmbus_chan_sched(hv_cpu->hyp_synic_event_page); 1334 vmbus_chan_sched(hv_cpu->para_synic_event_page); 1335 1336 vmbus_message_sched(hv_cpu, hv_cpu->hyp_synic_message_page); 1337 vmbus_message_sched(hv_cpu, hv_cpu->para_synic_message_page); 1338 } 1339 1340 static DEFINE_PER_CPU(bool, vmbus_irq_pending); 1341 static DEFINE_PER_CPU(struct task_struct *, vmbus_irqd); 1342 1343 static void vmbus_irqd_wake(void) 1344 { 1345 struct task_struct *tsk = __this_cpu_read(vmbus_irqd); 1346 1347 __this_cpu_write(vmbus_irq_pending, true); 1348 wake_up_process(tsk); 1349 } 1350 1351 static void vmbus_irqd_setup(unsigned int cpu) 1352 { 1353 sched_set_fifo(current); 1354 } 1355 1356 static int vmbus_irqd_should_run(unsigned int cpu) 1357 { 1358 return __this_cpu_read(vmbus_irq_pending); 1359 } 1360 1361 static void run_vmbus_irqd(unsigned int cpu) 1362 { 1363 __this_cpu_write(vmbus_irq_pending, false); 1364 __vmbus_isr(); 1365 } 1366 1367 static struct smp_hotplug_thread vmbus_irq_threads = { 1368 .store = &vmbus_irqd, 1369 .setup = vmbus_irqd_setup, 1370 .thread_should_run = vmbus_irqd_should_run, 1371 .thread_fn = run_vmbus_irqd, 1372 .thread_comm = "vmbus_irq/%u", 1373 }; 1374 1375 void vmbus_isr(void) 1376 { 1377 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 1378 vmbus_irqd_wake(); 1379 } else { 1380 static DEFINE_WAIT_OVERRIDE_MAP(vmbus_map, LD_WAIT_CONFIG); 1381 1382 /* 1383 * vmbus_isr is never force-threaded and always invoked at hard 1384 * IRQ level. __vmbus_isr() below can acquire a spinlock_t 1385 * which becomes a sleeping lock and must not be acquired in 1386 * this context. Therefore on PREEMPT_RT this will be threaded 1387 * via vmbus_irqd_wake(). On non-PREEMPT the annotation lets 1388 * lockdep know that acquiring a spinlock_t is not an issue. 1389 */ 1390 lock_map_acquire_try(&vmbus_map); 1391 __vmbus_isr(); 1392 lock_map_release(&vmbus_map); 1393 } 1394 } 1395 EXPORT_SYMBOL_FOR_MODULES(vmbus_isr, "mshv_vtl"); 1396 1397 static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id) 1398 { 1399 vmbus_isr(); 1400 return IRQ_HANDLED; 1401 } 1402 1403 static void vmbus_percpu_work(struct work_struct *work) 1404 { 1405 unsigned int cpu = smp_processor_id(); 1406 1407 hv_synic_init(cpu); 1408 } 1409 1410 static int vmbus_alloc_synic_and_connect(void) 1411 { 1412 int ret, cpu; 1413 struct work_struct __percpu *works; 1414 1415 ret = hv_synic_alloc(); 1416 if (ret < 0) 1417 goto err_alloc; 1418 1419 works = alloc_percpu(struct work_struct); 1420 if (!works) { 1421 ret = -ENOMEM; 1422 goto err_alloc; 1423 } 1424 1425 /* 1426 * Initialize the per-cpu interrupt state and stimer state. 1427 * Then connect to the host. 1428 */ 1429 cpus_read_lock(); 1430 for_each_online_cpu(cpu) { 1431 struct work_struct *work = per_cpu_ptr(works, cpu); 1432 1433 INIT_WORK(work, vmbus_percpu_work); 1434 schedule_work_on(cpu, work); 1435 } 1436 1437 for_each_online_cpu(cpu) 1438 flush_work(per_cpu_ptr(works, cpu)); 1439 1440 /* Register the callbacks for possible CPU online/offline'ing */ 1441 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, "hyperv/vmbus:online", 1442 hv_synic_init, hv_synic_cleanup); 1443 cpus_read_unlock(); 1444 free_percpu(works); 1445 if (ret < 0) 1446 goto err_alloc; 1447 hyperv_cpuhp_online = ret; 1448 1449 ret = vmbus_connect(); 1450 if (ret) 1451 goto err_connect; 1452 return 0; 1453 1454 err_connect: 1455 cpuhp_remove_state(hyperv_cpuhp_online); 1456 return -ENODEV; 1457 err_alloc: 1458 hv_synic_free(); 1459 return -ENOMEM; 1460 } 1461 1462 /* 1463 * vmbus_bus_init -Main vmbus driver initialization routine. 1464 * 1465 * Here, we 1466 * - initialize the vmbus driver context 1467 * - invoke the vmbus hv main init routine 1468 * - retrieve the channel offers 1469 */ 1470 static int vmbus_bus_init(void) 1471 { 1472 int ret; 1473 1474 ret = hv_init(); 1475 if (ret != 0) { 1476 pr_err("Unable to initialize the hypervisor - 0x%x\n", ret); 1477 return ret; 1478 } 1479 1480 ret = bus_register(&hv_bus); 1481 if (ret) 1482 return ret; 1483 1484 /* 1485 * VMbus interrupts are best modeled as per-cpu interrupts. If 1486 * on an architecture with support for per-cpu IRQs (e.g. ARM64), 1487 * allocate a per-cpu IRQ using standard Linux kernel functionality. 1488 * If not on such an architecture (e.g., x86/x64), then rely on 1489 * code in the arch-specific portion of the code tree to connect 1490 * the VMbus interrupt handler. 1491 */ 1492 1493 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 1494 ret = smpboot_register_percpu_thread(&vmbus_irq_threads); 1495 if (ret) 1496 goto err_kthread; 1497 } 1498 1499 if (vmbus_irq == -1) { 1500 hv_setup_vmbus_handler(vmbus_isr); 1501 } else { 1502 ret = request_percpu_irq(vmbus_irq, vmbus_percpu_isr, 1503 "Hyper-V VMbus", &vmbus_evt); 1504 if (ret) { 1505 pr_err("Can't request Hyper-V VMbus IRQ %d, Err %d", 1506 vmbus_irq, ret); 1507 goto err_setup; 1508 } 1509 } 1510 1511 /* 1512 * Cache the value as getting it involves a VM exit on x86(_64), and 1513 * doing that on each VP while initializing SynIC's wastes time. 1514 */ 1515 is_confidential = ms_hyperv.confidential_vmbus_available; 1516 if (is_confidential) 1517 pr_info("Establishing connection to the confidential VMBus\n"); 1518 hv_para_set_sint_proxy(!is_confidential); 1519 ret = vmbus_alloc_synic_and_connect(); 1520 if (ret) 1521 goto err_connect; 1522 1523 /* 1524 * Always register the vmbus unload panic notifier because we 1525 * need to shut the VMbus channel connection on panic. 1526 */ 1527 atomic_notifier_chain_register(&panic_notifier_list, 1528 &hyperv_panic_vmbus_unload_block); 1529 1530 vmbus_request_offers(); 1531 1532 return 0; 1533 1534 err_connect: 1535 if (vmbus_irq == -1) 1536 hv_remove_vmbus_handler(); 1537 else 1538 free_percpu_irq(vmbus_irq, &vmbus_evt); 1539 err_setup: 1540 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 1541 smpboot_unregister_percpu_thread(&vmbus_irq_threads); 1542 err_kthread: 1543 bus_unregister(&hv_bus); 1544 return ret; 1545 } 1546 1547 /** 1548 * __vmbus_driver_register() - Register a vmbus's driver 1549 * @hv_driver: Pointer to driver structure you want to register 1550 * @owner: owner module of the drv 1551 * @mod_name: module name string 1552 * 1553 * Registers the given driver with Linux through the 'driver_register()' call 1554 * and sets up the hyper-v vmbus handling for this driver. 1555 * It will return the state of the 'driver_register()' call. 1556 * 1557 */ 1558 int __vmbus_driver_register(struct hv_driver *hv_driver, struct module *owner, const char *mod_name) 1559 { 1560 int ret; 1561 1562 if (!hv_vmbus_exists()) 1563 return -ENODEV; 1564 1565 pr_info("registering driver %s\n", hv_driver->name); 1566 1567 hv_driver->driver.name = hv_driver->name; 1568 hv_driver->driver.owner = owner; 1569 hv_driver->driver.mod_name = mod_name; 1570 hv_driver->driver.bus = &hv_bus; 1571 1572 spin_lock_init(&hv_driver->dynids.lock); 1573 INIT_LIST_HEAD(&hv_driver->dynids.list); 1574 1575 ret = driver_register(&hv_driver->driver); 1576 1577 return ret; 1578 } 1579 EXPORT_SYMBOL_GPL(__vmbus_driver_register); 1580 1581 /** 1582 * vmbus_driver_unregister() - Unregister a vmbus's driver 1583 * @hv_driver: Pointer to driver structure you want to 1584 * un-register 1585 * 1586 * Un-register the given driver that was previous registered with a call to 1587 * vmbus_driver_register() 1588 */ 1589 void vmbus_driver_unregister(struct hv_driver *hv_driver) 1590 { 1591 if (hv_vmbus_exists()) { 1592 pr_info("unregistering driver %s\n", hv_driver->name); 1593 driver_unregister(&hv_driver->driver); 1594 vmbus_free_dynids(hv_driver); 1595 } 1596 } 1597 EXPORT_SYMBOL_GPL(vmbus_driver_unregister); 1598 1599 1600 /* 1601 * Called when last reference to channel is gone. 1602 */ 1603 static void vmbus_chan_release(struct kobject *kobj) 1604 { 1605 struct vmbus_channel *channel 1606 = container_of(kobj, struct vmbus_channel, kobj); 1607 1608 kfree_rcu(channel, rcu); 1609 } 1610 1611 struct vmbus_chan_attribute { 1612 struct attribute attr; 1613 ssize_t (*show)(struct vmbus_channel *chan, char *buf); 1614 ssize_t (*store)(struct vmbus_channel *chan, 1615 const char *buf, size_t count); 1616 }; 1617 #define VMBUS_CHAN_ATTR(_name, _mode, _show, _store) \ 1618 struct vmbus_chan_attribute chan_attr_##_name \ 1619 = __ATTR(_name, _mode, _show, _store) 1620 #define VMBUS_CHAN_ATTR_RW(_name) \ 1621 struct vmbus_chan_attribute chan_attr_##_name = __ATTR_RW(_name) 1622 #define VMBUS_CHAN_ATTR_RO(_name) \ 1623 struct vmbus_chan_attribute chan_attr_##_name = __ATTR_RO(_name) 1624 #define VMBUS_CHAN_ATTR_WO(_name) \ 1625 struct vmbus_chan_attribute chan_attr_##_name = __ATTR_WO(_name) 1626 1627 static ssize_t vmbus_chan_attr_show(struct kobject *kobj, 1628 struct attribute *attr, char *buf) 1629 { 1630 const struct vmbus_chan_attribute *attribute 1631 = container_of(attr, struct vmbus_chan_attribute, attr); 1632 struct vmbus_channel *chan 1633 = container_of(kobj, struct vmbus_channel, kobj); 1634 1635 if (!attribute->show) 1636 return -EIO; 1637 1638 return attribute->show(chan, buf); 1639 } 1640 1641 static ssize_t vmbus_chan_attr_store(struct kobject *kobj, 1642 struct attribute *attr, const char *buf, 1643 size_t count) 1644 { 1645 const struct vmbus_chan_attribute *attribute 1646 = container_of(attr, struct vmbus_chan_attribute, attr); 1647 struct vmbus_channel *chan 1648 = container_of(kobj, struct vmbus_channel, kobj); 1649 1650 if (!attribute->store) 1651 return -EIO; 1652 1653 return attribute->store(chan, buf, count); 1654 } 1655 1656 static const struct sysfs_ops vmbus_chan_sysfs_ops = { 1657 .show = vmbus_chan_attr_show, 1658 .store = vmbus_chan_attr_store, 1659 }; 1660 1661 static ssize_t out_mask_show(struct vmbus_channel *channel, char *buf) 1662 { 1663 struct hv_ring_buffer_info *rbi = &channel->outbound; 1664 ssize_t ret; 1665 1666 mutex_lock(&rbi->ring_buffer_mutex); 1667 if (!rbi->ring_buffer) { 1668 mutex_unlock(&rbi->ring_buffer_mutex); 1669 return -EINVAL; 1670 } 1671 1672 ret = sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask); 1673 mutex_unlock(&rbi->ring_buffer_mutex); 1674 return ret; 1675 } 1676 static VMBUS_CHAN_ATTR_RO(out_mask); 1677 1678 static ssize_t in_mask_show(struct vmbus_channel *channel, char *buf) 1679 { 1680 struct hv_ring_buffer_info *rbi = &channel->inbound; 1681 ssize_t ret; 1682 1683 mutex_lock(&rbi->ring_buffer_mutex); 1684 if (!rbi->ring_buffer) { 1685 mutex_unlock(&rbi->ring_buffer_mutex); 1686 return -EINVAL; 1687 } 1688 1689 ret = sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask); 1690 mutex_unlock(&rbi->ring_buffer_mutex); 1691 return ret; 1692 } 1693 static VMBUS_CHAN_ATTR_RO(in_mask); 1694 1695 static ssize_t read_avail_show(struct vmbus_channel *channel, char *buf) 1696 { 1697 struct hv_ring_buffer_info *rbi = &channel->inbound; 1698 ssize_t ret; 1699 1700 mutex_lock(&rbi->ring_buffer_mutex); 1701 if (!rbi->ring_buffer) { 1702 mutex_unlock(&rbi->ring_buffer_mutex); 1703 return -EINVAL; 1704 } 1705 1706 ret = sprintf(buf, "%u\n", hv_get_bytes_to_read(rbi)); 1707 mutex_unlock(&rbi->ring_buffer_mutex); 1708 return ret; 1709 } 1710 static VMBUS_CHAN_ATTR_RO(read_avail); 1711 1712 static ssize_t write_avail_show(struct vmbus_channel *channel, char *buf) 1713 { 1714 struct hv_ring_buffer_info *rbi = &channel->outbound; 1715 ssize_t ret; 1716 1717 mutex_lock(&rbi->ring_buffer_mutex); 1718 if (!rbi->ring_buffer) { 1719 mutex_unlock(&rbi->ring_buffer_mutex); 1720 return -EINVAL; 1721 } 1722 1723 ret = sprintf(buf, "%u\n", hv_get_bytes_to_write(rbi)); 1724 mutex_unlock(&rbi->ring_buffer_mutex); 1725 return ret; 1726 } 1727 static VMBUS_CHAN_ATTR_RO(write_avail); 1728 1729 static ssize_t target_cpu_show(struct vmbus_channel *channel, char *buf) 1730 { 1731 return sprintf(buf, "%u\n", channel->target_cpu); 1732 } 1733 1734 int vmbus_channel_set_cpu(struct vmbus_channel *channel, u32 target_cpu) 1735 { 1736 u32 origin_cpu; 1737 int ret = 0; 1738 1739 lockdep_assert_cpus_held(); 1740 lockdep_assert_held(&vmbus_connection.channel_mutex); 1741 1742 if (vmbus_proto_version < VERSION_WIN10_V4_1) 1743 return -EIO; 1744 1745 /* Validate target_cpu for the cpumask_test_cpu() operation below. */ 1746 if (target_cpu >= nr_cpumask_bits) 1747 return -EINVAL; 1748 1749 if (!cpumask_test_cpu(target_cpu, housekeeping_cpumask(HK_TYPE_MANAGED_IRQ))) 1750 return -EINVAL; 1751 1752 if (!cpu_online(target_cpu)) 1753 return -EINVAL; 1754 1755 /* 1756 * Synchronizes vmbus_channel_set_cpu() and channel closure: 1757 * 1758 * { Initially: state = CHANNEL_OPENED } 1759 * 1760 * CPU1 CPU2 1761 * 1762 * [vmbus_channel_set_cpu()] [vmbus_disconnect_ring()] 1763 * 1764 * LOCK channel_mutex LOCK channel_mutex 1765 * LOAD r1 = state LOAD r2 = state 1766 * IF (r1 == CHANNEL_OPENED) IF (r2 == CHANNEL_OPENED) 1767 * SEND MODIFYCHANNEL STORE state = CHANNEL_OPEN 1768 * [...] SEND CLOSECHANNEL 1769 * UNLOCK channel_mutex UNLOCK channel_mutex 1770 * 1771 * Forbids: r1 == r2 == CHANNEL_OPENED (i.e., CPU1's LOCK precedes 1772 * CPU2's LOCK) && CPU2's SEND precedes CPU1's SEND 1773 * 1774 * Note. The host processes the channel messages "sequentially", in 1775 * the order in which they are received on a per-partition basis. 1776 */ 1777 1778 /* 1779 * Hyper-V will ignore MODIFYCHANNEL messages for "non-open" channels; 1780 * avoid sending the message and fail here for such channels. 1781 */ 1782 if (channel->state != CHANNEL_OPENED_STATE) { 1783 ret = -EIO; 1784 goto end; 1785 } 1786 1787 origin_cpu = channel->target_cpu; 1788 if (target_cpu == origin_cpu) 1789 goto end; 1790 1791 if (vmbus_send_modifychannel(channel, 1792 hv_cpu_number_to_vp_number(target_cpu))) { 1793 ret = -EIO; 1794 goto end; 1795 } 1796 1797 /* 1798 * For version before VERSION_WIN10_V5_3, the following warning holds: 1799 * 1800 * Warning. At this point, there is *no* guarantee that the host will 1801 * have successfully processed the vmbus_send_modifychannel() request. 1802 * See the header comment of vmbus_send_modifychannel() for more info. 1803 * 1804 * Lags in the processing of the above vmbus_send_modifychannel() can 1805 * result in missed interrupts if the "old" target CPU is taken offline 1806 * before Hyper-V starts sending interrupts to the "new" target CPU. 1807 * But apart from this offlining scenario, the code tolerates such 1808 * lags. It will function correctly even if a channel interrupt comes 1809 * in on a CPU that is different from the channel target_cpu value. 1810 */ 1811 1812 channel->target_cpu = target_cpu; 1813 1814 /* See init_vp_index(). */ 1815 if (hv_is_perf_channel(channel)) 1816 hv_update_allocated_cpus(origin_cpu, target_cpu); 1817 1818 /* Currently set only for storvsc channels. */ 1819 if (channel->change_target_cpu_callback) { 1820 (*channel->change_target_cpu_callback)(channel, 1821 origin_cpu, target_cpu); 1822 } 1823 1824 end: 1825 return ret; 1826 } 1827 1828 static ssize_t target_cpu_store(struct vmbus_channel *channel, 1829 const char *buf, size_t count) 1830 { 1831 u32 target_cpu; 1832 ssize_t ret; 1833 1834 if (sscanf(buf, "%u", &target_cpu) != 1) 1835 return -EIO; 1836 1837 cpus_read_lock(); 1838 mutex_lock(&vmbus_connection.channel_mutex); 1839 ret = vmbus_channel_set_cpu(channel, target_cpu); 1840 mutex_unlock(&vmbus_connection.channel_mutex); 1841 cpus_read_unlock(); 1842 1843 return ret ?: count; 1844 } 1845 static VMBUS_CHAN_ATTR(cpu, 0644, target_cpu_show, target_cpu_store); 1846 1847 static ssize_t channel_pending_show(struct vmbus_channel *channel, 1848 char *buf) 1849 { 1850 return sprintf(buf, "%d\n", 1851 channel_pending(channel, 1852 vmbus_connection.monitor_pages[1])); 1853 } 1854 static VMBUS_CHAN_ATTR(pending, 0444, channel_pending_show, NULL); 1855 1856 static ssize_t channel_latency_show(struct vmbus_channel *channel, 1857 char *buf) 1858 { 1859 return sprintf(buf, "%d\n", 1860 channel_latency(channel, 1861 vmbus_connection.monitor_pages[1])); 1862 } 1863 static VMBUS_CHAN_ATTR(latency, 0444, channel_latency_show, NULL); 1864 1865 static ssize_t channel_interrupts_show(struct vmbus_channel *channel, char *buf) 1866 { 1867 return sprintf(buf, "%llu\n", channel->interrupts); 1868 } 1869 static VMBUS_CHAN_ATTR(interrupts, 0444, channel_interrupts_show, NULL); 1870 1871 static ssize_t channel_events_show(struct vmbus_channel *channel, char *buf) 1872 { 1873 return sprintf(buf, "%llu\n", channel->sig_events); 1874 } 1875 static VMBUS_CHAN_ATTR(events, 0444, channel_events_show, NULL); 1876 1877 static ssize_t channel_intr_in_full_show(struct vmbus_channel *channel, 1878 char *buf) 1879 { 1880 return sprintf(buf, "%llu\n", 1881 (unsigned long long)channel->intr_in_full); 1882 } 1883 static VMBUS_CHAN_ATTR(intr_in_full, 0444, channel_intr_in_full_show, NULL); 1884 1885 static ssize_t channel_intr_out_empty_show(struct vmbus_channel *channel, 1886 char *buf) 1887 { 1888 return sprintf(buf, "%llu\n", 1889 (unsigned long long)channel->intr_out_empty); 1890 } 1891 static VMBUS_CHAN_ATTR(intr_out_empty, 0444, channel_intr_out_empty_show, NULL); 1892 1893 static ssize_t channel_out_full_first_show(struct vmbus_channel *channel, 1894 char *buf) 1895 { 1896 return sprintf(buf, "%llu\n", 1897 (unsigned long long)channel->out_full_first); 1898 } 1899 static VMBUS_CHAN_ATTR(out_full_first, 0444, channel_out_full_first_show, NULL); 1900 1901 static ssize_t channel_out_full_total_show(struct vmbus_channel *channel, 1902 char *buf) 1903 { 1904 return sprintf(buf, "%llu\n", 1905 (unsigned long long)channel->out_full_total); 1906 } 1907 static VMBUS_CHAN_ATTR(out_full_total, 0444, channel_out_full_total_show, NULL); 1908 1909 static ssize_t subchannel_monitor_id_show(struct vmbus_channel *channel, 1910 char *buf) 1911 { 1912 return sprintf(buf, "%u\n", channel->offermsg.monitorid); 1913 } 1914 static VMBUS_CHAN_ATTR(monitor_id, 0444, subchannel_monitor_id_show, NULL); 1915 1916 static ssize_t subchannel_id_show(struct vmbus_channel *channel, 1917 char *buf) 1918 { 1919 return sprintf(buf, "%u\n", 1920 channel->offermsg.offer.sub_channel_index); 1921 } 1922 static VMBUS_CHAN_ATTR_RO(subchannel_id); 1923 1924 static int hv_mmap_ring_buffer_wrapper(struct file *filp, struct kobject *kobj, 1925 const struct bin_attribute *attr, 1926 struct vm_area_struct *vma) 1927 { 1928 struct vmbus_channel *channel = container_of(kobj, struct vmbus_channel, kobj); 1929 struct vm_area_desc desc; 1930 int err; 1931 1932 /* 1933 * hv_(create|remove)_ring_sysfs implementation ensures that 1934 * mmap_prepare_ring_buffer is not NULL. 1935 */ 1936 compat_set_desc_from_vma(&desc, filp, vma); 1937 err = channel->mmap_prepare_ring_buffer(channel, &desc); 1938 if (err) 1939 return err; 1940 1941 return __compat_vma_mmap(&desc, vma); 1942 } 1943 1944 static struct bin_attribute chan_attr_ring_buffer = { 1945 .attr = { 1946 .name = "ring", 1947 .mode = 0600, 1948 }, 1949 .mmap = hv_mmap_ring_buffer_wrapper, 1950 }; 1951 static struct attribute *vmbus_chan_attrs[] = { 1952 &chan_attr_out_mask.attr, 1953 &chan_attr_in_mask.attr, 1954 &chan_attr_read_avail.attr, 1955 &chan_attr_write_avail.attr, 1956 &chan_attr_cpu.attr, 1957 &chan_attr_pending.attr, 1958 &chan_attr_latency.attr, 1959 &chan_attr_interrupts.attr, 1960 &chan_attr_events.attr, 1961 &chan_attr_intr_in_full.attr, 1962 &chan_attr_intr_out_empty.attr, 1963 &chan_attr_out_full_first.attr, 1964 &chan_attr_out_full_total.attr, 1965 &chan_attr_monitor_id.attr, 1966 &chan_attr_subchannel_id.attr, 1967 NULL 1968 }; 1969 1970 static const struct bin_attribute *vmbus_chan_bin_attrs[] = { 1971 &chan_attr_ring_buffer, 1972 NULL 1973 }; 1974 1975 /* 1976 * Channel-level attribute_group callback function. Returns the permission for 1977 * each attribute, and returns 0 if an attribute is not visible. 1978 */ 1979 static umode_t vmbus_chan_attr_is_visible(struct kobject *kobj, 1980 struct attribute *attr, int idx) 1981 { 1982 const struct vmbus_channel *channel = 1983 container_of(kobj, struct vmbus_channel, kobj); 1984 1985 /* Hide the monitor attributes if the monitor mechanism is not used. */ 1986 if (!channel->offermsg.monitor_allocated && 1987 (attr == &chan_attr_pending.attr || 1988 attr == &chan_attr_latency.attr || 1989 attr == &chan_attr_monitor_id.attr)) 1990 return 0; 1991 1992 return attr->mode; 1993 } 1994 1995 static umode_t vmbus_chan_bin_attr_is_visible(struct kobject *kobj, 1996 const struct bin_attribute *attr, int idx) 1997 { 1998 const struct vmbus_channel *channel = 1999 container_of(kobj, struct vmbus_channel, kobj); 2000 2001 /* Hide ring attribute if channel's ring_sysfs_visible is set to false */ 2002 if (attr == &chan_attr_ring_buffer && !channel->ring_sysfs_visible) 2003 return 0; 2004 2005 return attr->attr.mode; 2006 } 2007 2008 static size_t vmbus_chan_bin_size(struct kobject *kobj, 2009 const struct bin_attribute *bin_attr, int a) 2010 { 2011 const struct vmbus_channel *channel = 2012 container_of(kobj, struct vmbus_channel, kobj); 2013 2014 return channel->ringbuffer_pagecount << PAGE_SHIFT; 2015 } 2016 2017 static const struct attribute_group vmbus_chan_group = { 2018 .attrs = vmbus_chan_attrs, 2019 .bin_attrs = vmbus_chan_bin_attrs, 2020 .is_visible = vmbus_chan_attr_is_visible, 2021 .is_bin_visible = vmbus_chan_bin_attr_is_visible, 2022 .bin_size = vmbus_chan_bin_size, 2023 }; 2024 2025 static const struct kobj_type vmbus_chan_ktype = { 2026 .sysfs_ops = &vmbus_chan_sysfs_ops, 2027 .release = vmbus_chan_release, 2028 }; 2029 2030 /** 2031 * hv_create_ring_sysfs() - create "ring" sysfs entry corresponding to ring buffers for a channel. 2032 * @channel: Pointer to vmbus_channel structure 2033 * @hv_mmap_prepare_ring_buffer: function pointer for initializing the function to be called on mmap 2034 * channel's "ring" sysfs node, which is for the ring buffer of that channel. 2035 * Function pointer is of below type: 2036 * int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel, 2037 * struct vm_area_desc *desc)) 2038 * This has a pointer to the channel and a pointer to vm_area_desc, 2039 * used for mmap_prepare, as arguments. 2040 * 2041 * Sysfs node for ring buffer of a channel is created along with other fields, however its 2042 * visibility is disabled by default. Sysfs creation needs to be controlled when the use-case 2043 * is running. 2044 * For example, HV_NIC device is used either by uio_hv_generic or hv_netvsc at any given point of 2045 * time, and "ring" sysfs is needed only when uio_hv_generic is bound to that device. To avoid 2046 * exposing the ring buffer by default, this function is responsible to enable visibility of 2047 * ring for userspace to use. 2048 * Note: Race conditions can happen with userspace and it is not encouraged to create new 2049 * use-cases for this. This was added to maintain backward compatibility, while solving 2050 * one of the race conditions in uio_hv_generic while creating sysfs. See comments with 2051 * vmbus_add_dynid() and vmbus_device_register(). 2052 * 2053 * Returns 0 on success or error code on failure. 2054 */ 2055 int hv_create_ring_sysfs(struct vmbus_channel *channel, 2056 int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel, 2057 struct vm_area_desc *desc)) 2058 { 2059 struct kobject *kobj = &channel->kobj; 2060 2061 channel->mmap_prepare_ring_buffer = hv_mmap_prepare_ring_buffer; 2062 channel->ring_sysfs_visible = true; 2063 2064 return sysfs_update_group(kobj, &vmbus_chan_group); 2065 } 2066 EXPORT_SYMBOL_GPL(hv_create_ring_sysfs); 2067 2068 /** 2069 * hv_remove_ring_sysfs() - remove ring sysfs entry corresponding to ring buffers for a channel. 2070 * @channel: Pointer to vmbus_channel structure 2071 * 2072 * Hide "ring" sysfs for a channel by changing its is_visible attribute and updating sysfs group. 2073 * 2074 * Returns 0 on success or error code on failure. 2075 */ 2076 int hv_remove_ring_sysfs(struct vmbus_channel *channel) 2077 { 2078 struct kobject *kobj = &channel->kobj; 2079 int ret; 2080 2081 channel->ring_sysfs_visible = false; 2082 ret = sysfs_update_group(kobj, &vmbus_chan_group); 2083 channel->mmap_prepare_ring_buffer = NULL; 2084 return ret; 2085 } 2086 EXPORT_SYMBOL_GPL(hv_remove_ring_sysfs); 2087 2088 /* 2089 * vmbus_add_channel_kobj - setup a sub-directory under device/channels 2090 */ 2091 int vmbus_add_channel_kobj(struct hv_device *dev, struct vmbus_channel *channel) 2092 { 2093 const struct device *device = &dev->device; 2094 struct kobject *kobj = &channel->kobj; 2095 u32 relid = channel->offermsg.child_relid; 2096 int ret; 2097 2098 kobj->kset = dev->channels_kset; 2099 ret = kobject_init_and_add(kobj, &vmbus_chan_ktype, NULL, 2100 "%u", relid); 2101 if (ret) { 2102 kobject_put(kobj); 2103 return ret; 2104 } 2105 2106 ret = sysfs_create_group(kobj, &vmbus_chan_group); 2107 2108 if (ret) { 2109 /* 2110 * The calling functions' error handling paths will cleanup the 2111 * empty channel directory. 2112 */ 2113 kobject_put(kobj); 2114 dev_err(device, "Unable to set up channel sysfs files\n"); 2115 return ret; 2116 } 2117 2118 kobject_uevent(kobj, KOBJ_ADD); 2119 2120 return 0; 2121 } 2122 2123 /* 2124 * vmbus_remove_channel_attr_group - remove the channel's attribute group 2125 */ 2126 void vmbus_remove_channel_attr_group(struct vmbus_channel *channel) 2127 { 2128 sysfs_remove_group(&channel->kobj, &vmbus_chan_group); 2129 } 2130 2131 /* 2132 * vmbus_device_create - Creates and registers a new child device 2133 * on the vmbus. 2134 */ 2135 struct hv_device *vmbus_device_create(const guid_t *type, 2136 const guid_t *instance, 2137 struct vmbus_channel *channel) 2138 { 2139 struct hv_device *child_device_obj; 2140 2141 child_device_obj = kzalloc_obj(struct hv_device); 2142 if (!child_device_obj) { 2143 pr_err("Unable to allocate device object for child device\n"); 2144 return NULL; 2145 } 2146 2147 child_device_obj->channel = channel; 2148 guid_copy(&child_device_obj->dev_type, type); 2149 guid_copy(&child_device_obj->dev_instance, instance); 2150 child_device_obj->vendor_id = PCI_VENDOR_ID_MICROSOFT; 2151 2152 return child_device_obj; 2153 } 2154 2155 /* 2156 * vmbus_device_register - Register the child device 2157 */ 2158 int vmbus_device_register(struct hv_device *child_device_obj) 2159 { 2160 struct kobject *kobj = &child_device_obj->device.kobj; 2161 int ret; 2162 2163 dev_set_name(&child_device_obj->device, "%pUl", 2164 &child_device_obj->channel->offermsg.offer.if_instance); 2165 2166 child_device_obj->device.bus = &hv_bus; 2167 child_device_obj->device.parent = vmbus_root_device; 2168 child_device_obj->device.release = vmbus_device_release; 2169 2170 child_device_obj->device.dma_parms = &child_device_obj->dma_parms; 2171 child_device_obj->device.dma_mask = &child_device_obj->dma_mask; 2172 dma_set_mask(&child_device_obj->device, DMA_BIT_MASK(64)); 2173 dma_set_coherent_mask(&child_device_obj->device, DMA_BIT_MASK(64)); 2174 2175 /* 2176 * Register with the LDM. This will kick off the driver/device 2177 * binding...which will eventually call vmbus_match() and vmbus_probe() 2178 */ 2179 ret = device_register(&child_device_obj->device); 2180 if (ret) { 2181 pr_err("Unable to register child device\n"); 2182 put_device(&child_device_obj->device); 2183 return ret; 2184 } 2185 2186 /* 2187 * If device_register() found a driver to assign to the device, the 2188 * driver's probe function has already run at this point. If that 2189 * probe function accesses or operates on the "channels" subdirectory 2190 * in sysfs, those operations will have failed because the "channels" 2191 * subdirectory doesn't exist until the code below runs. Or if the 2192 * probe function creates a /dev entry, a user space program could 2193 * find and open the /dev entry, and then create a race by accessing 2194 * the "channels" subdirectory while the creation steps are in progress 2195 * here. The race can't result in a kernel failure, but the user space 2196 * program may get an error in accessing "channels" or its 2197 * subdirectories. See also comments with vmbus_add_dynid() about a 2198 * related race condition. 2199 */ 2200 child_device_obj->channels_kset = kset_create_and_add("channels", 2201 NULL, kobj); 2202 if (!child_device_obj->channels_kset) { 2203 ret = -ENOMEM; 2204 goto err_dev_unregister; 2205 } 2206 2207 ret = vmbus_add_channel_kobj(child_device_obj, 2208 child_device_obj->channel); 2209 if (ret) { 2210 pr_err("Unable to register primary channel\n"); 2211 goto err_kset_unregister; 2212 } 2213 hv_debug_add_dev_dir(child_device_obj); 2214 2215 return 0; 2216 2217 err_kset_unregister: 2218 kset_unregister(child_device_obj->channels_kset); 2219 2220 err_dev_unregister: 2221 device_unregister(&child_device_obj->device); 2222 return ret; 2223 } 2224 2225 /* 2226 * vmbus_device_unregister - Remove the specified child device 2227 * from the vmbus. 2228 */ 2229 void vmbus_device_unregister(struct hv_device *device_obj) 2230 { 2231 pr_debug("child device %s unregistered\n", 2232 dev_name(&device_obj->device)); 2233 2234 kset_unregister(device_obj->channels_kset); 2235 2236 /* 2237 * Kick off the process of unregistering the device. 2238 * This will call vmbus_remove() and eventually vmbus_device_release() 2239 */ 2240 device_unregister(&device_obj->device); 2241 } 2242 EXPORT_SYMBOL_GPL(vmbus_device_unregister); 2243 2244 #ifdef CONFIG_ACPI 2245 /* 2246 * VMBUS is an acpi enumerated device. Get the information we 2247 * need from DSDT. 2248 */ 2249 static acpi_status vmbus_walk_resources(struct acpi_resource *res, void *ctx) 2250 { 2251 resource_size_t start = 0; 2252 resource_size_t end = 0; 2253 struct resource *new_res; 2254 struct resource **old_res = &hyperv_mmio; 2255 struct resource **prev_res = NULL; 2256 struct resource r; 2257 2258 switch (res->type) { 2259 2260 /* 2261 * "Address" descriptors are for bus windows. Ignore 2262 * "memory" descriptors, which are for registers on 2263 * devices. 2264 */ 2265 case ACPI_RESOURCE_TYPE_ADDRESS32: 2266 start = res->data.address32.address.minimum; 2267 end = res->data.address32.address.maximum; 2268 break; 2269 2270 case ACPI_RESOURCE_TYPE_ADDRESS64: 2271 start = res->data.address64.address.minimum; 2272 end = res->data.address64.address.maximum; 2273 break; 2274 2275 /* 2276 * The IRQ information is needed only on ARM64, which Hyper-V 2277 * sets up in the extended format. IRQ information is present 2278 * on x86/x64 in the non-extended format but it is not used by 2279 * Linux. So don't bother checking for the non-extended format. 2280 */ 2281 case ACPI_RESOURCE_TYPE_EXTENDED_IRQ: 2282 if (!acpi_dev_resource_interrupt(res, 0, &r)) { 2283 pr_err("Unable to parse Hyper-V ACPI interrupt\n"); 2284 return AE_ERROR; 2285 } 2286 /* ARM64 INTID for VMbus */ 2287 vmbus_interrupt = res->data.extended_irq.interrupts[0]; 2288 /* Linux IRQ number */ 2289 vmbus_irq = r.start; 2290 return AE_OK; 2291 2292 default: 2293 /* Unused resource type */ 2294 return AE_OK; 2295 2296 } 2297 /* 2298 * Ignore ranges that are below 1MB, as they're not 2299 * necessary or useful here. 2300 */ 2301 if (end < 0x100000) 2302 return AE_OK; 2303 2304 new_res = kzalloc_obj(*new_res, GFP_ATOMIC); 2305 if (!new_res) 2306 return AE_NO_MEMORY; 2307 2308 /* If this range overlaps the virtual TPM, truncate it. */ 2309 if (end >= VTPM_BASE_ADDRESS && start < VTPM_BASE_ADDRESS) 2310 end = VTPM_BASE_ADDRESS - 1; 2311 2312 new_res->name = "hyperv mmio"; 2313 new_res->flags = IORESOURCE_MEM; 2314 new_res->start = start; 2315 new_res->end = end; 2316 2317 /* 2318 * If two ranges are adjacent, merge them. 2319 */ 2320 do { 2321 if (!*old_res) { 2322 *old_res = new_res; 2323 break; 2324 } 2325 2326 if (((*old_res)->end + 1) == new_res->start) { 2327 (*old_res)->end = new_res->end; 2328 kfree(new_res); 2329 break; 2330 } 2331 2332 if ((*old_res)->start == new_res->end + 1) { 2333 (*old_res)->start = new_res->start; 2334 kfree(new_res); 2335 break; 2336 } 2337 2338 if ((*old_res)->start > new_res->end) { 2339 new_res->sibling = *old_res; 2340 if (prev_res) 2341 (*prev_res)->sibling = new_res; 2342 *old_res = new_res; 2343 break; 2344 } 2345 2346 prev_res = old_res; 2347 old_res = &(*old_res)->sibling; 2348 2349 } while (1); 2350 2351 return AE_OK; 2352 } 2353 #endif 2354 2355 static void vmbus_mmio_remove(void) 2356 { 2357 struct resource *cur_res; 2358 struct resource *next_res; 2359 2360 if (hyperv_mmio) { 2361 if (fb_mmio) { 2362 __release_region(hyperv_mmio, fb_mmio->start, 2363 resource_size(fb_mmio)); 2364 fb_mmio = NULL; 2365 } 2366 2367 for (cur_res = hyperv_mmio; cur_res; cur_res = next_res) { 2368 next_res = cur_res->sibling; 2369 kfree(cur_res); 2370 } 2371 } 2372 } 2373 2374 static void __maybe_unused vmbus_reserve_fb(void) 2375 { 2376 resource_size_t start = 0, size; 2377 resource_size_t low_mmio_base; 2378 struct pci_dev *pdev; 2379 2380 if (efi_enabled(EFI_BOOT)) { 2381 /* Gen2 VM: get FB base from EFI framebuffer */ 2382 if (IS_ENABLED(CONFIG_SYSFB)) { 2383 start = sysfb_primary_display.screen.lfb_base; 2384 size = max_t(__u32, sysfb_primary_display.screen.lfb_size, 0x800000); 2385 2386 low_mmio_base = hyperv_mmio->start; 2387 if (!low_mmio_base || upper_32_bits(low_mmio_base) || 2388 (start && start < low_mmio_base)) { 2389 pr_warn("Unexpected low mmio base %pa\n", &low_mmio_base); 2390 } else { 2391 /* 2392 * If the kdump/kexec or CVM kernel's lfb_base 2393 * is 0, fall back to the low mmio base. 2394 */ 2395 if (!start) 2396 start = low_mmio_base; 2397 /* 2398 * Reserve half of the space below 4GB for high 2399 * resolutions, but cap the reservation to 128MB. 2400 */ 2401 size = min((SZ_4G - start) / 2, SZ_128M); 2402 } 2403 } 2404 } else { 2405 /* Gen1 VM: get FB base from PCI */ 2406 pdev = pci_get_device(PCI_VENDOR_ID_MICROSOFT, 2407 PCI_DEVICE_ID_HYPERV_VIDEO, NULL); 2408 if (!pdev) 2409 return; 2410 2411 if (pdev->resource[0].flags & IORESOURCE_MEM) { 2412 start = pci_resource_start(pdev, 0); 2413 size = pci_resource_len(pdev, 0); 2414 } 2415 2416 /* 2417 * Release the PCI device so hyperv_drm driver can grab it 2418 * later. 2419 */ 2420 pci_dev_put(pdev); 2421 } 2422 2423 if (!start) { 2424 pr_warn("Unexpected framebuffer mmio base of zero\n"); 2425 return; 2426 } 2427 2428 /* 2429 * Make a claim for the frame buffer in the resource tree under the 2430 * first node, which will be the one below 4GB. The length seems to 2431 * be underreported, particularly in a Generation 1 VM. So start out 2432 * reserving a larger area and make it smaller until it succeeds. 2433 */ 2434 for (; !fb_mmio && (size >= 0x100000); size >>= 1) 2435 fb_mmio = __request_region(hyperv_mmio, start, size, fb_mmio_name, 0); 2436 2437 pr_info("hv_mmio=%pR,%pR fb=%pR\n", hyperv_mmio, hyperv_mmio->sibling, fb_mmio); 2438 } 2439 2440 /** 2441 * vmbus_allocate_mmio() - Pick a memory-mapped I/O range. 2442 * @new: If successful, supplied a pointer to the 2443 * allocated MMIO space. 2444 * @device_obj: Identifies the caller 2445 * @min: Minimum guest physical address of the 2446 * allocation 2447 * @max: Maximum guest physical address 2448 * @size: Size of the range to be allocated 2449 * @align: Alignment of the range to be allocated 2450 * @fb_overlap_ok: Whether this allocation can be allowed 2451 * to overlap the video frame buffer. 2452 * 2453 * This function walks the resources granted to VMBus by the 2454 * _CRS object in the ACPI namespace underneath the parent 2455 * "bridge" whether that's a root PCI bus in the Generation 1 2456 * case or a Module Device in the Generation 2 case. It then 2457 * attempts to allocate from the global MMIO pool in a way that 2458 * matches the constraints supplied in these parameters and by 2459 * that _CRS. 2460 * 2461 * Return: 0 on success, -errno on failure 2462 */ 2463 int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj, 2464 resource_size_t min, resource_size_t max, 2465 resource_size_t size, resource_size_t align, 2466 bool fb_overlap_ok) 2467 { 2468 struct resource *iter, *shadow; 2469 resource_size_t range_min, range_max, start, end; 2470 const char *dev_n = dev_name(&device_obj->device); 2471 int retval; 2472 2473 retval = -ENXIO; 2474 mutex_lock(&hyperv_mmio_lock); 2475 2476 /* 2477 * If overlaps with frame buffers are allowed, then first attempt to 2478 * make the allocation from within the reserved region. Because it 2479 * is already reserved, no shadow allocation is necessary. 2480 */ 2481 if (fb_overlap_ok && fb_mmio && !(min > fb_mmio->end) && 2482 !(max < fb_mmio->start)) { 2483 2484 range_min = fb_mmio->start; 2485 range_max = fb_mmio->end; 2486 start = (range_min + align - 1) & ~(align - 1); 2487 for (; start + size - 1 <= range_max; start += align) { 2488 *new = request_mem_region_exclusive(start, size, dev_n); 2489 if (*new) { 2490 retval = 0; 2491 goto exit; 2492 } 2493 } 2494 } 2495 2496 for (iter = hyperv_mmio; iter; iter = iter->sibling) { 2497 if ((iter->start >= max) || (iter->end <= min)) 2498 continue; 2499 2500 range_min = iter->start; 2501 range_max = iter->end; 2502 start = (range_min + align - 1) & ~(align - 1); 2503 for (; start + size - 1 <= range_max; start += align) { 2504 end = start + size - 1; 2505 2506 /* Skip the whole fb_mmio region if not fb_overlap_ok */ 2507 if (!fb_overlap_ok && fb_mmio && 2508 (((start >= fb_mmio->start) && (start <= fb_mmio->end)) || 2509 ((end >= fb_mmio->start) && (end <= fb_mmio->end)))) 2510 continue; 2511 2512 shadow = __request_region(iter, start, size, NULL, 2513 IORESOURCE_BUSY); 2514 if (!shadow) 2515 continue; 2516 2517 *new = request_mem_region_exclusive(start, size, dev_n); 2518 if (*new) { 2519 shadow->name = (char *)*new; 2520 retval = 0; 2521 goto exit; 2522 } 2523 2524 __release_region(iter, start, size); 2525 } 2526 } 2527 2528 exit: 2529 mutex_unlock(&hyperv_mmio_lock); 2530 return retval; 2531 } 2532 EXPORT_SYMBOL_GPL(vmbus_allocate_mmio); 2533 2534 /** 2535 * vmbus_free_mmio() - Free a memory-mapped I/O range. 2536 * @start: Base address of region to release. 2537 * @size: Size of the range to be allocated 2538 * 2539 * This function releases anything requested by 2540 * vmbus_mmio_allocate(). 2541 */ 2542 void vmbus_free_mmio(resource_size_t start, resource_size_t size) 2543 { 2544 struct resource *iter; 2545 2546 mutex_lock(&hyperv_mmio_lock); 2547 2548 /* 2549 * If all bytes of the MMIO range to be released are within the 2550 * special case fb_mmio shadow region, skip releasing the shadow 2551 * region since no corresponding __request_region() was done 2552 * in vmbus_allocate_mmio(). 2553 */ 2554 if (fb_mmio && start >= fb_mmio->start && 2555 (start + size - 1 <= fb_mmio->end)) 2556 goto skip_shadow_release; 2557 2558 for (iter = hyperv_mmio; iter; iter = iter->sibling) { 2559 if ((iter->start >= start + size) || (iter->end <= start)) 2560 continue; 2561 2562 __release_region(iter, start, size); 2563 } 2564 2565 skip_shadow_release: 2566 release_mem_region(start, size); 2567 mutex_unlock(&hyperv_mmio_lock); 2568 2569 } 2570 EXPORT_SYMBOL_GPL(vmbus_free_mmio); 2571 2572 #ifdef CONFIG_ACPI 2573 static int vmbus_acpi_add(struct platform_device *pdev) 2574 { 2575 struct acpi_device *ancestor; 2576 struct acpi_device *device; 2577 acpi_status result; 2578 int ret_val = -ENODEV; 2579 2580 device = ACPI_COMPANION(&pdev->dev); 2581 if (!device) 2582 return -ENODEV; 2583 2584 vmbus_root_device = &device->dev; 2585 2586 /* 2587 * Older versions of Hyper-V for ARM64 fail to include the _CCA 2588 * method on the top level VMbus device in the DSDT. But devices 2589 * are hardware coherent in all current Hyper-V use cases, so fix 2590 * up the ACPI device to behave as if _CCA is present and indicates 2591 * hardware coherence. 2592 */ 2593 ACPI_COMPANION_SET(&device->dev, device); 2594 if (IS_ENABLED(CONFIG_ACPI_CCA_REQUIRED) && 2595 device_get_dma_attr(&device->dev) == DEV_DMA_NOT_SUPPORTED) { 2596 pr_info("No ACPI _CCA found; assuming coherent device I/O\n"); 2597 device->flags.cca_seen = true; 2598 device->flags.coherent_dma = true; 2599 } 2600 2601 result = acpi_walk_resources(device->handle, METHOD_NAME__CRS, 2602 vmbus_walk_resources, NULL); 2603 2604 if (ACPI_FAILURE(result)) 2605 goto acpi_walk_err; 2606 /* 2607 * Some ancestor of the vmbus acpi device (Gen1 or Gen2 2608 * firmware) is the VMOD that has the mmio ranges. Get that. 2609 */ 2610 for (ancestor = acpi_dev_parent(device); 2611 ancestor && ancestor->handle != ACPI_ROOT_OBJECT; 2612 ancestor = acpi_dev_parent(ancestor)) { 2613 result = acpi_walk_resources(ancestor->handle, METHOD_NAME__CRS, 2614 vmbus_walk_resources, NULL); 2615 2616 if (ACPI_FAILURE(result)) 2617 continue; 2618 if (hyperv_mmio) { 2619 vmbus_reserve_fb(); 2620 break; 2621 } 2622 } 2623 ret_val = 0; 2624 2625 acpi_walk_err: 2626 if (ret_val) 2627 vmbus_mmio_remove(); 2628 return ret_val; 2629 } 2630 #else 2631 static int vmbus_acpi_add(struct platform_device *pdev) 2632 { 2633 return 0; 2634 } 2635 #endif 2636 #ifndef HYPERVISOR_CALLBACK_VECTOR 2637 static int vmbus_set_irq(struct platform_device *pdev) 2638 { 2639 struct irq_data *data; 2640 int irq; 2641 irq_hw_number_t hwirq; 2642 2643 irq = platform_get_irq(pdev, 0); 2644 /* platform_get_irq() may not return 0. */ 2645 if (irq < 0) 2646 return irq; 2647 2648 data = irq_get_irq_data(irq); 2649 if (!data) { 2650 pr_err("No interrupt data for VMBus virq %d\n", irq); 2651 return -ENODEV; 2652 } 2653 hwirq = irqd_to_hwirq(data); 2654 2655 vmbus_irq = irq; 2656 vmbus_interrupt = hwirq; 2657 pr_debug("VMBus virq %d, hwirq %d\n", vmbus_irq, vmbus_interrupt); 2658 2659 return 0; 2660 } 2661 #endif 2662 2663 static int vmbus_device_add(struct platform_device *pdev) 2664 { 2665 struct resource **cur_res = &hyperv_mmio; 2666 struct of_range range; 2667 struct of_range_parser parser; 2668 struct device_node *np = pdev->dev.of_node; 2669 int ret; 2670 2671 vmbus_root_device = &pdev->dev; 2672 2673 ret = of_range_parser_init(&parser, np); 2674 if (ret) 2675 return ret; 2676 2677 #ifndef HYPERVISOR_CALLBACK_VECTOR 2678 ret = vmbus_set_irq(pdev); 2679 if (ret) 2680 return ret; 2681 #endif 2682 for_each_of_range(&parser, &range) { 2683 struct resource *res; 2684 2685 res = kzalloc_obj(*res); 2686 if (!res) { 2687 vmbus_mmio_remove(); 2688 return -ENOMEM; 2689 } 2690 2691 res->name = "hyperv mmio"; 2692 res->flags = range.flags; 2693 res->start = range.cpu_addr; 2694 res->end = range.cpu_addr + range.size; 2695 2696 *cur_res = res; 2697 cur_res = &res->sibling; 2698 } 2699 2700 return ret; 2701 } 2702 2703 static int vmbus_platform_driver_probe(struct platform_device *pdev) 2704 { 2705 if (acpi_disabled) 2706 return vmbus_device_add(pdev); 2707 else 2708 return vmbus_acpi_add(pdev); 2709 } 2710 2711 static void vmbus_platform_driver_remove(struct platform_device *pdev) 2712 { 2713 vmbus_mmio_remove(); 2714 } 2715 2716 #ifdef CONFIG_PM_SLEEP 2717 static int vmbus_bus_suspend(struct device *dev) 2718 { 2719 struct hv_per_cpu_context *hv_cpu = per_cpu_ptr( 2720 hv_context.cpu_context, VMBUS_CONNECT_CPU); 2721 struct vmbus_channel *channel, *sc; 2722 2723 tasklet_disable(&hv_cpu->msg_dpc); 2724 vmbus_connection.ignore_any_offer_msg = true; 2725 /* The tasklet_enable() takes care of providing a memory barrier */ 2726 tasklet_enable(&hv_cpu->msg_dpc); 2727 2728 /* Drain all the workqueues as we are in suspend */ 2729 drain_workqueue(vmbus_connection.rescind_work_queue); 2730 drain_workqueue(vmbus_connection.work_queue); 2731 drain_workqueue(vmbus_connection.handle_primary_chan_wq); 2732 drain_workqueue(vmbus_connection.handle_sub_chan_wq); 2733 2734 mutex_lock(&vmbus_connection.channel_mutex); 2735 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { 2736 if (!is_hvsock_channel(channel)) 2737 continue; 2738 2739 vmbus_force_channel_rescinded(channel); 2740 } 2741 mutex_unlock(&vmbus_connection.channel_mutex); 2742 2743 /* 2744 * Wait until all the sub-channels and hv_sock channels have been 2745 * cleaned up. Sub-channels should be destroyed upon suspend, otherwise 2746 * they would conflict with the new sub-channels that will be created 2747 * in the resume path. hv_sock channels should also be destroyed, but 2748 * a hv_sock channel of an established hv_sock connection can not be 2749 * really destroyed since it may still be referenced by the userspace 2750 * application, so we just force the hv_sock channel to be rescinded 2751 * by vmbus_force_channel_rescinded(), and the userspace application 2752 * will thoroughly destroy the channel after hibernation. 2753 * 2754 * Note: the counter nr_chan_close_on_suspend may never go above 0 if 2755 * the VM has no sub-channel and hv_sock channel, e.g. a 1-vCPU VM. 2756 */ 2757 if (atomic_read(&vmbus_connection.nr_chan_close_on_suspend) > 0) 2758 wait_for_completion(&vmbus_connection.ready_for_suspend_event); 2759 2760 mutex_lock(&vmbus_connection.channel_mutex); 2761 2762 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { 2763 /* 2764 * Remove the channel from the array of channels and invalidate 2765 * the channel's relid. Upon resume, vmbus_onoffer() will fix 2766 * up the relid (and other fields, if necessary) and add the 2767 * channel back to the array. 2768 */ 2769 vmbus_channel_unmap_relid(channel); 2770 channel->offermsg.child_relid = INVALID_RELID; 2771 2772 if (is_hvsock_channel(channel)) { 2773 if (!channel->rescind) { 2774 pr_err("hv_sock channel not rescinded!\n"); 2775 WARN_ON_ONCE(1); 2776 } 2777 continue; 2778 } 2779 2780 list_for_each_entry(sc, &channel->sc_list, sc_list) { 2781 pr_err("Sub-channel not deleted!\n"); 2782 WARN_ON_ONCE(1); 2783 } 2784 } 2785 2786 mutex_unlock(&vmbus_connection.channel_mutex); 2787 2788 vmbus_initiate_unload(false); 2789 2790 return 0; 2791 } 2792 2793 static int vmbus_bus_resume(struct device *dev) 2794 { 2795 struct vmbus_channel *channel; 2796 struct vmbus_channel_msginfo *msginfo; 2797 size_t msgsize; 2798 int ret; 2799 2800 vmbus_connection.ignore_any_offer_msg = false; 2801 2802 /* 2803 * We only use the 'vmbus_proto_version', which was in use before 2804 * hibernation, to re-negotiate with the host. 2805 */ 2806 if (!vmbus_proto_version) { 2807 pr_err("Invalid proto version = 0x%x\n", vmbus_proto_version); 2808 return -EINVAL; 2809 } 2810 2811 msgsize = sizeof(*msginfo) + 2812 sizeof(struct vmbus_channel_initiate_contact); 2813 2814 msginfo = kzalloc(msgsize, GFP_KERNEL); 2815 2816 if (msginfo == NULL) 2817 return -ENOMEM; 2818 2819 ret = vmbus_negotiate_version(msginfo, vmbus_proto_version); 2820 2821 kfree(msginfo); 2822 2823 if (ret != 0) 2824 return ret; 2825 2826 vmbus_request_offers(); 2827 2828 mutex_lock(&vmbus_connection.channel_mutex); 2829 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { 2830 if (channel->offermsg.child_relid != INVALID_RELID) 2831 continue; 2832 2833 /* hvsock channels are not expected to be present. */ 2834 if (is_hvsock_channel(channel)) 2835 continue; 2836 2837 pr_err("channel %pUl/%pUl not present after resume.\n", 2838 &channel->offermsg.offer.if_type, 2839 &channel->offermsg.offer.if_instance); 2840 /* ToDo: Cleanup these channels here */ 2841 } 2842 mutex_unlock(&vmbus_connection.channel_mutex); 2843 2844 /* Reset the event for the next suspend. */ 2845 reinit_completion(&vmbus_connection.ready_for_suspend_event); 2846 2847 return 0; 2848 } 2849 #else 2850 #define vmbus_bus_suspend NULL 2851 #define vmbus_bus_resume NULL 2852 #endif /* CONFIG_PM_SLEEP */ 2853 2854 static const __maybe_unused struct of_device_id vmbus_of_match[] = { 2855 { 2856 .compatible = "microsoft,vmbus", 2857 }, 2858 { 2859 /* sentinel */ 2860 }, 2861 }; 2862 MODULE_DEVICE_TABLE(of, vmbus_of_match); 2863 2864 static const __maybe_unused struct acpi_device_id vmbus_acpi_device_ids[] = { 2865 {"VMBUS", 0}, 2866 {"VMBus", 0}, 2867 {"", 0}, 2868 }; 2869 MODULE_DEVICE_TABLE(acpi, vmbus_acpi_device_ids); 2870 2871 /* 2872 * Note: we must use the "no_irq" ops, otherwise hibernation can not work with 2873 * PCI device assignment, because "pci_dev_pm_ops" uses the "noirq" ops: in 2874 * the resume path, the pci "noirq" restore op runs before "non-noirq" op (see 2875 * resume_target_kernel() -> dpm_resume_start(), and hibernation_restore() -> 2876 * dpm_resume_end()). This means vmbus_bus_resume() and the pci-hyperv's 2877 * resume callback must also run via the "noirq" ops. 2878 * 2879 * Set suspend_noirq/resume_noirq to NULL for Suspend-to-Idle: see the comment 2880 * earlier in this file before vmbus_pm. 2881 */ 2882 2883 static const struct dev_pm_ops vmbus_bus_pm = { 2884 .suspend_noirq = NULL, 2885 .resume_noirq = NULL, 2886 .freeze_noirq = vmbus_bus_suspend, 2887 .thaw_noirq = vmbus_bus_resume, 2888 .poweroff_noirq = vmbus_bus_suspend, 2889 .restore_noirq = vmbus_bus_resume 2890 }; 2891 2892 static struct platform_driver vmbus_platform_driver = { 2893 .probe = vmbus_platform_driver_probe, 2894 .remove = vmbus_platform_driver_remove, 2895 .driver = { 2896 .name = "vmbus", 2897 .acpi_match_table = ACPI_PTR(vmbus_acpi_device_ids), 2898 .of_match_table = of_match_ptr(vmbus_of_match), 2899 .pm = &vmbus_bus_pm, 2900 .probe_type = PROBE_FORCE_SYNCHRONOUS, 2901 } 2902 }; 2903 2904 static void hv_kexec_handler(void) 2905 { 2906 vmbus_initiate_unload(false); 2907 /* Make sure conn_state is set as hv_synic_cleanup checks for it */ 2908 mb(); 2909 cpuhp_remove_state(hyperv_cpuhp_online); 2910 }; 2911 2912 static void hv_vmbus_syscore_shutdown(void *data) 2913 { 2914 if (kexec_is_in_progress()) 2915 hv_kexec_handler(); 2916 } 2917 2918 static void hv_crash_handler(struct pt_regs *regs) 2919 { 2920 int cpu; 2921 2922 if (!skip_vmbus_unload) 2923 vmbus_initiate_unload(true); 2924 /* 2925 * In crash handler we can't schedule synic cleanup for all CPUs, 2926 * doing the cleanup for current CPU only. This should be sufficient 2927 * for kdump. 2928 */ 2929 cpu = smp_processor_id(); 2930 hv_stimer_cleanup(cpu); 2931 hv_hyp_synic_disable_regs(cpu); 2932 }; 2933 2934 static int hv_synic_suspend(void *data) 2935 { 2936 /* 2937 * When we reach here, all the non-boot CPUs have been offlined, 2938 * and their stimers have been cleaned up by the cpuhp teardown 2939 * callback hv_stimer_cleanup(). 2940 * 2941 * hv_synic_suspend() only runs on CPU0 with interrupts disabled. 2942 * The stimer on CPU0 is not explicitly cleaned up here because: 2943 * 1) it's harmless as interrupts remain disabled between 2944 * syscore_suspend() and syscore_resume(), so the stimer cannot 2945 * fire during this window: see create_image() and 2946 * resume_target_kernel() 2947 * 2) the stimer on CPU0 is automatically disabled later by 2948 * syscore_suspend() -> timekeeping_suspend() -> tick_suspend() -> ... 2949 * -> clockevents_shutdown() -> ... -> hv_ce_shutdown() 2950 */ 2951 2952 hv_hyp_synic_disable_regs(0); 2953 2954 return 0; 2955 } 2956 2957 static void hv_synic_resume(void *data) 2958 { 2959 hv_hyp_synic_enable_regs(0); 2960 2961 /* 2962 * Note: we don't need to call hv_stimer_init(0), because the timer 2963 * on CPU0 is not unbound in hv_synic_suspend(), and the timer is 2964 * automatically re-enabled in timekeeping_resume(). 2965 */ 2966 } 2967 2968 /* The suspend and resume callbacks run only on CPU0, with irqs_disabled. */ 2969 static const struct syscore_ops hv_vmbus_syscore_ops = { 2970 .suspend = hv_synic_suspend, 2971 .resume = hv_synic_resume, 2972 .shutdown = hv_vmbus_syscore_shutdown, 2973 }; 2974 2975 static struct syscore hv_vmbus_syscore = { 2976 .ops = &hv_vmbus_syscore_ops, 2977 }; 2978 2979 static int __init hv_acpi_init(void) 2980 { 2981 int ret; 2982 2983 if (!hv_is_hyperv_initialized()) 2984 return -ENODEV; 2985 2986 if (hv_root_partition() && !hv_nested) 2987 /* 2988 * A non-nested root partition does not need VMBus client 2989 * functionality. However, the mshv_root module may have 2990 * a dependency on the VMBus module as described in 2991 * commit 840b740a35bf. Return success so the module 2992 * loads even though no VMBus initialization is done. 2993 */ 2994 return 0; 2995 2996 /* 2997 * Get ACPI resources first. 2998 */ 2999 ret = platform_driver_register(&vmbus_platform_driver); 3000 if (ret) 3001 return ret; 3002 3003 if (!vmbus_root_device) { 3004 ret = -ENODEV; 3005 goto cleanup; 3006 } 3007 3008 /* 3009 * If we're on an architecture with a hardcoded hypervisor 3010 * vector (i.e. x86/x64), override the VMbus interrupt found 3011 * in the ACPI tables. Ensure vmbus_irq is not set since the 3012 * normal Linux IRQ mechanism is not used in this case. 3013 */ 3014 #ifdef HYPERVISOR_CALLBACK_VECTOR 3015 vmbus_interrupt = HYPERVISOR_CALLBACK_VECTOR; 3016 vmbus_irq = -1; 3017 #endif 3018 3019 hv_debug_init(); 3020 3021 ret = vmbus_bus_init(); 3022 if (ret) 3023 goto cleanup; 3024 3025 hv_setup_crash_handler(hv_crash_handler); 3026 3027 register_syscore(&hv_vmbus_syscore); 3028 3029 return 0; 3030 3031 cleanup: 3032 platform_driver_unregister(&vmbus_platform_driver); 3033 vmbus_root_device = NULL; 3034 return ret; 3035 } 3036 3037 static void __exit vmbus_exit(void) 3038 { 3039 int cpu; 3040 3041 if (hv_root_partition() && !hv_nested) 3042 /* 3043 * If a non-nested root partition loaded the VMBus module, 3044 * hv_acpi_init() did not do any VMBus initialization. 3045 * There's nothing to clean up, so just return. 3046 */ 3047 return; 3048 3049 unregister_syscore(&hv_vmbus_syscore); 3050 3051 hv_remove_crash_handler(); 3052 hv_stimer_global_cleanup(); 3053 vmbus_disconnect(); 3054 if (vmbus_irq == -1) 3055 hv_remove_vmbus_handler(); 3056 else 3057 free_percpu_irq(vmbus_irq, &vmbus_evt); 3058 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 3059 smpboot_unregister_percpu_thread(&vmbus_irq_threads); 3060 3061 for_each_online_cpu(cpu) { 3062 struct hv_per_cpu_context *hv_cpu 3063 = per_cpu_ptr(hv_context.cpu_context, cpu); 3064 3065 tasklet_kill(&hv_cpu->msg_dpc); 3066 } 3067 hv_debug_rm_all_dir(); 3068 3069 vmbus_free_channels(); 3070 kfree(vmbus_connection.channels); 3071 3072 /* 3073 * The vmbus panic notifier is always registered, hence we should 3074 * also unconditionally unregister it here as well. 3075 */ 3076 atomic_notifier_chain_unregister(&panic_notifier_list, 3077 &hyperv_panic_vmbus_unload_block); 3078 3079 bus_unregister(&hv_bus); 3080 3081 cpuhp_remove_state(hyperv_cpuhp_online); 3082 hv_synic_free(); 3083 platform_driver_unregister(&vmbus_platform_driver); 3084 } 3085 3086 3087 MODULE_LICENSE("GPL"); 3088 MODULE_DESCRIPTION("Microsoft Hyper-V VMBus Driver"); 3089 3090 subsys_initcall(hv_acpi_init); 3091 module_exit(vmbus_exit); 3092