1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2009, Microsoft Corporation. 4 * 5 * Authors: 6 * Haiyang Zhang <haiyangz@microsoft.com> 7 * Hank Janssen <hjanssen@microsoft.com> 8 */ 9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/interrupt.h> 13 #include <linux/sched.h> 14 #include <linux/wait.h> 15 #include <linux/mm.h> 16 #include <linux/slab.h> 17 #include <linux/list.h> 18 #include <linux/module.h> 19 #include <linux/completion.h> 20 #include <linux/delay.h> 21 #include <linux/cpu.h> 22 #include <linux/hyperv.h> 23 #include <linux/export.h> 24 #include <asm/mshyperv.h> 25 #include <linux/sched/isolation.h> 26 27 #include "hyperv_vmbus.h" 28 29 static void init_vp_index(struct vmbus_channel *channel); 30 31 const struct vmbus_device vmbus_devs[] = { 32 /* IDE */ 33 { .dev_type = HV_IDE, 34 HV_IDE_GUID, 35 .perf_device = true, 36 .allowed_in_isolated = false, 37 }, 38 39 /* SCSI */ 40 { .dev_type = HV_SCSI, 41 HV_SCSI_GUID, 42 .perf_device = true, 43 .allowed_in_isolated = true, 44 }, 45 46 /* Fibre Channel */ 47 { .dev_type = HV_FC, 48 HV_SYNTHFC_GUID, 49 .perf_device = true, 50 .allowed_in_isolated = false, 51 }, 52 53 /* Synthetic NIC */ 54 { .dev_type = HV_NIC, 55 HV_NIC_GUID, 56 .perf_device = true, 57 .allowed_in_isolated = true, 58 }, 59 60 /* Network Direct */ 61 { .dev_type = HV_ND, 62 HV_ND_GUID, 63 .perf_device = true, 64 .allowed_in_isolated = false, 65 }, 66 67 /* PCIE */ 68 { .dev_type = HV_PCIE, 69 HV_PCIE_GUID, 70 .perf_device = false, 71 .allowed_in_isolated = true, 72 }, 73 74 /* Synthetic Frame Buffer */ 75 { .dev_type = HV_FB, 76 HV_SYNTHVID_GUID, 77 .perf_device = false, 78 .allowed_in_isolated = false, 79 }, 80 81 /* Synthetic Keyboard */ 82 { .dev_type = HV_KBD, 83 HV_KBD_GUID, 84 .perf_device = false, 85 .allowed_in_isolated = false, 86 }, 87 88 /* Synthetic MOUSE */ 89 { .dev_type = HV_MOUSE, 90 HV_MOUSE_GUID, 91 .perf_device = false, 92 .allowed_in_isolated = false, 93 }, 94 95 /* KVP */ 96 { .dev_type = HV_KVP, 97 HV_KVP_GUID, 98 .perf_device = false, 99 .allowed_in_isolated = false, 100 }, 101 102 /* Time Synch */ 103 { .dev_type = HV_TS, 104 HV_TS_GUID, 105 .perf_device = false, 106 .allowed_in_isolated = true, 107 }, 108 109 /* Heartbeat */ 110 { .dev_type = HV_HB, 111 HV_HEART_BEAT_GUID, 112 .perf_device = false, 113 .allowed_in_isolated = true, 114 }, 115 116 /* Shutdown */ 117 { .dev_type = HV_SHUTDOWN, 118 HV_SHUTDOWN_GUID, 119 .perf_device = false, 120 .allowed_in_isolated = true, 121 }, 122 123 /* File copy */ 124 /* fcopy always uses 16KB ring buffer size and is working well for last many years */ 125 { .pref_ring_size = 0x4000, 126 .dev_type = HV_FCOPY, 127 HV_FCOPY_GUID, 128 .perf_device = false, 129 .allowed_in_isolated = false, 130 }, 131 132 /* Backup */ 133 { .dev_type = HV_BACKUP, 134 HV_VSS_GUID, 135 .perf_device = false, 136 .allowed_in_isolated = false, 137 }, 138 139 /* Dynamic Memory */ 140 { .dev_type = HV_DM, 141 HV_DM_GUID, 142 .perf_device = false, 143 .allowed_in_isolated = false, 144 }, 145 146 /* 147 * Unknown GUID 148 * 64 KB ring buffer + 4 KB header should be sufficient size for any Hyper-V device apart 149 * from HV_NIC and HV_SCSI. This case avoid the fallback for unknown devices to allocate 150 * much bigger (2 MB) of ring size. 151 */ 152 { .pref_ring_size = 0x11000, 153 .dev_type = HV_UNKNOWN, 154 .perf_device = false, 155 .allowed_in_isolated = false, 156 }, 157 }; 158 EXPORT_SYMBOL_GPL(vmbus_devs); 159 160 static const struct { 161 guid_t guid; 162 } vmbus_unsupported_devs[] = { 163 { HV_AVMA1_GUID }, 164 { HV_AVMA2_GUID }, 165 { HV_RDV_GUID }, 166 { HV_IMC_GUID }, 167 }; 168 169 /* 170 * The rescinded channel may be blocked waiting for a response from the host; 171 * take care of that. 172 */ 173 static void vmbus_rescind_cleanup(struct vmbus_channel *channel) 174 { 175 struct vmbus_channel_msginfo *msginfo; 176 unsigned long flags; 177 178 179 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 180 channel->rescind = true; 181 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 182 msglistentry) { 183 184 if (msginfo->waiting_channel == channel) { 185 complete(&msginfo->waitevent); 186 break; 187 } 188 } 189 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 190 } 191 192 static bool is_unsupported_vmbus_devs(const guid_t *guid) 193 { 194 int i; 195 196 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++) 197 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid)) 198 return true; 199 return false; 200 } 201 202 static u16 hv_get_dev_type(const struct vmbus_channel *channel) 203 { 204 const guid_t *guid = &channel->offermsg.offer.if_type; 205 u16 i; 206 207 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid)) 208 return HV_UNKNOWN; 209 210 for (i = HV_IDE; i < HV_UNKNOWN; i++) { 211 if (guid_equal(guid, &vmbus_devs[i].guid)) 212 return i; 213 } 214 pr_info("Unknown GUID: %pUl\n", guid); 215 return i; 216 } 217 218 /** 219 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message 220 * @icmsghdrp: Pointer to msg header structure 221 * @buf: Raw buffer channel data 222 * @buflen: Length of the raw buffer channel data. 223 * @fw_version: The framework versions we can support. 224 * @fw_vercnt: The size of @fw_version. 225 * @srv_version: The service versions we can support. 226 * @srv_vercnt: The size of @srv_version. 227 * @nego_fw_version: The selected framework version. 228 * @nego_srv_version: The selected service version. 229 * 230 * Note: Versions are given in decreasing order. 231 * 232 * Set up and fill in default negotiate response message. 233 * Mainly used by Hyper-V drivers. 234 */ 235 bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf, 236 u32 buflen, const int *fw_version, int fw_vercnt, 237 const int *srv_version, int srv_vercnt, 238 int *nego_fw_version, int *nego_srv_version) 239 { 240 int icframe_major, icframe_minor; 241 int icmsg_major, icmsg_minor; 242 int fw_major, fw_minor; 243 int srv_major, srv_minor; 244 int i, j; 245 bool found_match = false; 246 struct icmsg_negotiate *negop; 247 248 /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */ 249 if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) { 250 pr_err_ratelimited("Invalid icmsg negotiate\n"); 251 return false; 252 } 253 254 icmsghdrp->icmsgsize = 0x10; 255 negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR]; 256 257 icframe_major = negop->icframe_vercnt; 258 icframe_minor = 0; 259 260 icmsg_major = negop->icmsg_vercnt; 261 icmsg_minor = 0; 262 263 /* Validate negop packet */ 264 if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT || 265 icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT || 266 ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) { 267 pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n", 268 icframe_major, icmsg_major); 269 goto fw_error; 270 } 271 272 /* 273 * Select the framework version number we will 274 * support. 275 */ 276 277 for (i = 0; i < fw_vercnt; i++) { 278 fw_major = (fw_version[i] >> 16); 279 fw_minor = (fw_version[i] & 0xFFFF); 280 281 for (j = 0; j < negop->icframe_vercnt; j++) { 282 if ((negop->icversion_data[j].major == fw_major) && 283 (negop->icversion_data[j].minor == fw_minor)) { 284 icframe_major = negop->icversion_data[j].major; 285 icframe_minor = negop->icversion_data[j].minor; 286 found_match = true; 287 break; 288 } 289 } 290 291 if (found_match) 292 break; 293 } 294 295 if (!found_match) 296 goto fw_error; 297 298 found_match = false; 299 300 for (i = 0; i < srv_vercnt; i++) { 301 srv_major = (srv_version[i] >> 16); 302 srv_minor = (srv_version[i] & 0xFFFF); 303 304 for (j = negop->icframe_vercnt; 305 (j < negop->icframe_vercnt + negop->icmsg_vercnt); 306 j++) { 307 308 if ((negop->icversion_data[j].major == srv_major) && 309 (negop->icversion_data[j].minor == srv_minor)) { 310 311 icmsg_major = negop->icversion_data[j].major; 312 icmsg_minor = negop->icversion_data[j].minor; 313 found_match = true; 314 break; 315 } 316 } 317 318 if (found_match) 319 break; 320 } 321 322 /* 323 * Respond with the framework and service 324 * version numbers we can support. 325 */ 326 327 fw_error: 328 if (!found_match) { 329 negop->icframe_vercnt = 0; 330 negop->icmsg_vercnt = 0; 331 } else { 332 negop->icframe_vercnt = 1; 333 negop->icmsg_vercnt = 1; 334 } 335 336 if (nego_fw_version) 337 *nego_fw_version = (icframe_major << 16) | icframe_minor; 338 339 if (nego_srv_version) 340 *nego_srv_version = (icmsg_major << 16) | icmsg_minor; 341 342 negop->icversion_data[0].major = icframe_major; 343 negop->icversion_data[0].minor = icframe_minor; 344 negop->icversion_data[1].major = icmsg_major; 345 negop->icversion_data[1].minor = icmsg_minor; 346 return found_match; 347 } 348 EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp); 349 350 /* 351 * alloc_channel - Allocate and initialize a vmbus channel object 352 */ 353 static struct vmbus_channel *alloc_channel(void) 354 { 355 struct vmbus_channel *channel; 356 357 channel = kzalloc_obj(*channel, GFP_ATOMIC); 358 if (!channel) 359 return NULL; 360 361 spin_lock_init(&channel->sched_lock); 362 init_completion(&channel->rescind_event); 363 364 INIT_LIST_HEAD(&channel->sc_list); 365 366 tasklet_init(&channel->callback_event, 367 vmbus_on_event, (unsigned long)channel); 368 369 hv_ringbuffer_pre_init(channel); 370 371 return channel; 372 } 373 374 /* 375 * free_channel - Release the resources used by the vmbus channel object 376 */ 377 static void free_channel(struct vmbus_channel *channel) 378 { 379 tasklet_kill(&channel->callback_event); 380 vmbus_remove_channel_attr_group(channel); 381 382 kobject_put(&channel->kobj); 383 } 384 385 void vmbus_channel_map_relid(struct vmbus_channel *channel) 386 { 387 u32 new_relid = channel->offermsg.child_relid; 388 389 if (WARN_ON(new_relid >= MAX_CHANNEL_RELIDS)) 390 return; 391 392 /* 393 * This function is always called in the tasklet for the connect CPU. 394 * So updating the relid hiwater mark does not need to be atomic. 395 */ 396 if (new_relid > READ_ONCE(vmbus_connection.relid_hiwater)) 397 WRITE_ONCE(vmbus_connection.relid_hiwater, new_relid); 398 399 /* 400 * The mapping of the channel's relid is visible from the CPUs that 401 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will 402 * execute: 403 * 404 * (a) In the "normal (i.e., not resuming from hibernation)" path, 405 * the full barrier in virt_store_mb() guarantees that the store 406 * is propagated to all CPUs before the add_channel_work work 407 * is queued. In turn, add_channel_work is queued before the 408 * channel's ring buffer is allocated/initialized and the 409 * OPENCHANNEL message for the channel is sent in vmbus_open(). 410 * Hyper-V won't start sending the interrupts for the channel 411 * before the OPENCHANNEL message is acked. The memory barrier 412 * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures 413 * that vmbus_chan_sched() must find the channel's relid in 414 * recv_int_page before retrieving the channel pointer from the 415 * array of channels. 416 * 417 * (b) In the "resuming from hibernation" path, the virt_store_mb() 418 * guarantees that the store is propagated to all CPUs before 419 * the VMBus connection is marked as ready for the resume event 420 * (cf. check_ready_for_resume_event()). The interrupt handler 421 * of the VMBus driver and vmbus_chan_sched() can not run before 422 * vmbus_bus_resume() has completed execution (cf. resume_noirq). 423 */ 424 virt_store_mb(vmbus_connection.channels[new_relid], channel); 425 } 426 427 void vmbus_channel_unmap_relid(struct vmbus_channel *channel) 428 { 429 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS)) 430 return; 431 WRITE_ONCE( 432 vmbus_connection.channels[channel->offermsg.child_relid], 433 NULL); 434 } 435 436 static void vmbus_release_relid(u32 relid) 437 { 438 struct vmbus_channel_relid_released msg; 439 int ret; 440 441 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released)); 442 msg.child_relid = relid; 443 msg.header.msgtype = CHANNELMSG_RELID_RELEASED; 444 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released), 445 true); 446 447 trace_vmbus_release_relid(&msg, ret); 448 } 449 450 void hv_process_channel_removal(struct vmbus_channel *channel) 451 { 452 lockdep_assert_held(&vmbus_connection.channel_mutex); 453 BUG_ON(!channel->rescind); 454 455 /* 456 * hv_process_channel_removal() could find INVALID_RELID only for 457 * hv_sock channels. See the inline comments in vmbus_onoffer(). 458 */ 459 WARN_ON(channel->offermsg.child_relid == INVALID_RELID && 460 !is_hvsock_channel(channel)); 461 462 /* 463 * Upon suspend, an in-use hv_sock channel is removed from the array of 464 * channels and the relid is invalidated. After hibernation, when the 465 * user-space application destroys the channel, it's unnecessary and 466 * unsafe to remove the channel from the array of channels. See also 467 * the inline comments before the call of vmbus_release_relid() below. 468 */ 469 if (channel->offermsg.child_relid != INVALID_RELID) 470 vmbus_channel_unmap_relid(channel); 471 472 if (channel->primary_channel == NULL) 473 list_del(&channel->listentry); 474 else 475 list_del(&channel->sc_list); 476 477 /* 478 * If this is a "perf" channel, updates the hv_numa_map[] masks so that 479 * init_vp_index() can (re-)use the CPU. 480 */ 481 if (hv_is_perf_channel(channel)) 482 hv_clear_allocated_cpu(channel->target_cpu); 483 484 /* 485 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and 486 * the relid is invalidated; after hibernation, when the user-space app 487 * destroys the channel, the relid is INVALID_RELID, and in this case 488 * it's unnecessary and unsafe to release the old relid, since the same 489 * relid can refer to a completely different channel now. 490 */ 491 if (channel->offermsg.child_relid != INVALID_RELID) 492 vmbus_release_relid(channel->offermsg.child_relid); 493 494 free_channel(channel); 495 } 496 497 void vmbus_free_channels(void) 498 { 499 struct vmbus_channel *channel, *tmp; 500 501 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list, 502 listentry) { 503 /* hv_process_channel_removal() needs this */ 504 channel->rescind = true; 505 506 vmbus_device_unregister(channel->device_obj); 507 } 508 } 509 510 /* Note: the function can run concurrently for primary/sub channels. */ 511 static void vmbus_add_channel_work(struct work_struct *work) 512 { 513 struct vmbus_channel *newchannel = 514 container_of(work, struct vmbus_channel, add_channel_work); 515 struct vmbus_channel *primary_channel = newchannel->primary_channel; 516 int ret; 517 518 /* 519 * This state is used to indicate a successful open 520 * so that when we do close the channel normally, we 521 * can cleanup properly. 522 */ 523 newchannel->state = CHANNEL_OPEN_STATE; 524 525 if (primary_channel != NULL) { 526 /* newchannel is a sub-channel. */ 527 struct hv_device *dev = primary_channel->device_obj; 528 529 if (vmbus_add_channel_kobj(dev, newchannel)) 530 goto err_deq_chan; 531 532 if (primary_channel->sc_creation_callback != NULL) 533 primary_channel->sc_creation_callback(newchannel); 534 535 newchannel->probe_done = true; 536 return; 537 } 538 539 /* 540 * Start the process of binding the primary channel to the driver 541 */ 542 newchannel->device_obj = vmbus_device_create( 543 &newchannel->offermsg.offer.if_type, 544 &newchannel->offermsg.offer.if_instance, 545 newchannel); 546 if (!newchannel->device_obj) 547 goto err_deq_chan; 548 549 newchannel->device_obj->device_id = newchannel->device_id; 550 /* 551 * Add the new device to the bus. This will kick off device-driver 552 * binding which eventually invokes the device driver's AddDevice() 553 * method. 554 * 555 * If vmbus_device_register() fails, the 'device_obj' is freed in 556 * vmbus_device_release() as called by device_unregister() in the 557 * error path of vmbus_device_register(). In the outside error 558 * path, there's no need to free it. 559 */ 560 ret = vmbus_device_register(newchannel->device_obj); 561 562 if (ret != 0) { 563 pr_err("unable to add child device object (relid %d)\n", 564 newchannel->offermsg.child_relid); 565 goto err_deq_chan; 566 } 567 568 newchannel->probe_done = true; 569 return; 570 571 err_deq_chan: 572 mutex_lock(&vmbus_connection.channel_mutex); 573 574 /* 575 * We need to set the flag, otherwise 576 * vmbus_onoffer_rescind() can be blocked. 577 */ 578 newchannel->probe_done = true; 579 580 if (primary_channel == NULL) 581 list_del(&newchannel->listentry); 582 else 583 list_del(&newchannel->sc_list); 584 585 /* vmbus_process_offer() has mapped the channel. */ 586 vmbus_channel_unmap_relid(newchannel); 587 588 mutex_unlock(&vmbus_connection.channel_mutex); 589 590 vmbus_release_relid(newchannel->offermsg.child_relid); 591 592 free_channel(newchannel); 593 } 594 595 /* 596 * vmbus_process_offer - Process the offer by creating a channel/device 597 * associated with this offer 598 */ 599 static void vmbus_process_offer(struct vmbus_channel *newchannel) 600 { 601 struct vmbus_channel *channel; 602 struct workqueue_struct *wq; 603 bool fnew = true; 604 605 /* 606 * Synchronize vmbus_process_offer() and CPU hotplugging: 607 * 608 * CPU1 CPU2 609 * 610 * [vmbus_process_offer()] [Hot removal of the CPU] 611 * 612 * CPU_READ_LOCK CPUS_WRITE_LOCK 613 * LOAD cpu_online_mask SEARCH chn_list 614 * STORE target_cpu LOAD target_cpu 615 * INSERT chn_list STORE cpu_online_mask 616 * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK 617 * 618 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE && 619 * CPU2's SEARCH from *not* seeing CPU1's INSERT 620 * 621 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT && 622 * CPU2's LOAD from *not* seing CPU1's STORE 623 */ 624 cpus_read_lock(); 625 626 /* 627 * Serializes the modifications of the chn_list list as well as 628 * the accesses to next_numa_node_id in init_vp_index(). 629 */ 630 mutex_lock(&vmbus_connection.channel_mutex); 631 632 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { 633 if (guid_equal(&channel->offermsg.offer.if_type, 634 &newchannel->offermsg.offer.if_type) && 635 guid_equal(&channel->offermsg.offer.if_instance, 636 &newchannel->offermsg.offer.if_instance)) { 637 fnew = false; 638 newchannel->primary_channel = channel; 639 break; 640 } 641 } 642 643 init_vp_index(newchannel); 644 645 /* Remember the channels that should be cleaned up upon suspend. */ 646 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel)) 647 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend); 648 649 /* 650 * Now that we have acquired the channel_mutex, 651 * we can release the potentially racing rescind thread. 652 */ 653 atomic_dec(&vmbus_connection.offer_in_progress); 654 655 if (fnew) { 656 list_add_tail(&newchannel->listentry, 657 &vmbus_connection.chn_list); 658 } else { 659 /* 660 * Check to see if this is a valid sub-channel. 661 */ 662 if (newchannel->offermsg.offer.sub_channel_index == 0) { 663 mutex_unlock(&vmbus_connection.channel_mutex); 664 cpus_read_unlock(); 665 /* 666 * Don't call free_channel(), because newchannel->kobj 667 * is not initialized yet. 668 */ 669 kfree(newchannel); 670 WARN_ON_ONCE(1); 671 return; 672 } 673 /* 674 * Process the sub-channel. 675 */ 676 list_add_tail(&newchannel->sc_list, &channel->sc_list); 677 } 678 679 vmbus_channel_map_relid(newchannel); 680 681 mutex_unlock(&vmbus_connection.channel_mutex); 682 cpus_read_unlock(); 683 684 /* 685 * vmbus_process_offer() mustn't call channel->sc_creation_callback() 686 * directly for sub-channels, because sc_creation_callback() -> 687 * vmbus_open() may never get the host's response to the 688 * OPEN_CHANNEL message (the host may rescind a channel at any time, 689 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind() 690 * may not wake up the vmbus_open() as it's blocked due to a non-zero 691 * vmbus_connection.offer_in_progress, and finally we have a deadlock. 692 * 693 * The above is also true for primary channels, if the related device 694 * drivers use sync probing mode by default. 695 * 696 * And, usually the handling of primary channels and sub-channels can 697 * depend on each other, so we should offload them to different 698 * workqueues to avoid possible deadlock, e.g. in sync-probing mode, 699 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() -> 700 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock 701 * and waits for all the sub-channels to appear, but the latter 702 * can't get the rtnl_lock and this blocks the handling of 703 * sub-channels. 704 */ 705 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work); 706 wq = fnew ? vmbus_connection.handle_primary_chan_wq : 707 vmbus_connection.handle_sub_chan_wq; 708 queue_work(wq, &newchannel->add_channel_work); 709 } 710 711 /* 712 * Check if CPUs used by other channels of the same device. 713 * It should only be called by init_vp_index(). 714 */ 715 static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn) 716 { 717 struct vmbus_channel *primary = chn->primary_channel; 718 struct vmbus_channel *sc; 719 720 lockdep_assert_held(&vmbus_connection.channel_mutex); 721 722 if (!primary) 723 return false; 724 725 if (primary->target_cpu == cpu) 726 return true; 727 728 list_for_each_entry(sc, &primary->sc_list, sc_list) 729 if (sc != chn && sc->target_cpu == cpu) 730 return true; 731 732 return false; 733 } 734 735 /* 736 * We use this state to statically distribute the channel interrupt load. 737 */ 738 static int next_numa_node_id; 739 740 /* 741 * We can statically distribute the incoming channel interrupt load 742 * by binding a channel to VCPU. 743 * 744 * For non-performance critical channels we assign the VMBUS_CONNECT_CPU. 745 * Performance critical channels will be distributed evenly among all 746 * the available NUMA nodes. Once the node is assigned, we will assign 747 * the CPU based on a simple round robin scheme. 748 */ 749 static void init_vp_index(struct vmbus_channel *channel) 750 { 751 bool perf_chn = hv_is_perf_channel(channel); 752 u32 i, ncpu = num_online_cpus(); 753 cpumask_var_t available_mask; 754 struct cpumask *allocated_mask; 755 const struct cpumask *node_mask; 756 const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ); 757 u32 target_cpu; 758 int numa_node; 759 760 if (!perf_chn || 761 !alloc_cpumask_var(&available_mask, GFP_KERNEL) || 762 !cpumask_intersects(hk_mask, cpu_online_mask)) { 763 /* 764 * If the channel is not a performance critical 765 * channel, bind it to VMBUS_CONNECT_CPU. 766 * In case alloc_cpumask_var() fails, bind it to 767 * VMBUS_CONNECT_CPU. 768 * If all the cpus are isolated, bind it to 769 * VMBUS_CONNECT_CPU. 770 */ 771 channel->target_cpu = VMBUS_CONNECT_CPU; 772 if (perf_chn) 773 hv_set_allocated_cpu(VMBUS_CONNECT_CPU); 774 return; 775 } 776 777 for (i = 1; i <= ncpu + 1; i++) { 778 while (true) { 779 numa_node = next_numa_node_id++; 780 if (numa_node == nr_node_ids) { 781 next_numa_node_id = 0; 782 continue; 783 } 784 /* 785 * Try next NUMA node if current NUMA node has no CPU 786 * or doesn't contain any housekeeping CPU. 787 */ 788 node_mask = cpumask_of_node(numa_node); 789 if (!cpumask_intersects(node_mask, hk_mask)) 790 continue; 791 break; 792 } 793 allocated_mask = &hv_context.hv_numa_map[numa_node]; 794 795 retry: 796 cpumask_xor(available_mask, allocated_mask, node_mask); 797 cpumask_and(available_mask, available_mask, hk_mask); 798 799 if (cpumask_empty(available_mask)) { 800 /* 801 * We have cycled through all the CPUs in the node; 802 * reset the allocated map. 803 */ 804 cpumask_clear(allocated_mask); 805 goto retry; 806 } 807 808 target_cpu = cpumask_first(available_mask); 809 cpumask_set_cpu(target_cpu, allocated_mask); 810 811 if (channel->offermsg.offer.sub_channel_index >= ncpu || 812 i > ncpu || !hv_cpuself_used(target_cpu, channel)) 813 break; 814 } 815 816 channel->target_cpu = target_cpu; 817 818 free_cpumask_var(available_mask); 819 } 820 821 #define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */ 822 #define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */ 823 #define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS) 824 #define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */ 825 #define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS) 826 827 static void vmbus_wait_for_unload(void) 828 { 829 int cpu; 830 void *page_addr; 831 struct hv_message *msg; 832 struct vmbus_channel_message_header *hdr; 833 u32 message_type, i; 834 835 /* 836 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was 837 * used for initial contact or to CPU0 depending on host version. When 838 * we're crashing on a different CPU let's hope that IRQ handler on 839 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still 840 * functional and vmbus_unload_response() will complete 841 * vmbus_connection.unload_event. If not, the last thing we can do is 842 * read message pages for all CPUs directly. 843 * 844 * Wait up to 100 seconds since an Azure host must writeback any dirty 845 * data in its disk cache before the VMbus UNLOAD request will 846 * complete. This flushing has been empirically observed to take up 847 * to 50 seconds in cases with a lot of dirty data, so allow additional 848 * leeway and for inaccuracies in mdelay(). But eventually time out so 849 * that the panic path can't get hung forever in case the response 850 * message isn't seen. 851 */ 852 for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) { 853 if (completion_done(&vmbus_connection.unload_event)) 854 goto completed; 855 856 for_each_present_cpu(cpu) { 857 struct hv_per_cpu_context *hv_cpu 858 = per_cpu_ptr(hv_context.cpu_context, cpu); 859 860 /* 861 * In a CoCo VM the hyp_synic_message_page is not allocated 862 * in hv_synic_alloc(). Instead it is set/cleared in 863 * hv_hyp_synic_enable_regs() and hv_hyp_synic_disable_regs() 864 * such that it is set only when the CPU is online. If 865 * not all present CPUs are online, the message page 866 * might be NULL, so skip such CPUs. 867 */ 868 page_addr = hv_cpu->hyp_synic_message_page; 869 if (!page_addr) 870 continue; 871 872 msg = (struct hv_message *)page_addr 873 + VMBUS_MESSAGE_SINT; 874 875 message_type = READ_ONCE(msg->header.message_type); 876 if (message_type == HVMSG_NONE) 877 continue; 878 879 hdr = (struct vmbus_channel_message_header *) 880 msg->u.payload; 881 882 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE) 883 complete(&vmbus_connection.unload_event); 884 885 vmbus_signal_eom(msg, message_type); 886 } 887 888 /* 889 * Give a notice periodically so someone watching the 890 * serial output won't think it is completely hung. 891 */ 892 if (!(i % UNLOAD_MSG_LOOPS)) 893 pr_notice("Waiting for VMBus UNLOAD to complete\n"); 894 895 mdelay(UNLOAD_DELAY_UNIT_MS); 896 } 897 pr_err("Continuing even though VMBus UNLOAD did not complete\n"); 898 899 completed: 900 /* 901 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all 902 * maybe-pending messages on all CPUs to be able to receive new 903 * messages after we reconnect. 904 */ 905 for_each_present_cpu(cpu) { 906 struct hv_per_cpu_context *hv_cpu 907 = per_cpu_ptr(hv_context.cpu_context, cpu); 908 909 page_addr = hv_cpu->hyp_synic_message_page; 910 if (!page_addr) 911 continue; 912 913 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT; 914 msg->header.message_type = HVMSG_NONE; 915 } 916 } 917 918 /* 919 * vmbus_unload_response - Handler for the unload response. 920 */ 921 static void vmbus_unload_response(struct vmbus_channel_message_header *hdr) 922 { 923 /* 924 * This is a global event; just wakeup the waiting thread. 925 * Once we successfully unload, we can cleanup the monitor state. 926 * 927 * NB. A malicious or compromised Hyper-V could send a spurious 928 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call 929 * of the complete() below. Make sure that unload_event has been 930 * initialized by the time this complete() is executed. 931 */ 932 complete(&vmbus_connection.unload_event); 933 } 934 935 void vmbus_initiate_unload(bool crash) 936 { 937 struct vmbus_channel_message_header hdr; 938 enum vmbus_connect_state old_state; 939 940 old_state = xchg(&vmbus_connection.conn_state, DISCONNECTED); 941 if (old_state == DISCONNECTED || old_state == CONNECTING) 942 return; 943 944 reinit_completion(&vmbus_connection.unload_event); 945 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header)); 946 hdr.msgtype = CHANNELMSG_UNLOAD; 947 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header), 948 !crash); 949 950 /* 951 * vmbus_initiate_unload() is also called on crash and the crash can be 952 * happening in an interrupt context, where scheduling is impossible. 953 */ 954 if (!crash) 955 wait_for_completion(&vmbus_connection.unload_event); 956 else 957 vmbus_wait_for_unload(); 958 } 959 EXPORT_SYMBOL_GPL(vmbus_initiate_unload); 960 961 static void vmbus_setup_channel_state(struct vmbus_channel *channel, 962 struct vmbus_channel_offer_channel *offer) 963 { 964 /* 965 * Setup state for signalling the host. 966 */ 967 channel->sig_event = VMBUS_EVENT_CONNECTION_ID; 968 969 channel->is_dedicated_interrupt = 970 (offer->is_dedicated_interrupt != 0); 971 channel->sig_event = offer->connection_id; 972 973 memcpy(&channel->offermsg, offer, 974 sizeof(struct vmbus_channel_offer_channel)); 975 channel->monitor_grp = (u8)offer->monitorid / 32; 976 channel->monitor_bit = (u8)offer->monitorid % 32; 977 channel->device_id = hv_get_dev_type(channel); 978 } 979 980 /* 981 * find_primary_channel_by_offer - Get the channel object given the new offer. 982 * This is only used in the resume path of hibernation. 983 */ 984 static struct vmbus_channel * 985 find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer) 986 { 987 struct vmbus_channel *channel = NULL, *iter; 988 const guid_t *inst1, *inst2; 989 990 /* Ignore sub-channel offers. */ 991 if (offer->offer.sub_channel_index != 0) 992 return NULL; 993 994 mutex_lock(&vmbus_connection.channel_mutex); 995 996 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) { 997 inst1 = &iter->offermsg.offer.if_instance; 998 inst2 = &offer->offer.if_instance; 999 1000 if (guid_equal(inst1, inst2)) { 1001 channel = iter; 1002 break; 1003 } 1004 } 1005 1006 mutex_unlock(&vmbus_connection.channel_mutex); 1007 1008 return channel; 1009 } 1010 1011 static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer) 1012 { 1013 const guid_t *guid = &offer->offer.if_type; 1014 u16 i; 1015 1016 if (!hv_is_isolation_supported()) 1017 return true; 1018 1019 if (is_hvsock_offer(offer)) 1020 return true; 1021 1022 for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) { 1023 if (guid_equal(guid, &vmbus_devs[i].guid)) 1024 return vmbus_devs[i].allowed_in_isolated; 1025 } 1026 return false; 1027 } 1028 1029 /* 1030 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition. 1031 * 1032 */ 1033 static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) 1034 { 1035 struct vmbus_channel_offer_channel *offer; 1036 struct vmbus_channel *oldchannel, *newchannel; 1037 size_t offer_sz; 1038 bool co_ring_buffer, co_external_memory; 1039 1040 offer = (struct vmbus_channel_offer_channel *)hdr; 1041 1042 trace_vmbus_onoffer(offer); 1043 1044 if (!vmbus_is_valid_offer(offer)) { 1045 pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n", 1046 offer->child_relid); 1047 atomic_dec(&vmbus_connection.offer_in_progress); 1048 return; 1049 } 1050 1051 co_ring_buffer = is_co_ring_buffer(offer); 1052 co_external_memory = is_co_external_memory(offer); 1053 if (!co_ring_buffer && co_external_memory) { 1054 pr_err("Invalid offer relid=%d: the ring buffer isn't encrypted\n", 1055 offer->child_relid); 1056 return; 1057 } 1058 if (co_ring_buffer || co_external_memory) { 1059 if (vmbus_proto_version < VERSION_WIN10_V6_0 || !vmbus_is_confidential()) { 1060 pr_err("Invalid offer relid=%d: no support for confidential VMBus\n", 1061 offer->child_relid); 1062 atomic_dec(&vmbus_connection.offer_in_progress); 1063 return; 1064 } 1065 } 1066 1067 oldchannel = find_primary_channel_by_offer(offer); 1068 1069 if (oldchannel != NULL) { 1070 /* 1071 * We're resuming from hibernation: all the sub-channel and 1072 * hv_sock channels we had before the hibernation should have 1073 * been cleaned up, and now we must be seeing a re-offered 1074 * primary channel that we had before the hibernation. 1075 */ 1076 1077 /* 1078 * { Initially: channel relid = INVALID_RELID, 1079 * channels[valid_relid] = NULL } 1080 * 1081 * CPU1 CPU2 1082 * 1083 * [vmbus_onoffer()] [vmbus_device_release()] 1084 * 1085 * LOCK channel_mutex LOCK channel_mutex 1086 * STORE channel relid = valid_relid LOAD r1 = channel relid 1087 * MAP_RELID channel if (r1 != INVALID_RELID) 1088 * UNLOCK channel_mutex UNMAP_RELID channel 1089 * UNLOCK channel_mutex 1090 * 1091 * Forbids: r1 == valid_relid && 1092 * channels[valid_relid] == channel 1093 * 1094 * Note. r1 can be INVALID_RELID only for an hv_sock channel. 1095 * None of the hv_sock channels which were present before the 1096 * suspend are re-offered upon the resume. See the WARN_ON() 1097 * in hv_process_channel_removal(). 1098 */ 1099 mutex_lock(&vmbus_connection.channel_mutex); 1100 1101 atomic_dec(&vmbus_connection.offer_in_progress); 1102 1103 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID); 1104 /* Fix up the relid. */ 1105 oldchannel->offermsg.child_relid = offer->child_relid; 1106 1107 offer_sz = sizeof(*offer); 1108 if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) { 1109 /* 1110 * This is not an error, since the host can also change 1111 * the other field(s) of the offer, e.g. on WS RS5 1112 * (Build 17763), the offer->connection_id of the 1113 * Mellanox VF vmbus device can change when the host 1114 * reoffers the device upon resume. 1115 */ 1116 pr_debug("vmbus offer changed: relid=%d\n", 1117 offer->child_relid); 1118 1119 print_hex_dump_debug("Old vmbus offer: ", 1120 DUMP_PREFIX_OFFSET, 16, 4, 1121 &oldchannel->offermsg, offer_sz, 1122 false); 1123 print_hex_dump_debug("New vmbus offer: ", 1124 DUMP_PREFIX_OFFSET, 16, 4, 1125 offer, offer_sz, false); 1126 1127 /* Fix up the old channel. */ 1128 vmbus_setup_channel_state(oldchannel, offer); 1129 } 1130 1131 /* Add the channel back to the array of channels. */ 1132 vmbus_channel_map_relid(oldchannel); 1133 mutex_unlock(&vmbus_connection.channel_mutex); 1134 return; 1135 } 1136 1137 /* Allocate the channel object and save this offer. */ 1138 newchannel = alloc_channel(); 1139 if (!newchannel) { 1140 vmbus_release_relid(offer->child_relid); 1141 atomic_dec(&vmbus_connection.offer_in_progress); 1142 pr_err("Unable to allocate channel object\n"); 1143 return; 1144 } 1145 newchannel->co_ring_buffer = co_ring_buffer; 1146 newchannel->co_external_memory = co_external_memory; 1147 1148 vmbus_setup_channel_state(newchannel, offer); 1149 1150 vmbus_process_offer(newchannel); 1151 } 1152 1153 static void check_ready_for_suspend_event(void) 1154 { 1155 /* 1156 * If all the sub-channels or hv_sock channels have been cleaned up, 1157 * then it's safe to suspend. 1158 */ 1159 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend)) 1160 complete(&vmbus_connection.ready_for_suspend_event); 1161 } 1162 1163 /* 1164 * vmbus_onoffer_rescind - Rescind offer handler. 1165 * 1166 * We queue a work item to process this offer synchronously 1167 */ 1168 static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) 1169 { 1170 struct vmbus_channel_rescind_offer *rescind; 1171 struct vmbus_channel *channel; 1172 struct device *dev; 1173 bool clean_up_chan_for_suspend; 1174 1175 rescind = (struct vmbus_channel_rescind_offer *)hdr; 1176 1177 trace_vmbus_onoffer_rescind(rescind); 1178 1179 /* 1180 * The offer msg and the corresponding rescind msg 1181 * from the host are guranteed to be ordered - 1182 * offer comes in first and then the rescind. 1183 * Since we process these events in work elements, 1184 * and with preemption, we may end up processing 1185 * the events out of order. We rely on the synchronization 1186 * provided by offer_in_progress and by channel_mutex for 1187 * ordering these events: 1188 * 1189 * { Initially: offer_in_progress = 1 } 1190 * 1191 * CPU1 CPU2 1192 * 1193 * [vmbus_onoffer()] [vmbus_onoffer_rescind()] 1194 * 1195 * LOCK channel_mutex WAIT_ON offer_in_progress == 0 1196 * DECREMENT offer_in_progress LOCK channel_mutex 1197 * STORE channels[] LOAD channels[] 1198 * UNLOCK channel_mutex UNLOCK channel_mutex 1199 * 1200 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE 1201 */ 1202 1203 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) { 1204 /* 1205 * We wait here until any channel offer is currently 1206 * being processed. 1207 */ 1208 msleep(1); 1209 } 1210 1211 mutex_lock(&vmbus_connection.channel_mutex); 1212 channel = relid2channel(rescind->child_relid); 1213 if (channel != NULL) { 1214 /* 1215 * Guarantee that no other instance of vmbus_onoffer_rescind() 1216 * has got a reference to the channel object. Synchronize on 1217 * &vmbus_connection.channel_mutex. 1218 */ 1219 if (channel->rescind_ref) { 1220 mutex_unlock(&vmbus_connection.channel_mutex); 1221 return; 1222 } 1223 channel->rescind_ref = true; 1224 } 1225 mutex_unlock(&vmbus_connection.channel_mutex); 1226 1227 if (channel == NULL) { 1228 /* 1229 * We failed in processing the offer message; 1230 * we would have cleaned up the relid in that 1231 * failure path. 1232 */ 1233 return; 1234 } 1235 1236 clean_up_chan_for_suspend = is_hvsock_channel(channel) || 1237 is_sub_channel(channel); 1238 /* 1239 * Before setting channel->rescind in vmbus_rescind_cleanup(), we 1240 * should make sure the channel callback is not running any more. 1241 */ 1242 vmbus_reset_channel_cb(channel); 1243 1244 /* 1245 * Now wait for offer handling to complete. 1246 */ 1247 vmbus_rescind_cleanup(channel); 1248 while (READ_ONCE(channel->probe_done) == false) { 1249 /* 1250 * We wait here until any channel offer is currently 1251 * being processed. 1252 */ 1253 msleep(1); 1254 } 1255 1256 /* 1257 * At this point, the rescind handling can proceed safely. 1258 */ 1259 1260 if (channel->device_obj) { 1261 if (channel->chn_rescind_callback) { 1262 channel->chn_rescind_callback(channel); 1263 1264 if (clean_up_chan_for_suspend) 1265 check_ready_for_suspend_event(); 1266 1267 return; 1268 } 1269 /* 1270 * We will have to unregister this device from the 1271 * driver core. 1272 */ 1273 dev = get_device(&channel->device_obj->device); 1274 if (dev) { 1275 vmbus_device_unregister(channel->device_obj); 1276 put_device(dev); 1277 } 1278 } else if (channel->primary_channel != NULL) { 1279 /* 1280 * Sub-channel is being rescinded. Following is the channel 1281 * close sequence when initiated from the driveri (refer to 1282 * vmbus_close() for details): 1283 * 1. Close all sub-channels first 1284 * 2. Then close the primary channel. 1285 */ 1286 mutex_lock(&vmbus_connection.channel_mutex); 1287 if (channel->state == CHANNEL_OPEN_STATE) { 1288 /* 1289 * The channel is currently not open; 1290 * it is safe for us to cleanup the channel. 1291 */ 1292 hv_process_channel_removal(channel); 1293 } else { 1294 complete(&channel->rescind_event); 1295 } 1296 mutex_unlock(&vmbus_connection.channel_mutex); 1297 } 1298 1299 /* The "channel" may have been freed. Do not access it any longer. */ 1300 1301 if (clean_up_chan_for_suspend) 1302 check_ready_for_suspend_event(); 1303 } 1304 1305 void vmbus_hvsock_device_unregister(struct vmbus_channel *channel) 1306 { 1307 BUG_ON(!is_hvsock_channel(channel)); 1308 1309 /* We always get a rescind msg when a connection is closed. */ 1310 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind)) 1311 msleep(1); 1312 1313 vmbus_device_unregister(channel->device_obj); 1314 } 1315 EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister); 1316 1317 1318 /* 1319 * vmbus_onoffers_delivered - 1320 * The CHANNELMSG_ALLOFFERS_DELIVERED message arrives after all 1321 * boot-time offers are delivered. A boot-time offer is for the primary 1322 * channel for any virtual hardware configured in the VM at the time it boots. 1323 * Boot-time offers include offers for physical devices assigned to the VM 1324 * via Hyper-V's Discrete Device Assignment (DDA) functionality that are 1325 * handled as virtual PCI devices in Linux (e.g., NVMe devices and GPUs). 1326 * Boot-time offers do not include offers for VMBus sub-channels. Because 1327 * devices can be hot-added to the VM after it is booted, additional channel 1328 * offers that aren't boot-time offers can be received at any time after the 1329 * all-offers-delivered message. 1330 * 1331 * SR-IOV NIC Virtual Functions (VFs) assigned to a VM are not considered 1332 * to be assigned to the VM at boot-time, and offers for VFs may occur after 1333 * the all-offers-delivered message. VFs are optional accelerators to the 1334 * synthetic VMBus NIC and are effectively hot-added only after the VMBus 1335 * NIC channel is opened (once it knows the guest can support it, via the 1336 * sriov bit in the netvsc protocol). 1337 */ 1338 static void vmbus_onoffers_delivered( 1339 struct vmbus_channel_message_header *hdr) 1340 { 1341 complete(&vmbus_connection.all_offers_delivered_event); 1342 } 1343 1344 /* 1345 * vmbus_onopen_result - Open result handler. 1346 * 1347 * This is invoked when we received a response to our channel open request. 1348 * Find the matching request, copy the response and signal the requesting 1349 * thread. 1350 */ 1351 static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr) 1352 { 1353 struct vmbus_channel_open_result *result; 1354 struct vmbus_channel_msginfo *msginfo; 1355 struct vmbus_channel_message_header *requestheader; 1356 struct vmbus_channel_open_channel *openmsg; 1357 unsigned long flags; 1358 1359 result = (struct vmbus_channel_open_result *)hdr; 1360 1361 trace_vmbus_onopen_result(result); 1362 1363 /* 1364 * Find the open msg, copy the result and signal/unblock the wait event 1365 */ 1366 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1367 1368 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1369 msglistentry) { 1370 requestheader = 1371 (struct vmbus_channel_message_header *)msginfo->msg; 1372 1373 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) { 1374 openmsg = 1375 (struct vmbus_channel_open_channel *)msginfo->msg; 1376 if (openmsg->child_relid == result->child_relid && 1377 openmsg->openid == result->openid) { 1378 memcpy(&msginfo->response.open_result, 1379 result, 1380 sizeof( 1381 struct vmbus_channel_open_result)); 1382 complete(&msginfo->waitevent); 1383 break; 1384 } 1385 } 1386 } 1387 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1388 } 1389 1390 /* 1391 * vmbus_ongpadl_created - GPADL created handler. 1392 * 1393 * This is invoked when we received a response to our gpadl create request. 1394 * Find the matching request, copy the response and signal the requesting 1395 * thread. 1396 */ 1397 static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr) 1398 { 1399 struct vmbus_channel_gpadl_created *gpadlcreated; 1400 struct vmbus_channel_msginfo *msginfo; 1401 struct vmbus_channel_message_header *requestheader; 1402 struct vmbus_channel_gpadl_header *gpadlheader; 1403 unsigned long flags; 1404 1405 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr; 1406 1407 trace_vmbus_ongpadl_created(gpadlcreated); 1408 1409 /* 1410 * Find the establish msg, copy the result and signal/unblock the wait 1411 * event 1412 */ 1413 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1414 1415 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1416 msglistentry) { 1417 requestheader = 1418 (struct vmbus_channel_message_header *)msginfo->msg; 1419 1420 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) { 1421 gpadlheader = 1422 (struct vmbus_channel_gpadl_header *)requestheader; 1423 1424 if ((gpadlcreated->child_relid == 1425 gpadlheader->child_relid) && 1426 (gpadlcreated->gpadl == gpadlheader->gpadl)) { 1427 memcpy(&msginfo->response.gpadl_created, 1428 gpadlcreated, 1429 sizeof( 1430 struct vmbus_channel_gpadl_created)); 1431 complete(&msginfo->waitevent); 1432 break; 1433 } 1434 } 1435 } 1436 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1437 } 1438 1439 /* 1440 * vmbus_onmodifychannel_response - Modify Channel response handler. 1441 * 1442 * This is invoked when we received a response to our channel modify request. 1443 * Find the matching request, copy the response and signal the requesting thread. 1444 */ 1445 static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr) 1446 { 1447 struct vmbus_channel_modifychannel_response *response; 1448 struct vmbus_channel_msginfo *msginfo; 1449 unsigned long flags; 1450 1451 response = (struct vmbus_channel_modifychannel_response *)hdr; 1452 1453 trace_vmbus_onmodifychannel_response(response); 1454 1455 /* 1456 * Find the modify msg, copy the response and signal/unblock the wait event. 1457 */ 1458 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1459 1460 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) { 1461 struct vmbus_channel_message_header *responseheader = 1462 (struct vmbus_channel_message_header *)msginfo->msg; 1463 1464 if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) { 1465 struct vmbus_channel_modifychannel *modifymsg; 1466 1467 modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg; 1468 if (modifymsg->child_relid == response->child_relid) { 1469 memcpy(&msginfo->response.modify_response, response, 1470 sizeof(*response)); 1471 complete(&msginfo->waitevent); 1472 break; 1473 } 1474 } 1475 } 1476 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1477 } 1478 1479 /* 1480 * vmbus_ongpadl_torndown - GPADL torndown handler. 1481 * 1482 * This is invoked when we received a response to our gpadl teardown request. 1483 * Find the matching request, copy the response and signal the requesting 1484 * thread. 1485 */ 1486 static void vmbus_ongpadl_torndown( 1487 struct vmbus_channel_message_header *hdr) 1488 { 1489 struct vmbus_channel_gpadl_torndown *gpadl_torndown; 1490 struct vmbus_channel_msginfo *msginfo; 1491 struct vmbus_channel_message_header *requestheader; 1492 struct vmbus_channel_gpadl_teardown *gpadl_teardown; 1493 unsigned long flags; 1494 1495 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr; 1496 1497 trace_vmbus_ongpadl_torndown(gpadl_torndown); 1498 1499 /* 1500 * Find the open msg, copy the result and signal/unblock the wait event 1501 */ 1502 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1503 1504 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1505 msglistentry) { 1506 requestheader = 1507 (struct vmbus_channel_message_header *)msginfo->msg; 1508 1509 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) { 1510 gpadl_teardown = 1511 (struct vmbus_channel_gpadl_teardown *)requestheader; 1512 1513 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) { 1514 memcpy(&msginfo->response.gpadl_torndown, 1515 gpadl_torndown, 1516 sizeof( 1517 struct vmbus_channel_gpadl_torndown)); 1518 complete(&msginfo->waitevent); 1519 break; 1520 } 1521 } 1522 } 1523 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1524 } 1525 1526 /* 1527 * vmbus_onversion_response - Version response handler 1528 * 1529 * This is invoked when we received a response to our initiate contact request. 1530 * Find the matching request, copy the response and signal the requesting 1531 * thread. 1532 */ 1533 static void vmbus_onversion_response( 1534 struct vmbus_channel_message_header *hdr) 1535 { 1536 struct vmbus_channel_msginfo *msginfo; 1537 struct vmbus_channel_message_header *requestheader; 1538 struct vmbus_channel_version_response *version_response; 1539 unsigned long flags; 1540 1541 version_response = (struct vmbus_channel_version_response *)hdr; 1542 1543 trace_vmbus_onversion_response(version_response); 1544 1545 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1546 1547 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1548 msglistentry) { 1549 requestheader = 1550 (struct vmbus_channel_message_header *)msginfo->msg; 1551 1552 if (requestheader->msgtype == 1553 CHANNELMSG_INITIATE_CONTACT) { 1554 memcpy(&msginfo->response.version_response, 1555 version_response, 1556 sizeof(struct vmbus_channel_version_response)); 1557 complete(&msginfo->waitevent); 1558 } 1559 } 1560 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1561 } 1562 1563 /* Channel message dispatch table */ 1564 const struct vmbus_channel_message_table_entry 1565 channel_message_table[CHANNELMSG_COUNT] = { 1566 { CHANNELMSG_INVALID, 0, NULL, 0}, 1567 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer, 1568 sizeof(struct vmbus_channel_offer_channel)}, 1569 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind, 1570 sizeof(struct vmbus_channel_rescind_offer) }, 1571 { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0}, 1572 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0}, 1573 { CHANNELMSG_OPENCHANNEL, 0, NULL, 0}, 1574 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result, 1575 sizeof(struct vmbus_channel_open_result)}, 1576 { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0}, 1577 { CHANNELMSG_GPADL_HEADER, 0, NULL, 0}, 1578 { CHANNELMSG_GPADL_BODY, 0, NULL, 0}, 1579 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created, 1580 sizeof(struct vmbus_channel_gpadl_created)}, 1581 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0}, 1582 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown, 1583 sizeof(struct vmbus_channel_gpadl_torndown) }, 1584 { CHANNELMSG_RELID_RELEASED, 0, NULL, 0}, 1585 { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0}, 1586 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response, 1587 sizeof(struct vmbus_channel_version_response)}, 1588 { CHANNELMSG_UNLOAD, 0, NULL, 0}, 1589 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0}, 1590 { CHANNELMSG_18, 0, NULL, 0}, 1591 { CHANNELMSG_19, 0, NULL, 0}, 1592 { CHANNELMSG_20, 0, NULL, 0}, 1593 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0}, 1594 { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0}, 1595 { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0}, 1596 { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response, 1597 sizeof(struct vmbus_channel_modifychannel_response)}, 1598 }; 1599 1600 /* 1601 * vmbus_onmessage - Handler for channel protocol messages. 1602 * 1603 * This is invoked in the vmbus worker thread context. 1604 */ 1605 void vmbus_onmessage(struct vmbus_channel_message_header *hdr) 1606 { 1607 trace_vmbus_on_message(hdr); 1608 1609 /* 1610 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go 1611 * out of bound and the message_handler pointer can not be NULL. 1612 */ 1613 channel_message_table[hdr->msgtype].message_handler(hdr); 1614 } 1615 1616 /* 1617 * vmbus_request_offers - Send a request to get all our pending offers 1618 * and wait for all boot-time offers to arrive. 1619 */ 1620 int vmbus_request_offers(void) 1621 { 1622 struct vmbus_channel_message_header *msg; 1623 struct vmbus_channel_msginfo *msginfo; 1624 int ret; 1625 1626 msginfo = kzalloc(sizeof(*msginfo) + 1627 sizeof(struct vmbus_channel_message_header), 1628 GFP_KERNEL); 1629 if (!msginfo) 1630 return -ENOMEM; 1631 1632 msg = (struct vmbus_channel_message_header *)msginfo->msg; 1633 1634 msg->msgtype = CHANNELMSG_REQUESTOFFERS; 1635 1636 /* 1637 * This REQUESTOFFERS message will result in the host sending an all 1638 * offers delivered message after all the boot-time offers are sent. 1639 */ 1640 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header), 1641 true); 1642 1643 trace_vmbus_request_offers(ret); 1644 1645 if (ret != 0) { 1646 pr_err("Unable to request offers - %d\n", ret); 1647 1648 goto cleanup; 1649 } 1650 1651 /* 1652 * Wait for the host to send all boot-time offers. 1653 * Keeping it as a best-effort mechanism, where a warning is 1654 * printed if a timeout occurs, and execution is resumed. 1655 */ 1656 if (!wait_for_completion_timeout(&vmbus_connection.all_offers_delivered_event, 1657 secs_to_jiffies(60))) { 1658 pr_warn("timed out waiting for all boot-time offers to be delivered.\n"); 1659 } 1660 1661 /* 1662 * Flush handling of offer messages (which may initiate work on 1663 * other work queues). 1664 */ 1665 flush_workqueue(vmbus_connection.work_queue); 1666 1667 /* 1668 * Flush workqueue for processing the incoming offers. Subchannel 1669 * offers and their processing can happen later, so there is no need to 1670 * flush that workqueue here. 1671 */ 1672 flush_workqueue(vmbus_connection.handle_primary_chan_wq); 1673 1674 cleanup: 1675 kfree(msginfo); 1676 1677 return ret; 1678 } 1679 1680 void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel, 1681 void (*sc_cr_cb)(struct vmbus_channel *new_sc)) 1682 { 1683 primary_channel->sc_creation_callback = sc_cr_cb; 1684 } 1685 EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback); 1686 1687 void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel, 1688 void (*chn_rescind_cb)(struct vmbus_channel *)) 1689 { 1690 channel->chn_rescind_callback = chn_rescind_cb; 1691 } 1692 EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback); 1693