1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2009, Microsoft Corporation. 4 * 5 * Authors: 6 * Haiyang Zhang <haiyangz@microsoft.com> 7 * Hank Janssen <hjanssen@microsoft.com> 8 */ 9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/sched.h> 13 #include <linux/wait.h> 14 #include <linux/mm.h> 15 #include <linux/slab.h> 16 #include <linux/module.h> 17 #include <linux/hyperv.h> 18 #include <linux/uio.h> 19 #include <linux/interrupt.h> 20 #include <linux/set_memory.h> 21 #include <asm/page.h> 22 #include <asm/mshyperv.h> 23 24 #include "hyperv_vmbus.h" 25 26 /* 27 * hv_gpadl_size - Return the real size of a gpadl, the size that Hyper-V uses 28 * 29 * For BUFFER gpadl, Hyper-V uses the exact same size as the guest does. 30 * 31 * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the header 32 * (because of the alignment requirement), however, the hypervisor only 33 * uses the first HV_HYP_PAGE_SIZE as the header, therefore leaving a 34 * (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. And since there are two rings in a 35 * ringbuffer, the total size for a RING gpadl that Hyper-V uses is the 36 * total size that the guest uses minus twice of the gap size. 37 */ 38 static inline u32 hv_gpadl_size(enum hv_gpadl_type type, u32 size) 39 { 40 switch (type) { 41 case HV_GPADL_BUFFER: 42 return size; 43 case HV_GPADL_RING: 44 /* The size of a ringbuffer must be page-aligned */ 45 BUG_ON(size % PAGE_SIZE); 46 /* 47 * Two things to notice here: 48 * 1) We're processing two ring buffers as a unit 49 * 2) We're skipping any space larger than HV_HYP_PAGE_SIZE in 50 * the first guest-size page of each of the two ring buffers. 51 * So we effectively subtract out two guest-size pages, and add 52 * back two Hyper-V size pages. 53 */ 54 return size - 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE); 55 } 56 BUG(); 57 return 0; 58 } 59 60 /* 61 * hv_ring_gpadl_send_hvpgoffset - Calculate the send offset (in unit of 62 * HV_HYP_PAGE) in a ring gpadl based on the 63 * offset in the guest 64 * 65 * @offset: the offset (in bytes) where the send ringbuffer starts in the 66 * virtual address space of the guest 67 */ 68 static inline u32 hv_ring_gpadl_send_hvpgoffset(u32 offset) 69 { 70 71 /* 72 * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the 73 * header (because of the alignment requirement), however, the 74 * hypervisor only uses the first HV_HYP_PAGE_SIZE as the header, 75 * therefore leaving a (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. 76 * 77 * And to calculate the effective send offset in gpadl, we need to 78 * substract this gap. 79 */ 80 return (offset - (PAGE_SIZE - HV_HYP_PAGE_SIZE)) >> HV_HYP_PAGE_SHIFT; 81 } 82 83 /* 84 * hv_gpadl_hvpfn - Return the Hyper-V page PFN of the @i th Hyper-V page in 85 * the gpadl 86 * 87 * @type: the type of the gpadl 88 * @kbuffer: the pointer to the gpadl in the guest 89 * @size: the total size (in bytes) of the gpadl 90 * @send_offset: the offset (in bytes) where the send ringbuffer starts in the 91 * virtual address space of the guest 92 * @i: the index 93 */ 94 static inline u64 hv_gpadl_hvpfn(enum hv_gpadl_type type, void *kbuffer, 95 u32 size, u32 send_offset, int i) 96 { 97 int send_idx = hv_ring_gpadl_send_hvpgoffset(send_offset); 98 unsigned long delta = 0UL; 99 100 switch (type) { 101 case HV_GPADL_BUFFER: 102 break; 103 case HV_GPADL_RING: 104 if (i == 0) 105 delta = 0; 106 else if (i <= send_idx) 107 delta = PAGE_SIZE - HV_HYP_PAGE_SIZE; 108 else 109 delta = 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE); 110 break; 111 default: 112 BUG(); 113 break; 114 } 115 116 return virt_to_hvpfn(kbuffer + delta + (HV_HYP_PAGE_SIZE * i)); 117 } 118 119 /* 120 * vmbus_setevent- Trigger an event notification on the specified 121 * channel. 122 */ 123 void vmbus_setevent(struct vmbus_channel *channel) 124 { 125 struct hv_monitor_page *monitorpage; 126 127 trace_vmbus_setevent(channel); 128 129 /* 130 * For channels marked as in "low latency" mode 131 * bypass the monitor page mechanism. 132 */ 133 if (channel->offermsg.monitor_allocated && !channel->low_latency) { 134 vmbus_send_interrupt(channel->offermsg.child_relid); 135 136 /* Get the child to parent monitor page */ 137 monitorpage = vmbus_connection.monitor_pages[1]; 138 139 sync_set_bit(channel->monitor_bit, 140 (unsigned long *)&monitorpage->trigger_group 141 [channel->monitor_grp].pending); 142 143 } else { 144 vmbus_set_event(channel); 145 } 146 } 147 EXPORT_SYMBOL_GPL(vmbus_setevent); 148 149 /* vmbus_free_ring - drop mapping of ring buffer */ 150 void vmbus_free_ring(struct vmbus_channel *channel) 151 { 152 hv_ringbuffer_cleanup(&channel->outbound); 153 hv_ringbuffer_cleanup(&channel->inbound); 154 155 if (channel->ringbuffer_page) { 156 __free_pages(channel->ringbuffer_page, 157 get_order(channel->ringbuffer_pagecount 158 << PAGE_SHIFT)); 159 channel->ringbuffer_page = NULL; 160 } 161 } 162 EXPORT_SYMBOL_GPL(vmbus_free_ring); 163 164 /* vmbus_alloc_ring - allocate and map pages for ring buffer */ 165 int vmbus_alloc_ring(struct vmbus_channel *newchannel, 166 u32 send_size, u32 recv_size) 167 { 168 struct page *page; 169 int order; 170 171 if (send_size % PAGE_SIZE || recv_size % PAGE_SIZE) 172 return -EINVAL; 173 174 /* Allocate the ring buffer */ 175 order = get_order(send_size + recv_size); 176 page = alloc_pages_node(cpu_to_node(newchannel->target_cpu), 177 GFP_KERNEL|__GFP_ZERO, order); 178 179 if (!page) 180 page = alloc_pages(GFP_KERNEL|__GFP_ZERO, order); 181 182 if (!page) 183 return -ENOMEM; 184 185 newchannel->ringbuffer_page = page; 186 newchannel->ringbuffer_pagecount = (send_size + recv_size) >> PAGE_SHIFT; 187 newchannel->ringbuffer_send_offset = send_size >> PAGE_SHIFT; 188 189 return 0; 190 } 191 EXPORT_SYMBOL_GPL(vmbus_alloc_ring); 192 193 /* Used for Hyper-V Socket: a guest client's connect() to the host */ 194 int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id, 195 const guid_t *shv_host_servie_id) 196 { 197 struct vmbus_channel_tl_connect_request conn_msg; 198 int ret; 199 200 memset(&conn_msg, 0, sizeof(conn_msg)); 201 conn_msg.header.msgtype = CHANNELMSG_TL_CONNECT_REQUEST; 202 conn_msg.guest_endpoint_id = *shv_guest_servie_id; 203 conn_msg.host_service_id = *shv_host_servie_id; 204 205 ret = vmbus_post_msg(&conn_msg, sizeof(conn_msg), true); 206 207 trace_vmbus_send_tl_connect_request(&conn_msg, ret); 208 209 return ret; 210 } 211 EXPORT_SYMBOL_GPL(vmbus_send_tl_connect_request); 212 213 static int send_modifychannel_without_ack(struct vmbus_channel *channel, u32 target_vp) 214 { 215 struct vmbus_channel_modifychannel msg; 216 int ret; 217 218 memset(&msg, 0, sizeof(msg)); 219 msg.header.msgtype = CHANNELMSG_MODIFYCHANNEL; 220 msg.child_relid = channel->offermsg.child_relid; 221 msg.target_vp = target_vp; 222 223 ret = vmbus_post_msg(&msg, sizeof(msg), true); 224 trace_vmbus_send_modifychannel(&msg, ret); 225 226 return ret; 227 } 228 229 static int send_modifychannel_with_ack(struct vmbus_channel *channel, u32 target_vp) 230 { 231 struct vmbus_channel_modifychannel *msg; 232 struct vmbus_channel_msginfo *info; 233 unsigned long flags; 234 int ret; 235 236 info = kzalloc(sizeof(struct vmbus_channel_msginfo) + 237 sizeof(struct vmbus_channel_modifychannel), 238 GFP_KERNEL); 239 if (!info) 240 return -ENOMEM; 241 242 init_completion(&info->waitevent); 243 info->waiting_channel = channel; 244 245 msg = (struct vmbus_channel_modifychannel *)info->msg; 246 msg->header.msgtype = CHANNELMSG_MODIFYCHANNEL; 247 msg->child_relid = channel->offermsg.child_relid; 248 msg->target_vp = target_vp; 249 250 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 251 list_add_tail(&info->msglistentry, &vmbus_connection.chn_msg_list); 252 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 253 254 ret = vmbus_post_msg(msg, sizeof(*msg), true); 255 trace_vmbus_send_modifychannel(msg, ret); 256 if (ret != 0) { 257 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 258 list_del(&info->msglistentry); 259 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 260 goto free_info; 261 } 262 263 /* 264 * Release channel_mutex; otherwise, vmbus_onoffer_rescind() could block on 265 * the mutex and be unable to signal the completion. 266 * 267 * See the caller target_cpu_store() for information about the usage of the 268 * mutex. 269 */ 270 mutex_unlock(&vmbus_connection.channel_mutex); 271 wait_for_completion(&info->waitevent); 272 mutex_lock(&vmbus_connection.channel_mutex); 273 274 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 275 list_del(&info->msglistentry); 276 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 277 278 if (info->response.modify_response.status) 279 ret = -EAGAIN; 280 281 free_info: 282 kfree(info); 283 return ret; 284 } 285 286 /* 287 * Set/change the vCPU (@target_vp) the channel (@child_relid) will interrupt. 288 * 289 * CHANNELMSG_MODIFYCHANNEL messages are aynchronous. When VMbus version 5.3 290 * or later is negotiated, Hyper-V always sends an ACK in response to such a 291 * message. For VMbus version 5.2 and earlier, it never sends an ACK. With- 292 * out an ACK, we can not know when the host will stop interrupting the "old" 293 * vCPU and start interrupting the "new" vCPU for the given channel. 294 * 295 * The CHANNELMSG_MODIFYCHANNEL message type is supported since VMBus version 296 * VERSION_WIN10_V4_1. 297 */ 298 int vmbus_send_modifychannel(struct vmbus_channel *channel, u32 target_vp) 299 { 300 if (vmbus_proto_version >= VERSION_WIN10_V5_3) 301 return send_modifychannel_with_ack(channel, target_vp); 302 return send_modifychannel_without_ack(channel, target_vp); 303 } 304 EXPORT_SYMBOL_GPL(vmbus_send_modifychannel); 305 306 /* 307 * create_gpadl_header - Creates a gpadl for the specified buffer 308 */ 309 static int create_gpadl_header(enum hv_gpadl_type type, void *kbuffer, 310 u32 size, u32 send_offset, 311 struct vmbus_channel_msginfo **msginfo) 312 { 313 int i; 314 int pagecount; 315 struct vmbus_channel_gpadl_header *gpadl_header; 316 struct vmbus_channel_gpadl_body *gpadl_body; 317 struct vmbus_channel_msginfo *msgheader; 318 struct vmbus_channel_msginfo *msgbody = NULL; 319 u32 msgsize; 320 321 int pfnsum, pfncount, pfnleft, pfncurr, pfnsize; 322 323 pagecount = hv_gpadl_size(type, size) >> HV_HYP_PAGE_SHIFT; 324 325 /* do we need a gpadl body msg */ 326 pfnsize = MAX_SIZE_CHANNEL_MESSAGE - 327 sizeof(struct vmbus_channel_gpadl_header) - 328 sizeof(struct gpa_range); 329 pfncount = pfnsize / sizeof(u64); 330 331 if (pagecount > pfncount) { 332 /* we need a gpadl body */ 333 /* fill in the header */ 334 msgsize = sizeof(struct vmbus_channel_msginfo) + 335 sizeof(struct vmbus_channel_gpadl_header) + 336 sizeof(struct gpa_range) + pfncount * sizeof(u64); 337 msgheader = kzalloc(msgsize, GFP_KERNEL); 338 if (!msgheader) 339 goto nomem; 340 341 INIT_LIST_HEAD(&msgheader->submsglist); 342 msgheader->msgsize = msgsize; 343 344 gpadl_header = (struct vmbus_channel_gpadl_header *) 345 msgheader->msg; 346 gpadl_header->rangecount = 1; 347 gpadl_header->range_buflen = sizeof(struct gpa_range) + 348 pagecount * sizeof(u64); 349 gpadl_header->range[0].byte_offset = 0; 350 gpadl_header->range[0].byte_count = hv_gpadl_size(type, size); 351 for (i = 0; i < pfncount; i++) 352 gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn( 353 type, kbuffer, size, send_offset, i); 354 *msginfo = msgheader; 355 356 pfnsum = pfncount; 357 pfnleft = pagecount - pfncount; 358 359 /* how many pfns can we fit */ 360 pfnsize = MAX_SIZE_CHANNEL_MESSAGE - 361 sizeof(struct vmbus_channel_gpadl_body); 362 pfncount = pfnsize / sizeof(u64); 363 364 /* fill in the body */ 365 while (pfnleft) { 366 if (pfnleft > pfncount) 367 pfncurr = pfncount; 368 else 369 pfncurr = pfnleft; 370 371 msgsize = sizeof(struct vmbus_channel_msginfo) + 372 sizeof(struct vmbus_channel_gpadl_body) + 373 pfncurr * sizeof(u64); 374 msgbody = kzalloc(msgsize, GFP_KERNEL); 375 376 if (!msgbody) { 377 struct vmbus_channel_msginfo *pos = NULL; 378 struct vmbus_channel_msginfo *tmp = NULL; 379 /* 380 * Free up all the allocated messages. 381 */ 382 list_for_each_entry_safe(pos, tmp, 383 &msgheader->submsglist, 384 msglistentry) { 385 386 list_del(&pos->msglistentry); 387 kfree(pos); 388 } 389 390 goto nomem; 391 } 392 393 msgbody->msgsize = msgsize; 394 gpadl_body = 395 (struct vmbus_channel_gpadl_body *)msgbody->msg; 396 397 /* 398 * Gpadl is u32 and we are using a pointer which could 399 * be 64-bit 400 * This is governed by the guest/host protocol and 401 * so the hypervisor guarantees that this is ok. 402 */ 403 for (i = 0; i < pfncurr; i++) 404 gpadl_body->pfn[i] = hv_gpadl_hvpfn(type, 405 kbuffer, size, send_offset, pfnsum + i); 406 407 /* add to msg header */ 408 list_add_tail(&msgbody->msglistentry, 409 &msgheader->submsglist); 410 pfnsum += pfncurr; 411 pfnleft -= pfncurr; 412 } 413 } else { 414 /* everything fits in a header */ 415 msgsize = sizeof(struct vmbus_channel_msginfo) + 416 sizeof(struct vmbus_channel_gpadl_header) + 417 sizeof(struct gpa_range) + pagecount * sizeof(u64); 418 msgheader = kzalloc(msgsize, GFP_KERNEL); 419 if (msgheader == NULL) 420 goto nomem; 421 422 INIT_LIST_HEAD(&msgheader->submsglist); 423 msgheader->msgsize = msgsize; 424 425 gpadl_header = (struct vmbus_channel_gpadl_header *) 426 msgheader->msg; 427 gpadl_header->rangecount = 1; 428 gpadl_header->range_buflen = sizeof(struct gpa_range) + 429 pagecount * sizeof(u64); 430 gpadl_header->range[0].byte_offset = 0; 431 gpadl_header->range[0].byte_count = hv_gpadl_size(type, size); 432 for (i = 0; i < pagecount; i++) 433 gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn( 434 type, kbuffer, size, send_offset, i); 435 436 *msginfo = msgheader; 437 } 438 439 return 0; 440 nomem: 441 kfree(msgheader); 442 kfree(msgbody); 443 return -ENOMEM; 444 } 445 446 /* 447 * __vmbus_establish_gpadl - Establish a GPADL for a buffer or ringbuffer 448 * 449 * @channel: a channel 450 * @type: the type of the corresponding GPADL, only meaningful for the guest. 451 * @kbuffer: from kmalloc or vmalloc 452 * @size: page-size multiple 453 * @send_offset: the offset (in bytes) where the send ring buffer starts, 454 * should be 0 for BUFFER type gpadl 455 * @gpadl_handle: some funky thing 456 */ 457 static int __vmbus_establish_gpadl(struct vmbus_channel *channel, 458 enum hv_gpadl_type type, void *kbuffer, 459 u32 size, u32 send_offset, 460 struct vmbus_gpadl *gpadl) 461 { 462 struct vmbus_channel_gpadl_header *gpadlmsg; 463 struct vmbus_channel_gpadl_body *gpadl_body; 464 struct vmbus_channel_msginfo *msginfo = NULL; 465 struct vmbus_channel_msginfo *submsginfo, *tmp; 466 struct list_head *curr; 467 u32 next_gpadl_handle; 468 unsigned long flags; 469 int ret = 0; 470 471 next_gpadl_handle = 472 (atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1); 473 474 ret = create_gpadl_header(type, kbuffer, size, send_offset, &msginfo); 475 if (ret) 476 return ret; 477 478 ret = set_memory_decrypted((unsigned long)kbuffer, 479 PFN_UP(size)); 480 if (ret) { 481 dev_warn(&channel->device_obj->device, 482 "Failed to set host visibility for new GPADL %d.\n", 483 ret); 484 return ret; 485 } 486 487 init_completion(&msginfo->waitevent); 488 msginfo->waiting_channel = channel; 489 490 gpadlmsg = (struct vmbus_channel_gpadl_header *)msginfo->msg; 491 gpadlmsg->header.msgtype = CHANNELMSG_GPADL_HEADER; 492 gpadlmsg->child_relid = channel->offermsg.child_relid; 493 gpadlmsg->gpadl = next_gpadl_handle; 494 495 496 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 497 list_add_tail(&msginfo->msglistentry, 498 &vmbus_connection.chn_msg_list); 499 500 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 501 502 if (channel->rescind) { 503 ret = -ENODEV; 504 goto cleanup; 505 } 506 507 ret = vmbus_post_msg(gpadlmsg, msginfo->msgsize - 508 sizeof(*msginfo), true); 509 510 trace_vmbus_establish_gpadl_header(gpadlmsg, ret); 511 512 if (ret != 0) 513 goto cleanup; 514 515 list_for_each(curr, &msginfo->submsglist) { 516 submsginfo = (struct vmbus_channel_msginfo *)curr; 517 gpadl_body = 518 (struct vmbus_channel_gpadl_body *)submsginfo->msg; 519 520 gpadl_body->header.msgtype = 521 CHANNELMSG_GPADL_BODY; 522 gpadl_body->gpadl = next_gpadl_handle; 523 524 ret = vmbus_post_msg(gpadl_body, 525 submsginfo->msgsize - sizeof(*submsginfo), 526 true); 527 528 trace_vmbus_establish_gpadl_body(gpadl_body, ret); 529 530 if (ret != 0) 531 goto cleanup; 532 533 } 534 wait_for_completion(&msginfo->waitevent); 535 536 if (msginfo->response.gpadl_created.creation_status != 0) { 537 pr_err("Failed to establish GPADL: err = 0x%x\n", 538 msginfo->response.gpadl_created.creation_status); 539 540 ret = -EDQUOT; 541 goto cleanup; 542 } 543 544 if (channel->rescind) { 545 ret = -ENODEV; 546 goto cleanup; 547 } 548 549 /* At this point, we received the gpadl created msg */ 550 gpadl->gpadl_handle = gpadlmsg->gpadl; 551 gpadl->buffer = kbuffer; 552 gpadl->size = size; 553 554 555 cleanup: 556 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 557 list_del(&msginfo->msglistentry); 558 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 559 list_for_each_entry_safe(submsginfo, tmp, &msginfo->submsglist, 560 msglistentry) { 561 kfree(submsginfo); 562 } 563 564 kfree(msginfo); 565 566 if (ret) 567 set_memory_encrypted((unsigned long)kbuffer, 568 PFN_UP(size)); 569 570 return ret; 571 } 572 573 /* 574 * vmbus_establish_gpadl - Establish a GPADL for the specified buffer 575 * 576 * @channel: a channel 577 * @kbuffer: from kmalloc or vmalloc 578 * @size: page-size multiple 579 * @gpadl_handle: some funky thing 580 */ 581 int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer, 582 u32 size, struct vmbus_gpadl *gpadl) 583 { 584 return __vmbus_establish_gpadl(channel, HV_GPADL_BUFFER, kbuffer, size, 585 0U, gpadl); 586 } 587 EXPORT_SYMBOL_GPL(vmbus_establish_gpadl); 588 589 /** 590 * request_arr_init - Allocates memory for the requestor array. Each slot 591 * keeps track of the next available slot in the array. Initially, each 592 * slot points to the next one (as in a Linked List). The last slot 593 * does not point to anything, so its value is U64_MAX by default. 594 * @size The size of the array 595 */ 596 static u64 *request_arr_init(u32 size) 597 { 598 int i; 599 u64 *req_arr; 600 601 req_arr = kcalloc(size, sizeof(u64), GFP_KERNEL); 602 if (!req_arr) 603 return NULL; 604 605 for (i = 0; i < size - 1; i++) 606 req_arr[i] = i + 1; 607 608 /* Last slot (no more available slots) */ 609 req_arr[i] = U64_MAX; 610 611 return req_arr; 612 } 613 614 /* 615 * vmbus_alloc_requestor - Initializes @rqstor's fields. 616 * Index 0 is the first free slot 617 * @size: Size of the requestor array 618 */ 619 static int vmbus_alloc_requestor(struct vmbus_requestor *rqstor, u32 size) 620 { 621 u64 *rqst_arr; 622 unsigned long *bitmap; 623 624 rqst_arr = request_arr_init(size); 625 if (!rqst_arr) 626 return -ENOMEM; 627 628 bitmap = bitmap_zalloc(size, GFP_KERNEL); 629 if (!bitmap) { 630 kfree(rqst_arr); 631 return -ENOMEM; 632 } 633 634 rqstor->req_arr = rqst_arr; 635 rqstor->req_bitmap = bitmap; 636 rqstor->size = size; 637 rqstor->next_request_id = 0; 638 spin_lock_init(&rqstor->req_lock); 639 640 return 0; 641 } 642 643 /* 644 * vmbus_free_requestor - Frees memory allocated for @rqstor 645 * @rqstor: Pointer to the requestor struct 646 */ 647 static void vmbus_free_requestor(struct vmbus_requestor *rqstor) 648 { 649 kfree(rqstor->req_arr); 650 bitmap_free(rqstor->req_bitmap); 651 } 652 653 static int __vmbus_open(struct vmbus_channel *newchannel, 654 void *userdata, u32 userdatalen, 655 void (*onchannelcallback)(void *context), void *context) 656 { 657 struct vmbus_channel_open_channel *open_msg; 658 struct vmbus_channel_msginfo *open_info = NULL; 659 struct page *page = newchannel->ringbuffer_page; 660 u32 send_pages, recv_pages; 661 unsigned long flags; 662 int err; 663 664 if (userdatalen > MAX_USER_DEFINED_BYTES) 665 return -EINVAL; 666 667 send_pages = newchannel->ringbuffer_send_offset; 668 recv_pages = newchannel->ringbuffer_pagecount - send_pages; 669 670 if (newchannel->state != CHANNEL_OPEN_STATE) 671 return -EINVAL; 672 673 /* Create and init requestor */ 674 if (newchannel->rqstor_size) { 675 if (vmbus_alloc_requestor(&newchannel->requestor, newchannel->rqstor_size)) 676 return -ENOMEM; 677 } 678 679 newchannel->state = CHANNEL_OPENING_STATE; 680 newchannel->onchannel_callback = onchannelcallback; 681 newchannel->channel_callback_context = context; 682 683 if (!newchannel->max_pkt_size) 684 newchannel->max_pkt_size = VMBUS_DEFAULT_MAX_PKT_SIZE; 685 686 /* Establish the gpadl for the ring buffer */ 687 newchannel->ringbuffer_gpadlhandle.gpadl_handle = 0; 688 689 err = __vmbus_establish_gpadl(newchannel, HV_GPADL_RING, 690 page_address(newchannel->ringbuffer_page), 691 (send_pages + recv_pages) << PAGE_SHIFT, 692 newchannel->ringbuffer_send_offset << PAGE_SHIFT, 693 &newchannel->ringbuffer_gpadlhandle); 694 if (err) 695 goto error_clean_ring; 696 697 err = hv_ringbuffer_init(&newchannel->outbound, 698 page, send_pages, 0); 699 if (err) 700 goto error_free_gpadl; 701 702 err = hv_ringbuffer_init(&newchannel->inbound, &page[send_pages], 703 recv_pages, newchannel->max_pkt_size); 704 if (err) 705 goto error_free_gpadl; 706 707 /* Create and init the channel open message */ 708 open_info = kzalloc(sizeof(*open_info) + 709 sizeof(struct vmbus_channel_open_channel), 710 GFP_KERNEL); 711 if (!open_info) { 712 err = -ENOMEM; 713 goto error_free_gpadl; 714 } 715 716 init_completion(&open_info->waitevent); 717 open_info->waiting_channel = newchannel; 718 719 open_msg = (struct vmbus_channel_open_channel *)open_info->msg; 720 open_msg->header.msgtype = CHANNELMSG_OPENCHANNEL; 721 open_msg->openid = newchannel->offermsg.child_relid; 722 open_msg->child_relid = newchannel->offermsg.child_relid; 723 open_msg->ringbuffer_gpadlhandle 724 = newchannel->ringbuffer_gpadlhandle.gpadl_handle; 725 /* 726 * The unit of ->downstream_ringbuffer_pageoffset is HV_HYP_PAGE and 727 * the unit of ->ringbuffer_send_offset (i.e. send_pages) is PAGE, so 728 * here we calculate it into HV_HYP_PAGE. 729 */ 730 open_msg->downstream_ringbuffer_pageoffset = 731 hv_ring_gpadl_send_hvpgoffset(send_pages << PAGE_SHIFT); 732 open_msg->target_vp = hv_cpu_number_to_vp_number(newchannel->target_cpu); 733 734 if (userdatalen) 735 memcpy(open_msg->userdata, userdata, userdatalen); 736 737 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 738 list_add_tail(&open_info->msglistentry, 739 &vmbus_connection.chn_msg_list); 740 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 741 742 if (newchannel->rescind) { 743 err = -ENODEV; 744 goto error_clean_msglist; 745 } 746 747 err = vmbus_post_msg(open_msg, 748 sizeof(struct vmbus_channel_open_channel), true); 749 750 trace_vmbus_open(open_msg, err); 751 752 if (err != 0) 753 goto error_clean_msglist; 754 755 wait_for_completion(&open_info->waitevent); 756 757 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 758 list_del(&open_info->msglistentry); 759 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 760 761 if (newchannel->rescind) { 762 err = -ENODEV; 763 goto error_free_info; 764 } 765 766 if (open_info->response.open_result.status) { 767 err = -EAGAIN; 768 goto error_free_info; 769 } 770 771 newchannel->state = CHANNEL_OPENED_STATE; 772 kfree(open_info); 773 return 0; 774 775 error_clean_msglist: 776 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 777 list_del(&open_info->msglistentry); 778 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 779 error_free_info: 780 kfree(open_info); 781 error_free_gpadl: 782 vmbus_teardown_gpadl(newchannel, &newchannel->ringbuffer_gpadlhandle); 783 error_clean_ring: 784 hv_ringbuffer_cleanup(&newchannel->outbound); 785 hv_ringbuffer_cleanup(&newchannel->inbound); 786 vmbus_free_requestor(&newchannel->requestor); 787 newchannel->state = CHANNEL_OPEN_STATE; 788 return err; 789 } 790 791 /* 792 * vmbus_connect_ring - Open the channel but reuse ring buffer 793 */ 794 int vmbus_connect_ring(struct vmbus_channel *newchannel, 795 void (*onchannelcallback)(void *context), void *context) 796 { 797 return __vmbus_open(newchannel, NULL, 0, onchannelcallback, context); 798 } 799 EXPORT_SYMBOL_GPL(vmbus_connect_ring); 800 801 /* 802 * vmbus_open - Open the specified channel. 803 */ 804 int vmbus_open(struct vmbus_channel *newchannel, 805 u32 send_ringbuffer_size, u32 recv_ringbuffer_size, 806 void *userdata, u32 userdatalen, 807 void (*onchannelcallback)(void *context), void *context) 808 { 809 int err; 810 811 err = vmbus_alloc_ring(newchannel, send_ringbuffer_size, 812 recv_ringbuffer_size); 813 if (err) 814 return err; 815 816 err = __vmbus_open(newchannel, userdata, userdatalen, 817 onchannelcallback, context); 818 if (err) 819 vmbus_free_ring(newchannel); 820 821 return err; 822 } 823 EXPORT_SYMBOL_GPL(vmbus_open); 824 825 /* 826 * vmbus_teardown_gpadl -Teardown the specified GPADL handle 827 */ 828 int vmbus_teardown_gpadl(struct vmbus_channel *channel, struct vmbus_gpadl *gpadl) 829 { 830 struct vmbus_channel_gpadl_teardown *msg; 831 struct vmbus_channel_msginfo *info; 832 unsigned long flags; 833 int ret; 834 835 info = kzalloc(sizeof(*info) + 836 sizeof(struct vmbus_channel_gpadl_teardown), GFP_KERNEL); 837 if (!info) 838 return -ENOMEM; 839 840 init_completion(&info->waitevent); 841 info->waiting_channel = channel; 842 843 msg = (struct vmbus_channel_gpadl_teardown *)info->msg; 844 845 msg->header.msgtype = CHANNELMSG_GPADL_TEARDOWN; 846 msg->child_relid = channel->offermsg.child_relid; 847 msg->gpadl = gpadl->gpadl_handle; 848 849 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 850 list_add_tail(&info->msglistentry, 851 &vmbus_connection.chn_msg_list); 852 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 853 854 if (channel->rescind) 855 goto post_msg_err; 856 857 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_gpadl_teardown), 858 true); 859 860 trace_vmbus_teardown_gpadl(msg, ret); 861 862 if (ret) 863 goto post_msg_err; 864 865 wait_for_completion(&info->waitevent); 866 867 gpadl->gpadl_handle = 0; 868 869 post_msg_err: 870 /* 871 * If the channel has been rescinded; 872 * we will be awakened by the rescind 873 * handler; set the error code to zero so we don't leak memory. 874 */ 875 if (channel->rescind) 876 ret = 0; 877 878 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 879 list_del(&info->msglistentry); 880 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 881 882 kfree(info); 883 884 ret = set_memory_encrypted((unsigned long)gpadl->buffer, 885 PFN_UP(gpadl->size)); 886 if (ret) 887 pr_warn("Fail to set mem host visibility in GPADL teardown %d.\n", ret); 888 889 return ret; 890 } 891 EXPORT_SYMBOL_GPL(vmbus_teardown_gpadl); 892 893 void vmbus_reset_channel_cb(struct vmbus_channel *channel) 894 { 895 unsigned long flags; 896 897 /* 898 * vmbus_on_event(), running in the per-channel tasklet, can race 899 * with vmbus_close_internal() in the case of SMP guest, e.g., when 900 * the former is accessing channel->inbound.ring_buffer, the latter 901 * could be freeing the ring_buffer pages, so here we must stop it 902 * first. 903 * 904 * vmbus_chan_sched() might call the netvsc driver callback function 905 * that ends up scheduling NAPI work that accesses the ring buffer. 906 * At this point, we have to ensure that any such work is completed 907 * and that the channel ring buffer is no longer being accessed, cf. 908 * the calls to napi_disable() in netvsc_device_remove(). 909 */ 910 tasklet_disable(&channel->callback_event); 911 912 /* See the inline comments in vmbus_chan_sched(). */ 913 spin_lock_irqsave(&channel->sched_lock, flags); 914 channel->onchannel_callback = NULL; 915 spin_unlock_irqrestore(&channel->sched_lock, flags); 916 917 channel->sc_creation_callback = NULL; 918 919 /* Re-enable tasklet for use on re-open */ 920 tasklet_enable(&channel->callback_event); 921 } 922 923 static int vmbus_close_internal(struct vmbus_channel *channel) 924 { 925 struct vmbus_channel_close_channel *msg; 926 int ret; 927 928 vmbus_reset_channel_cb(channel); 929 930 /* 931 * In case a device driver's probe() fails (e.g., 932 * util_probe() -> vmbus_open() returns -ENOMEM) and the device is 933 * rescinded later (e.g., we dynamically disable an Integrated Service 934 * in Hyper-V Manager), the driver's remove() invokes vmbus_close(): 935 * here we should skip most of the below cleanup work. 936 */ 937 if (channel->state != CHANNEL_OPENED_STATE) 938 return -EINVAL; 939 940 channel->state = CHANNEL_OPEN_STATE; 941 942 /* Send a closing message */ 943 944 msg = &channel->close_msg.msg; 945 946 msg->header.msgtype = CHANNELMSG_CLOSECHANNEL; 947 msg->child_relid = channel->offermsg.child_relid; 948 949 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_close_channel), 950 true); 951 952 trace_vmbus_close_internal(msg, ret); 953 954 if (ret) { 955 pr_err("Close failed: close post msg return is %d\n", ret); 956 /* 957 * If we failed to post the close msg, 958 * it is perhaps better to leak memory. 959 */ 960 } 961 962 /* Tear down the gpadl for the channel's ring buffer */ 963 else if (channel->ringbuffer_gpadlhandle.gpadl_handle) { 964 ret = vmbus_teardown_gpadl(channel, &channel->ringbuffer_gpadlhandle); 965 if (ret) { 966 pr_err("Close failed: teardown gpadl return %d\n", ret); 967 /* 968 * If we failed to teardown gpadl, 969 * it is perhaps better to leak memory. 970 */ 971 } 972 } 973 974 if (!ret) 975 vmbus_free_requestor(&channel->requestor); 976 977 return ret; 978 } 979 980 /* disconnect ring - close all channels */ 981 int vmbus_disconnect_ring(struct vmbus_channel *channel) 982 { 983 struct vmbus_channel *cur_channel, *tmp; 984 int ret; 985 986 if (channel->primary_channel != NULL) 987 return -EINVAL; 988 989 list_for_each_entry_safe(cur_channel, tmp, &channel->sc_list, sc_list) { 990 if (cur_channel->rescind) 991 wait_for_completion(&cur_channel->rescind_event); 992 993 mutex_lock(&vmbus_connection.channel_mutex); 994 if (vmbus_close_internal(cur_channel) == 0) { 995 vmbus_free_ring(cur_channel); 996 997 if (cur_channel->rescind) 998 hv_process_channel_removal(cur_channel); 999 } 1000 mutex_unlock(&vmbus_connection.channel_mutex); 1001 } 1002 1003 /* 1004 * Now close the primary. 1005 */ 1006 mutex_lock(&vmbus_connection.channel_mutex); 1007 ret = vmbus_close_internal(channel); 1008 mutex_unlock(&vmbus_connection.channel_mutex); 1009 1010 return ret; 1011 } 1012 EXPORT_SYMBOL_GPL(vmbus_disconnect_ring); 1013 1014 /* 1015 * vmbus_close - Close the specified channel 1016 */ 1017 void vmbus_close(struct vmbus_channel *channel) 1018 { 1019 if (vmbus_disconnect_ring(channel) == 0) 1020 vmbus_free_ring(channel); 1021 } 1022 EXPORT_SYMBOL_GPL(vmbus_close); 1023 1024 /** 1025 * vmbus_sendpacket_getid() - Send the specified buffer on the given channel 1026 * @channel: Pointer to vmbus_channel structure 1027 * @buffer: Pointer to the buffer you want to send the data from. 1028 * @bufferlen: Maximum size of what the buffer holds. 1029 * @requestid: Identifier of the request 1030 * @trans_id: Identifier of the transaction associated to this request, if 1031 * the send is successful; undefined, otherwise. 1032 * @type: Type of packet that is being sent e.g. negotiate, time 1033 * packet etc. 1034 * @flags: 0 or VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED 1035 * 1036 * Sends data in @buffer directly to Hyper-V via the vmbus. 1037 * This will send the data unparsed to Hyper-V. 1038 * 1039 * Mainly used by Hyper-V drivers. 1040 */ 1041 int vmbus_sendpacket_getid(struct vmbus_channel *channel, void *buffer, 1042 u32 bufferlen, u64 requestid, u64 *trans_id, 1043 enum vmbus_packet_type type, u32 flags) 1044 { 1045 struct vmpacket_descriptor desc; 1046 u32 packetlen = sizeof(struct vmpacket_descriptor) + bufferlen; 1047 u32 packetlen_aligned = ALIGN(packetlen, sizeof(u64)); 1048 struct kvec bufferlist[3]; 1049 u64 aligned_data = 0; 1050 int num_vecs = ((bufferlen != 0) ? 3 : 1); 1051 1052 1053 /* Setup the descriptor */ 1054 desc.type = type; /* VmbusPacketTypeDataInBand; */ 1055 desc.flags = flags; /* VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; */ 1056 /* in 8-bytes granularity */ 1057 desc.offset8 = sizeof(struct vmpacket_descriptor) >> 3; 1058 desc.len8 = (u16)(packetlen_aligned >> 3); 1059 desc.trans_id = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */ 1060 1061 bufferlist[0].iov_base = &desc; 1062 bufferlist[0].iov_len = sizeof(struct vmpacket_descriptor); 1063 bufferlist[1].iov_base = buffer; 1064 bufferlist[1].iov_len = bufferlen; 1065 bufferlist[2].iov_base = &aligned_data; 1066 bufferlist[2].iov_len = (packetlen_aligned - packetlen); 1067 1068 return hv_ringbuffer_write(channel, bufferlist, num_vecs, requestid, trans_id); 1069 } 1070 EXPORT_SYMBOL(vmbus_sendpacket_getid); 1071 1072 /** 1073 * vmbus_sendpacket() - Send the specified buffer on the given channel 1074 * @channel: Pointer to vmbus_channel structure 1075 * @buffer: Pointer to the buffer you want to send the data from. 1076 * @bufferlen: Maximum size of what the buffer holds. 1077 * @requestid: Identifier of the request 1078 * @type: Type of packet that is being sent e.g. negotiate, time 1079 * packet etc. 1080 * @flags: 0 or VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED 1081 * 1082 * Sends data in @buffer directly to Hyper-V via the vmbus. 1083 * This will send the data unparsed to Hyper-V. 1084 * 1085 * Mainly used by Hyper-V drivers. 1086 */ 1087 int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer, 1088 u32 bufferlen, u64 requestid, 1089 enum vmbus_packet_type type, u32 flags) 1090 { 1091 return vmbus_sendpacket_getid(channel, buffer, bufferlen, 1092 requestid, NULL, type, flags); 1093 } 1094 EXPORT_SYMBOL(vmbus_sendpacket); 1095 1096 /* 1097 * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer 1098 * packets using a GPADL Direct packet type. This interface allows you 1099 * to control notifying the host. This will be useful for sending 1100 * batched data. Also the sender can control the send flags 1101 * explicitly. 1102 */ 1103 int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel, 1104 struct hv_page_buffer pagebuffers[], 1105 u32 pagecount, void *buffer, u32 bufferlen, 1106 u64 requestid) 1107 { 1108 int i; 1109 struct vmbus_channel_packet_page_buffer desc; 1110 u32 descsize; 1111 u32 packetlen; 1112 u32 packetlen_aligned; 1113 struct kvec bufferlist[3]; 1114 u64 aligned_data = 0; 1115 1116 if (pagecount > MAX_PAGE_BUFFER_COUNT) 1117 return -EINVAL; 1118 1119 /* 1120 * Adjust the size down since vmbus_channel_packet_page_buffer is the 1121 * largest size we support 1122 */ 1123 descsize = sizeof(struct vmbus_channel_packet_page_buffer) - 1124 ((MAX_PAGE_BUFFER_COUNT - pagecount) * 1125 sizeof(struct hv_page_buffer)); 1126 packetlen = descsize + bufferlen; 1127 packetlen_aligned = ALIGN(packetlen, sizeof(u64)); 1128 1129 /* Setup the descriptor */ 1130 desc.type = VM_PKT_DATA_USING_GPA_DIRECT; 1131 desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; 1132 desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */ 1133 desc.length8 = (u16)(packetlen_aligned >> 3); 1134 desc.transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */ 1135 desc.reserved = 0; 1136 desc.rangecount = pagecount; 1137 1138 for (i = 0; i < pagecount; i++) { 1139 desc.range[i].len = pagebuffers[i].len; 1140 desc.range[i].offset = pagebuffers[i].offset; 1141 desc.range[i].pfn = pagebuffers[i].pfn; 1142 } 1143 1144 bufferlist[0].iov_base = &desc; 1145 bufferlist[0].iov_len = descsize; 1146 bufferlist[1].iov_base = buffer; 1147 bufferlist[1].iov_len = bufferlen; 1148 bufferlist[2].iov_base = &aligned_data; 1149 bufferlist[2].iov_len = (packetlen_aligned - packetlen); 1150 1151 return hv_ringbuffer_write(channel, bufferlist, 3, requestid, NULL); 1152 } 1153 EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer); 1154 1155 /* 1156 * vmbus_sendpacket_multipagebuffer - Send a multi-page buffer packet 1157 * using a GPADL Direct packet type. 1158 * The buffer includes the vmbus descriptor. 1159 */ 1160 int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel, 1161 struct vmbus_packet_mpb_array *desc, 1162 u32 desc_size, 1163 void *buffer, u32 bufferlen, u64 requestid) 1164 { 1165 u32 packetlen; 1166 u32 packetlen_aligned; 1167 struct kvec bufferlist[3]; 1168 u64 aligned_data = 0; 1169 1170 packetlen = desc_size + bufferlen; 1171 packetlen_aligned = ALIGN(packetlen, sizeof(u64)); 1172 1173 /* Setup the descriptor */ 1174 desc->type = VM_PKT_DATA_USING_GPA_DIRECT; 1175 desc->flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; 1176 desc->dataoffset8 = desc_size >> 3; /* in 8-bytes granularity */ 1177 desc->length8 = (u16)(packetlen_aligned >> 3); 1178 desc->transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */ 1179 desc->reserved = 0; 1180 desc->rangecount = 1; 1181 1182 bufferlist[0].iov_base = desc; 1183 bufferlist[0].iov_len = desc_size; 1184 bufferlist[1].iov_base = buffer; 1185 bufferlist[1].iov_len = bufferlen; 1186 bufferlist[2].iov_base = &aligned_data; 1187 bufferlist[2].iov_len = (packetlen_aligned - packetlen); 1188 1189 return hv_ringbuffer_write(channel, bufferlist, 3, requestid, NULL); 1190 } 1191 EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc); 1192 1193 /** 1194 * __vmbus_recvpacket() - Retrieve the user packet on the specified channel 1195 * @channel: Pointer to vmbus_channel structure 1196 * @buffer: Pointer to the buffer you want to receive the data into. 1197 * @bufferlen: Maximum size of what the buffer can hold. 1198 * @buffer_actual_len: The actual size of the data after it was received. 1199 * @requestid: Identifier of the request 1200 * @raw: true means keep the vmpacket_descriptor header in the received data. 1201 * 1202 * Receives directly from the hyper-v vmbus and puts the data it received 1203 * into Buffer. This will receive the data unparsed from hyper-v. 1204 * 1205 * Mainly used by Hyper-V drivers. 1206 */ 1207 static inline int 1208 __vmbus_recvpacket(struct vmbus_channel *channel, void *buffer, 1209 u32 bufferlen, u32 *buffer_actual_len, u64 *requestid, 1210 bool raw) 1211 { 1212 return hv_ringbuffer_read(channel, buffer, bufferlen, 1213 buffer_actual_len, requestid, raw); 1214 1215 } 1216 1217 int vmbus_recvpacket(struct vmbus_channel *channel, void *buffer, 1218 u32 bufferlen, u32 *buffer_actual_len, 1219 u64 *requestid) 1220 { 1221 return __vmbus_recvpacket(channel, buffer, bufferlen, 1222 buffer_actual_len, requestid, false); 1223 } 1224 EXPORT_SYMBOL(vmbus_recvpacket); 1225 1226 /* 1227 * vmbus_recvpacket_raw - Retrieve the raw packet on the specified channel 1228 */ 1229 int vmbus_recvpacket_raw(struct vmbus_channel *channel, void *buffer, 1230 u32 bufferlen, u32 *buffer_actual_len, 1231 u64 *requestid) 1232 { 1233 return __vmbus_recvpacket(channel, buffer, bufferlen, 1234 buffer_actual_len, requestid, true); 1235 } 1236 EXPORT_SYMBOL_GPL(vmbus_recvpacket_raw); 1237 1238 /* 1239 * vmbus_next_request_id - Returns a new request id. It is also 1240 * the index at which the guest memory address is stored. 1241 * Uses a spin lock to avoid race conditions. 1242 * @channel: Pointer to the VMbus channel struct 1243 * @rqst_add: Guest memory address to be stored in the array 1244 */ 1245 u64 vmbus_next_request_id(struct vmbus_channel *channel, u64 rqst_addr) 1246 { 1247 struct vmbus_requestor *rqstor = &channel->requestor; 1248 unsigned long flags; 1249 u64 current_id; 1250 1251 /* Check rqstor has been initialized */ 1252 if (!channel->rqstor_size) 1253 return VMBUS_NO_RQSTOR; 1254 1255 lock_requestor(channel, flags); 1256 current_id = rqstor->next_request_id; 1257 1258 /* Requestor array is full */ 1259 if (current_id >= rqstor->size) { 1260 unlock_requestor(channel, flags); 1261 return VMBUS_RQST_ERROR; 1262 } 1263 1264 rqstor->next_request_id = rqstor->req_arr[current_id]; 1265 rqstor->req_arr[current_id] = rqst_addr; 1266 1267 /* The already held spin lock provides atomicity */ 1268 bitmap_set(rqstor->req_bitmap, current_id, 1); 1269 1270 unlock_requestor(channel, flags); 1271 1272 /* 1273 * Cannot return an ID of 0, which is reserved for an unsolicited 1274 * message from Hyper-V; Hyper-V does not acknowledge (respond to) 1275 * VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED requests with ID of 1276 * 0 sent by the guest. 1277 */ 1278 return current_id + 1; 1279 } 1280 EXPORT_SYMBOL_GPL(vmbus_next_request_id); 1281 1282 /* As in vmbus_request_addr_match() but without the requestor lock */ 1283 u64 __vmbus_request_addr_match(struct vmbus_channel *channel, u64 trans_id, 1284 u64 rqst_addr) 1285 { 1286 struct vmbus_requestor *rqstor = &channel->requestor; 1287 u64 req_addr; 1288 1289 /* Check rqstor has been initialized */ 1290 if (!channel->rqstor_size) 1291 return VMBUS_NO_RQSTOR; 1292 1293 /* Hyper-V can send an unsolicited message with ID of 0 */ 1294 if (!trans_id) 1295 return VMBUS_RQST_ERROR; 1296 1297 /* Data corresponding to trans_id is stored at trans_id - 1 */ 1298 trans_id--; 1299 1300 /* Invalid trans_id */ 1301 if (trans_id >= rqstor->size || !test_bit(trans_id, rqstor->req_bitmap)) 1302 return VMBUS_RQST_ERROR; 1303 1304 req_addr = rqstor->req_arr[trans_id]; 1305 if (rqst_addr == VMBUS_RQST_ADDR_ANY || req_addr == rqst_addr) { 1306 rqstor->req_arr[trans_id] = rqstor->next_request_id; 1307 rqstor->next_request_id = trans_id; 1308 1309 /* The already held spin lock provides atomicity */ 1310 bitmap_clear(rqstor->req_bitmap, trans_id, 1); 1311 } 1312 1313 return req_addr; 1314 } 1315 EXPORT_SYMBOL_GPL(__vmbus_request_addr_match); 1316 1317 /* 1318 * vmbus_request_addr_match - Clears/removes @trans_id from the @channel's 1319 * requestor, provided the memory address stored at @trans_id equals @rqst_addr 1320 * (or provided @rqst_addr matches the sentinel value VMBUS_RQST_ADDR_ANY). 1321 * 1322 * Returns the memory address stored at @trans_id, or VMBUS_RQST_ERROR if 1323 * @trans_id is not contained in the requestor. 1324 * 1325 * Acquires and releases the requestor spin lock. 1326 */ 1327 u64 vmbus_request_addr_match(struct vmbus_channel *channel, u64 trans_id, 1328 u64 rqst_addr) 1329 { 1330 unsigned long flags; 1331 u64 req_addr; 1332 1333 lock_requestor(channel, flags); 1334 req_addr = __vmbus_request_addr_match(channel, trans_id, rqst_addr); 1335 unlock_requestor(channel, flags); 1336 1337 return req_addr; 1338 } 1339 EXPORT_SYMBOL_GPL(vmbus_request_addr_match); 1340 1341 /* 1342 * vmbus_request_addr - Returns the memory address stored at @trans_id 1343 * in @rqstor. Uses a spin lock to avoid race conditions. 1344 * @channel: Pointer to the VMbus channel struct 1345 * @trans_id: Request id sent back from Hyper-V. Becomes the requestor's 1346 * next request id. 1347 */ 1348 u64 vmbus_request_addr(struct vmbus_channel *channel, u64 trans_id) 1349 { 1350 return vmbus_request_addr_match(channel, trans_id, VMBUS_RQST_ADDR_ANY); 1351 } 1352 EXPORT_SYMBOL_GPL(vmbus_request_addr); 1353