1 /*- 2 * Copyright (c) 2018 VMware, Inc. 3 * 4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) 5 */ 6 7 /* VMCI QueuePair API implementation. */ 8 9 #include <sys/cdefs.h> 10 __FBSDID("$FreeBSD$"); 11 12 #include "vmci.h" 13 #include "vmci_driver.h" 14 #include "vmci_event.h" 15 #include "vmci_kernel_api.h" 16 #include "vmci_kernel_defs.h" 17 #include "vmci_queue_pair.h" 18 19 #define LGPFX "vmci_queue_pair: " 20 21 struct queue_pair_entry { 22 vmci_list_item(queue_pair_entry) list_item; 23 struct vmci_handle handle; 24 vmci_id peer; 25 uint32_t flags; 26 uint64_t produce_size; 27 uint64_t consume_size; 28 uint32_t ref_count; 29 }; 30 31 struct qp_guest_endpoint { 32 struct queue_pair_entry qp; 33 uint64_t num_ppns; 34 void *produce_q; 35 void *consume_q; 36 bool hibernate_failure; 37 struct ppn_set ppn_set; 38 }; 39 40 struct queue_pair_list { 41 vmci_list(queue_pair_entry) head; 42 volatile int hibernate; 43 vmci_mutex mutex; 44 }; 45 46 #define QPE_NUM_PAGES(_QPE) \ 47 ((uint32_t)(CEILING(_QPE.produce_size, PAGE_SIZE) + \ 48 CEILING(_QPE.consume_size, PAGE_SIZE) + 2)) 49 50 static struct queue_pair_list qp_guest_endpoints; 51 52 static struct queue_pair_entry *queue_pair_list_find_entry( 53 struct queue_pair_list *qp_list, struct vmci_handle handle); 54 static void queue_pair_list_add_entry(struct queue_pair_list *qp_list, 55 struct queue_pair_entry *entry); 56 static void queue_pair_list_remove_entry(struct queue_pair_list *qp_list, 57 struct queue_pair_entry *entry); 58 static struct queue_pair_entry *queue_pair_list_get_head( 59 struct queue_pair_list *qp_list); 60 static int queue_pair_notify_peer_local(bool attach, 61 struct vmci_handle handle); 62 static struct qp_guest_endpoint *qp_guest_endpoint_create( 63 struct vmci_handle handle, vmci_id peer, uint32_t flags, 64 uint64_t produce_size, uint64_t consume_size, 65 void *produce_q, void *consume_q); 66 static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry); 67 static int vmci_queue_pair_alloc_hypercall( 68 const struct qp_guest_endpoint *entry); 69 static int vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle, 70 struct vmci_queue **produce_q, uint64_t produce_size, 71 struct vmci_queue **consume_q, uint64_t consume_size, 72 vmci_id peer, uint32_t flags, 73 vmci_privilege_flags priv_flags); 74 static int vmci_queue_pair_detach_guest_work(struct vmci_handle handle); 75 static int vmci_queue_pair_detach_hypercall(struct vmci_handle handle); 76 77 /* 78 *------------------------------------------------------------------------------ 79 * 80 * vmci_queue_pair_alloc -- 81 * 82 * Allocates a VMCI QueuePair. Only checks validity of input arguments. The 83 * real work is done in the host or guest specific function. 84 * 85 * Results: 86 * VMCI_SUCCESS on success, appropriate error code otherwise. 87 * 88 * Side effects: 89 * None. 90 * 91 *------------------------------------------------------------------------------ 92 */ 93 94 int 95 vmci_queue_pair_alloc(struct vmci_handle *handle, struct vmci_queue **produce_q, 96 uint64_t produce_size, struct vmci_queue **consume_q, uint64_t consume_size, 97 vmci_id peer, uint32_t flags, vmci_privilege_flags priv_flags) 98 { 99 100 if (!handle || !produce_q || !consume_q || 101 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS)) 102 return (VMCI_ERROR_INVALID_ARGS); 103 104 return (vmci_queue_pair_alloc_guest_work(handle, produce_q, 105 produce_size, consume_q, consume_size, peer, flags, priv_flags)); 106 } 107 108 /* 109 *------------------------------------------------------------------------------ 110 * 111 * vmci_queue_pair_detach -- 112 * 113 * Detaches from a VMCI QueuePair. Only checks validity of input argument. 114 * Real work is done in the host or guest specific function. 115 * 116 * Results: 117 * Success or failure. 118 * 119 * Side effects: 120 * Memory is freed. 121 * 122 *------------------------------------------------------------------------------ 123 */ 124 125 int 126 vmci_queue_pair_detach(struct vmci_handle handle) 127 { 128 129 if (VMCI_HANDLE_INVALID(handle)) 130 return (VMCI_ERROR_INVALID_ARGS); 131 132 return (vmci_queue_pair_detach_guest_work(handle)); 133 } 134 135 /* 136 *------------------------------------------------------------------------------ 137 * 138 * queue_pair_list_init -- 139 * 140 * Initializes the list of QueuePairs. 141 * 142 * Results: 143 * Success or failure. 144 * 145 * Side effects: 146 * None. 147 * 148 *------------------------------------------------------------------------------ 149 */ 150 151 static inline int 152 queue_pair_list_init(struct queue_pair_list *qp_list) 153 { 154 int ret; 155 156 vmci_list_init(&qp_list->head); 157 atomic_store_int(&qp_list->hibernate, 0); 158 ret = vmci_mutex_init(&qp_list->mutex, "VMCI QP List lock"); 159 return (ret); 160 } 161 162 /* 163 *------------------------------------------------------------------------------ 164 * 165 * queue_pair_list_destroy -- 166 * 167 * Destroy the list's mutex. 168 * 169 * Results: 170 * None. 171 * 172 * Side effects: 173 * None. 174 * 175 *------------------------------------------------------------------------------ 176 */ 177 178 static inline void 179 queue_pair_list_destroy(struct queue_pair_list *qp_list) 180 { 181 182 vmci_mutex_destroy(&qp_list->mutex); 183 vmci_list_init(&qp_list->head); 184 } 185 186 /* 187 *------------------------------------------------------------------------------ 188 * 189 * queue_pair_list_find_entry -- 190 * 191 * Finds the entry in the list corresponding to a given handle. Assumes that 192 * the list is locked. 193 * 194 * Results: 195 * Pointer to entry. 196 * 197 * Side effects: 198 * None. 199 * 200 *------------------------------------------------------------------------------ 201 */ 202 203 static struct queue_pair_entry * 204 queue_pair_list_find_entry(struct queue_pair_list *qp_list, 205 struct vmci_handle handle) 206 { 207 struct queue_pair_entry *next; 208 209 if (VMCI_HANDLE_INVALID(handle)) 210 return (NULL); 211 212 vmci_list_scan(next, &qp_list->head, list_item) { 213 if (VMCI_HANDLE_EQUAL(next->handle, handle)) 214 return (next); 215 } 216 217 return (NULL); 218 } 219 220 /* 221 *------------------------------------------------------------------------------ 222 * 223 * queue_pair_list_add_entry -- 224 * 225 * Adds the given entry to the list. Assumes that the list is locked. 226 * 227 * Results: 228 * None. 229 * 230 * Side effects: 231 * None. 232 * 233 *------------------------------------------------------------------------------ 234 */ 235 236 static void 237 queue_pair_list_add_entry(struct queue_pair_list *qp_list, 238 struct queue_pair_entry *entry) 239 { 240 241 if (entry) 242 vmci_list_insert(&qp_list->head, entry, list_item); 243 } 244 245 /* 246 *------------------------------------------------------------------------------ 247 * 248 * queue_pair_list_remove_entry -- 249 * 250 * Removes the given entry from the list. Assumes that the list is locked. 251 * 252 * Results: 253 * None. 254 * 255 * Side effects: 256 * None. 257 * 258 *------------------------------------------------------------------------------ 259 */ 260 261 static void 262 queue_pair_list_remove_entry(struct queue_pair_list *qp_list, 263 struct queue_pair_entry *entry) 264 { 265 266 if (entry) 267 vmci_list_remove(entry, list_item); 268 } 269 270 /* 271 *------------------------------------------------------------------------------ 272 * 273 * queue_pair_list_get_head -- 274 * 275 * Returns the entry from the head of the list. Assumes that the list is 276 * locked. 277 * 278 * Results: 279 * Pointer to entry. 280 * 281 * Side effects: 282 * None. 283 * 284 *------------------------------------------------------------------------------ 285 */ 286 287 static struct queue_pair_entry * 288 queue_pair_list_get_head(struct queue_pair_list *qp_list) 289 { 290 291 return (vmci_list_first(&qp_list->head)); 292 } 293 294 /* 295 *------------------------------------------------------------------------------ 296 * 297 * vmci_qp_guest_endpoints_init -- 298 * 299 * Initalizes data structure state keeping track of queue pair guest 300 * endpoints. 301 * 302 * Results: 303 * VMCI_SUCCESS on success and appropriate failure code otherwise. 304 * 305 * Side effects: 306 * None. 307 * 308 *------------------------------------------------------------------------------ 309 */ 310 311 int 312 vmci_qp_guest_endpoints_init(void) 313 { 314 315 return (queue_pair_list_init(&qp_guest_endpoints)); 316 } 317 318 /* 319 *------------------------------------------------------------------------------ 320 * 321 * vmci_qp_guest_endpoints_exit -- 322 * 323 * Destroys all guest queue pair endpoints. If active guest queue pairs 324 * still exist, hypercalls to attempt detach from these queue pairs will be 325 * made. Any failure to detach is silently ignored. 326 * 327 * Results: 328 * None. 329 * 330 * Side effects: 331 * None. 332 * 333 *------------------------------------------------------------------------------ 334 */ 335 336 void 337 vmci_qp_guest_endpoints_exit(void) 338 { 339 struct qp_guest_endpoint *entry; 340 341 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 342 343 while ((entry = 344 (struct qp_guest_endpoint *)queue_pair_list_get_head( 345 &qp_guest_endpoints)) != NULL) { 346 /* 347 * Don't make a hypercall for local QueuePairs. 348 */ 349 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL)) 350 vmci_queue_pair_detach_hypercall(entry->qp.handle); 351 /* 352 * We cannot fail the exit, so let's reset ref_count. 353 */ 354 entry->qp.ref_count = 0; 355 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp); 356 qp_guest_endpoint_destroy(entry); 357 } 358 359 atomic_store_int(&qp_guest_endpoints.hibernate, 0); 360 vmci_mutex_release(&qp_guest_endpoints.mutex); 361 queue_pair_list_destroy(&qp_guest_endpoints); 362 } 363 364 /* 365 *------------------------------------------------------------------------------ 366 * 367 * vmci_qp_guest_endpoints_sync -- 368 * 369 * Use this as a synchronization point when setting globals, for example, 370 * during device shutdown. 371 * 372 * Results: 373 * true. 374 * 375 * Side effects: 376 * None. 377 * 378 *------------------------------------------------------------------------------ 379 */ 380 381 void 382 vmci_qp_guest_endpoints_sync(void) 383 { 384 385 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 386 vmci_mutex_release(&qp_guest_endpoints.mutex); 387 } 388 389 /* 390 *------------------------------------------------------------------------------ 391 * 392 * qp_guest_endpoint_create -- 393 * 394 * Allocates and initializes a qp_guest_endpoint structure. Allocates a 395 * QueuePair rid (and handle) iff the given entry has an invalid handle. 396 * 0 through VMCI_RESERVED_RESOURCE_ID_MAX are reserved handles. Assumes 397 * that the QP list mutex is held by the caller. 398 * 399 * Results: 400 * Pointer to structure intialized. 401 * 402 * Side effects: 403 * None. 404 * 405 *------------------------------------------------------------------------------ 406 */ 407 408 struct qp_guest_endpoint * 409 qp_guest_endpoint_create(struct vmci_handle handle, vmci_id peer, 410 uint32_t flags, uint64_t produce_size, uint64_t consume_size, 411 void *produce_q, void *consume_q) 412 { 413 struct qp_guest_endpoint *entry; 414 static vmci_id queue_pair_rid; 415 const uint64_t num_ppns = CEILING(produce_size, PAGE_SIZE) + 416 CEILING(consume_size, PAGE_SIZE) + 417 2; /* One page each for the queue headers. */ 418 419 queue_pair_rid = VMCI_RESERVED_RESOURCE_ID_MAX + 1; 420 421 ASSERT((produce_size || consume_size) && produce_q && consume_q); 422 423 if (VMCI_HANDLE_INVALID(handle)) { 424 vmci_id context_id = vmci_get_context_id(); 425 vmci_id old_rid = queue_pair_rid; 426 427 /* 428 * Generate a unique QueuePair rid. Keep on trying until we 429 * wrap around in the RID space. 430 */ 431 ASSERT(old_rid > VMCI_RESERVED_RESOURCE_ID_MAX); 432 do { 433 handle = VMCI_MAKE_HANDLE(context_id, queue_pair_rid); 434 entry = 435 (struct qp_guest_endpoint *) 436 queue_pair_list_find_entry(&qp_guest_endpoints, 437 handle); 438 queue_pair_rid++; 439 if (UNLIKELY(!queue_pair_rid)) { 440 /* 441 * Skip the reserved rids. 442 */ 443 queue_pair_rid = 444 VMCI_RESERVED_RESOURCE_ID_MAX + 1; 445 } 446 } while (entry && queue_pair_rid != old_rid); 447 448 if (UNLIKELY(entry != NULL)) { 449 ASSERT(queue_pair_rid == old_rid); 450 /* 451 * We wrapped around --- no rids were free. 452 */ 453 return (NULL); 454 } 455 } 456 457 ASSERT(!VMCI_HANDLE_INVALID(handle) && 458 queue_pair_list_find_entry(&qp_guest_endpoints, handle) == NULL); 459 entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL); 460 if (entry) { 461 entry->qp.handle = handle; 462 entry->qp.peer = peer; 463 entry->qp.flags = flags; 464 entry->qp.produce_size = produce_size; 465 entry->qp.consume_size = consume_size; 466 entry->qp.ref_count = 0; 467 entry->num_ppns = num_ppns; 468 memset(&entry->ppn_set, 0, sizeof(entry->ppn_set)); 469 entry->produce_q = produce_q; 470 entry->consume_q = consume_q; 471 } 472 return (entry); 473 } 474 475 /* 476 *------------------------------------------------------------------------------ 477 * 478 * qp_guest_endpoint_destroy -- 479 * 480 * Frees a qp_guest_endpoint structure. 481 * 482 * Results: 483 * None. 484 * 485 * Side effects: 486 * None. 487 * 488 *------------------------------------------------------------------------------ 489 */ 490 491 void 492 qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry) 493 { 494 495 ASSERT(entry); 496 ASSERT(entry->qp.ref_count == 0); 497 498 vmci_free_ppn_set(&entry->ppn_set); 499 vmci_free_queue(entry->produce_q, entry->qp.produce_size); 500 vmci_free_queue(entry->consume_q, entry->qp.consume_size); 501 vmci_free_kernel_mem(entry, sizeof(*entry)); 502 } 503 504 /* 505 *------------------------------------------------------------------------------ 506 * 507 * vmci_queue_pair_alloc_hypercall -- 508 * 509 * Helper to make a QueuePairAlloc hypercall when the driver is 510 * supporting a guest device. 511 * 512 * Results: 513 * Result of the hypercall. 514 * 515 * Side effects: 516 * Memory is allocated & freed. 517 * 518 *------------------------------------------------------------------------------ 519 */ 520 static int 521 vmci_queue_pair_alloc_hypercall(const struct qp_guest_endpoint *entry) 522 { 523 struct vmci_queue_pair_alloc_msg *alloc_msg; 524 size_t msg_size; 525 int result; 526 527 if (!entry || entry->num_ppns <= 2) 528 return (VMCI_ERROR_INVALID_ARGS); 529 530 ASSERT(!(entry->qp.flags & VMCI_QPFLAG_LOCAL)); 531 532 msg_size = sizeof(*alloc_msg) + (size_t)entry->num_ppns * sizeof(PPN); 533 alloc_msg = vmci_alloc_kernel_mem(msg_size, VMCI_MEMORY_NORMAL); 534 if (!alloc_msg) 535 return (VMCI_ERROR_NO_MEM); 536 537 alloc_msg->hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 538 VMCI_QUEUEPAIR_ALLOC); 539 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE; 540 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE; 541 alloc_msg->handle = entry->qp.handle; 542 alloc_msg->peer = entry->qp.peer; 543 alloc_msg->flags = entry->qp.flags; 544 alloc_msg->produce_size = entry->qp.produce_size; 545 alloc_msg->consume_size = entry->qp.consume_size; 546 alloc_msg->num_ppns = entry->num_ppns; 547 result = vmci_populate_ppn_list((uint8_t *)alloc_msg + 548 sizeof(*alloc_msg), &entry->ppn_set); 549 if (result == VMCI_SUCCESS) 550 result = vmci_send_datagram((struct vmci_datagram *)alloc_msg); 551 vmci_free_kernel_mem(alloc_msg, msg_size); 552 553 return (result); 554 } 555 556 /* 557 *------------------------------------------------------------------------------ 558 * 559 * vmci_queue_pair_alloc_guest_work -- 560 * 561 * This functions handles the actual allocation of a VMCI queue pair guest 562 * endpoint. Allocates physical pages for the queue pair. It makes OS 563 * dependent calls through generic wrappers. 564 * 565 * Results: 566 * Success or failure. 567 * 568 * Side effects: 569 * Memory is allocated. 570 * 571 *------------------------------------------------------------------------------ 572 */ 573 574 static int 575 vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle, 576 struct vmci_queue **produce_q, uint64_t produce_size, 577 struct vmci_queue **consume_q, uint64_t consume_size, vmci_id peer, 578 uint32_t flags, vmci_privilege_flags priv_flags) 579 { 580 struct qp_guest_endpoint *queue_pair_entry = NULL; 581 void *my_consume_q = NULL; 582 void *my_produce_q = NULL; 583 const uint64_t num_consume_pages = CEILING(consume_size, PAGE_SIZE) + 1; 584 const uint64_t num_produce_pages = CEILING(produce_size, PAGE_SIZE) + 1; 585 int result; 586 587 ASSERT(handle && produce_q && consume_q && 588 (produce_size || consume_size)); 589 590 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS) 591 return (VMCI_ERROR_NO_ACCESS); 592 593 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 594 595 if ((atomic_load_int(&qp_guest_endpoints.hibernate) == 1) && 596 !(flags & VMCI_QPFLAG_LOCAL)) { 597 /* 598 * While guest OS is in hibernate state, creating non-local 599 * queue pairs is not allowed after the point where the VMCI 600 * guest driver converted the existing queue pairs to local 601 * ones. 602 */ 603 604 result = VMCI_ERROR_UNAVAILABLE; 605 goto error; 606 } 607 608 if ((queue_pair_entry = 609 (struct qp_guest_endpoint *)queue_pair_list_find_entry( 610 &qp_guest_endpoints, *handle)) != NULL) { 611 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) { 612 /* Local attach case. */ 613 if (queue_pair_entry->qp.ref_count > 1) { 614 VMCI_LOG_DEBUG(LGPFX"Error attempting to " 615 "attach more than once.\n"); 616 result = VMCI_ERROR_UNAVAILABLE; 617 goto error_keep_entry; 618 } 619 620 if (queue_pair_entry->qp.produce_size != consume_size || 621 queue_pair_entry->qp.consume_size != produce_size || 622 queue_pair_entry->qp.flags != 623 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) { 624 VMCI_LOG_DEBUG(LGPFX"Error mismatched " 625 "queue pair in local attach.\n"); 626 result = VMCI_ERROR_QUEUEPAIR_MISMATCH; 627 goto error_keep_entry; 628 } 629 630 /* 631 * Do a local attach. We swap the consume and produce 632 * queues for the attacher and deliver an attach event. 633 */ 634 result = queue_pair_notify_peer_local(true, *handle); 635 if (result < VMCI_SUCCESS) 636 goto error_keep_entry; 637 my_produce_q = queue_pair_entry->consume_q; 638 my_consume_q = queue_pair_entry->produce_q; 639 goto out; 640 } 641 result = VMCI_ERROR_ALREADY_EXISTS; 642 goto error_keep_entry; 643 } 644 645 my_produce_q = vmci_alloc_queue(produce_size, flags); 646 if (!my_produce_q) { 647 VMCI_LOG_WARNING(LGPFX"Error allocating pages for produce " 648 "queue.\n"); 649 result = VMCI_ERROR_NO_MEM; 650 goto error; 651 } 652 653 my_consume_q = vmci_alloc_queue(consume_size, flags); 654 if (!my_consume_q) { 655 VMCI_LOG_WARNING(LGPFX"Error allocating pages for consume " 656 "queue.\n"); 657 result = VMCI_ERROR_NO_MEM; 658 goto error; 659 } 660 661 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags, 662 produce_size, consume_size, my_produce_q, my_consume_q); 663 if (!queue_pair_entry) { 664 VMCI_LOG_WARNING(LGPFX"Error allocating memory in %s.\n", 665 __FUNCTION__); 666 result = VMCI_ERROR_NO_MEM; 667 goto error; 668 } 669 670 result = vmci_alloc_ppn_set(my_produce_q, num_produce_pages, 671 my_consume_q, num_consume_pages, &queue_pair_entry->ppn_set); 672 if (result < VMCI_SUCCESS) { 673 VMCI_LOG_WARNING(LGPFX"vmci_alloc_ppn_set failed.\n"); 674 goto error; 675 } 676 677 /* 678 * It's only necessary to notify the host if this queue pair will be 679 * attached to from another context. 680 */ 681 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) { 682 /* Local create case. */ 683 vmci_id context_id = vmci_get_context_id(); 684 685 /* 686 * Enforce similar checks on local queue pairs as we do for 687 * regular ones. The handle's context must match the creator 688 * or attacher context id (here they are both the current 689 * context id) and the attach-only flag cannot exist during 690 * create. We also ensure specified peer is this context or 691 * an invalid one. 692 */ 693 if (queue_pair_entry->qp.handle.context != context_id || 694 (queue_pair_entry->qp.peer != VMCI_INVALID_ID && 695 queue_pair_entry->qp.peer != context_id)) { 696 result = VMCI_ERROR_NO_ACCESS; 697 goto error; 698 } 699 700 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) { 701 result = VMCI_ERROR_NOT_FOUND; 702 goto error; 703 } 704 } else { 705 result = vmci_queue_pair_alloc_hypercall(queue_pair_entry); 706 if (result < VMCI_SUCCESS) { 707 VMCI_LOG_WARNING( 708 LGPFX"vmci_queue_pair_alloc_hypercall result = " 709 "%d.\n", result); 710 goto error; 711 } 712 } 713 714 queue_pair_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp); 715 716 out: 717 queue_pair_entry->qp.ref_count++; 718 *handle = queue_pair_entry->qp.handle; 719 *produce_q = (struct vmci_queue *)my_produce_q; 720 *consume_q = (struct vmci_queue *)my_consume_q; 721 722 /* 723 * We should initialize the queue pair header pages on a local queue 724 * pair create. For non-local queue pairs, the hypervisor initializes 725 * the header pages in the create step. 726 */ 727 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) && 728 queue_pair_entry->qp.ref_count == 1) { 729 vmci_queue_header_init((*produce_q)->q_header, *handle); 730 vmci_queue_header_init((*consume_q)->q_header, *handle); 731 } 732 733 vmci_mutex_release(&qp_guest_endpoints.mutex); 734 735 return (VMCI_SUCCESS); 736 737 error: 738 vmci_mutex_release(&qp_guest_endpoints.mutex); 739 if (queue_pair_entry) { 740 /* The queues will be freed inside the destroy routine. */ 741 qp_guest_endpoint_destroy(queue_pair_entry); 742 } else { 743 if (my_produce_q) 744 vmci_free_queue(my_produce_q, produce_size); 745 if (my_consume_q) 746 vmci_free_queue(my_consume_q, consume_size); 747 } 748 return (result); 749 750 error_keep_entry: 751 /* This path should only be used when an existing entry was found. */ 752 ASSERT(queue_pair_entry->qp.ref_count > 0); 753 vmci_mutex_release(&qp_guest_endpoints.mutex); 754 return (result); 755 } 756 757 /* 758 *------------------------------------------------------------------------------ 759 * 760 * vmci_queue_pair_detach_hypercall -- 761 * 762 * Helper to make a QueuePairDetach hypercall when the driver is supporting 763 * a guest device. 764 * 765 * Results: 766 * Result of the hypercall. 767 * 768 * Side effects: 769 * None. 770 * 771 *------------------------------------------------------------------------------ 772 */ 773 774 int 775 vmci_queue_pair_detach_hypercall(struct vmci_handle handle) 776 { 777 struct vmci_queue_pair_detach_msg detach_msg; 778 779 detach_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 780 VMCI_QUEUEPAIR_DETACH); 781 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE; 782 detach_msg.hdr.payload_size = sizeof(handle); 783 detach_msg.handle = handle; 784 785 return (vmci_send_datagram((struct vmci_datagram *)&detach_msg)); 786 } 787 788 /* 789 *------------------------------------------------------------------------------ 790 * 791 * vmci_queue_pair_detach_guest_work -- 792 * 793 * Helper for VMCI QueuePair detach interface. Frees the physical pages for 794 * the queue pair. 795 * 796 * Results: 797 * Success or failure. 798 * 799 * Side effects: 800 * Memory may be freed. 801 * 802 *------------------------------------------------------------------------------ 803 */ 804 805 static int 806 vmci_queue_pair_detach_guest_work(struct vmci_handle handle) 807 { 808 struct qp_guest_endpoint *entry; 809 int result; 810 uint32_t ref_count; 811 812 ASSERT(!VMCI_HANDLE_INVALID(handle)); 813 814 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 815 816 entry = (struct qp_guest_endpoint *)queue_pair_list_find_entry( 817 &qp_guest_endpoints, handle); 818 if (!entry) { 819 vmci_mutex_release(&qp_guest_endpoints.mutex); 820 return (VMCI_ERROR_NOT_FOUND); 821 } 822 823 ASSERT(entry->qp.ref_count >= 1); 824 825 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) { 826 result = VMCI_SUCCESS; 827 828 if (entry->qp.ref_count > 1) { 829 result = queue_pair_notify_peer_local(false, handle); 830 831 /* 832 * We can fail to notify a local queuepair because we 833 * can't allocate. We still want to release the entry 834 * if that happens, so don't bail out yet. 835 */ 836 } 837 } else { 838 result = vmci_queue_pair_detach_hypercall(handle); 839 if (entry->hibernate_failure) { 840 if (result == VMCI_ERROR_NOT_FOUND) { 841 842 /* 843 * If a queue pair detach failed when entering 844 * hibernation, the guest driver and the device 845 * may disagree on its existence when coming 846 * out of hibernation. The guest driver will 847 * regard it as a non-local queue pair, but 848 * the device state is gone, since the device 849 * has been powered off. In this case, we 850 * treat the queue pair as a local queue pair 851 * with no peer. 852 */ 853 854 ASSERT(entry->qp.ref_count == 1); 855 result = VMCI_SUCCESS; 856 } 857 } 858 if (result < VMCI_SUCCESS) { 859 860 /* 861 * We failed to notify a non-local queuepair. That other 862 * queuepair might still be accessing the shared 863 * memory, so don't release the entry yet. It will get 864 * cleaned up by vmci_queue_pair_Exit() if necessary 865 * (assuming we are going away, otherwise why did this 866 * fail?). 867 */ 868 869 vmci_mutex_release(&qp_guest_endpoints.mutex); 870 return (result); 871 } 872 } 873 874 /* 875 * If we get here then we either failed to notify a local queuepair, or 876 * we succeeded in all cases. Release the entry if required. 877 */ 878 879 entry->qp.ref_count--; 880 if (entry->qp.ref_count == 0) 881 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp); 882 883 /* If we didn't remove the entry, this could change once we unlock. */ 884 ref_count = entry ? entry->qp.ref_count : 885 0xffffffff; /* 886 * Value does not matter, silence the 887 * compiler. 888 */ 889 890 vmci_mutex_release(&qp_guest_endpoints.mutex); 891 892 if (ref_count == 0) 893 qp_guest_endpoint_destroy(entry); 894 return (result); 895 } 896 897 /* 898 *------------------------------------------------------------------------------ 899 * 900 * queue_pair_notify_peer_local -- 901 * 902 * Dispatches a queue pair event message directly into the local event 903 * queue. 904 * 905 * Results: 906 * VMCI_SUCCESS on success, error code otherwise 907 * 908 * Side effects: 909 * None. 910 * 911 *------------------------------------------------------------------------------ 912 */ 913 914 static int 915 queue_pair_notify_peer_local(bool attach, struct vmci_handle handle) 916 { 917 struct vmci_event_msg *e_msg; 918 struct vmci_event_payload_qp *e_payload; 919 /* buf is only 48 bytes. */ 920 vmci_id context_id; 921 context_id = vmci_get_context_id(); 922 char buf[sizeof(*e_msg) + sizeof(*e_payload)]; 923 924 e_msg = (struct vmci_event_msg *)buf; 925 e_payload = vmci_event_msg_payload(e_msg); 926 927 e_msg->hdr.dst = VMCI_MAKE_HANDLE(context_id, VMCI_EVENT_HANDLER); 928 e_msg->hdr.src = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 929 VMCI_CONTEXT_RESOURCE_ID); 930 e_msg->hdr.payload_size = sizeof(*e_msg) + sizeof(*e_payload) - 931 sizeof(e_msg->hdr); 932 e_msg->event_data.event = attach ? VMCI_EVENT_QP_PEER_ATTACH : 933 VMCI_EVENT_QP_PEER_DETACH; 934 e_payload->peer_id = context_id; 935 e_payload->handle = handle; 936 937 return (vmci_event_dispatch((struct vmci_datagram *)e_msg)); 938 } 939