1 /*- 2 * Copyright (c) 2018 VMware, Inc. All Rights Reserved. 3 * 4 * SPDX-License-Identifier: (BSD-2-Clause AND GPL-2.0) 5 */ 6 7 /* VMCI QueuePair API implementation. */ 8 9 #include <sys/cdefs.h> 10 __FBSDID("$FreeBSD$"); 11 12 #include "vmci.h" 13 #include "vmci_driver.h" 14 #include "vmci_event.h" 15 #include "vmci_kernel_api.h" 16 #include "vmci_kernel_defs.h" 17 #include "vmci_queue_pair.h" 18 19 #define LGPFX "vmci_queue_pair: " 20 21 struct queue_pair_entry { 22 vmci_list_item(queue_pair_entry) list_item; 23 struct vmci_handle handle; 24 vmci_id peer; 25 uint32_t flags; 26 uint64_t produce_size; 27 uint64_t consume_size; 28 uint32_t ref_count; 29 }; 30 31 struct qp_guest_endpoint { 32 struct queue_pair_entry qp; 33 uint64_t num_ppns; 34 void *produce_q; 35 void *consume_q; 36 bool hibernate_failure; 37 struct ppn_set ppn_set; 38 }; 39 40 struct queue_pair_list { 41 vmci_list(queue_pair_entry) head; 42 volatile int hibernate; 43 vmci_mutex mutex; 44 }; 45 46 #define QPE_NUM_PAGES(_QPE) \ 47 ((uint32_t)(CEILING(_QPE.produce_size, PAGE_SIZE) + \ 48 CEILING(_QPE.consume_size, PAGE_SIZE) + 2)) 49 50 static struct queue_pair_list qp_guest_endpoints; 51 52 static struct queue_pair_entry *queue_pair_list_find_entry( 53 struct queue_pair_list *qp_list, struct vmci_handle handle); 54 static void queue_pair_list_add_entry(struct queue_pair_list *qp_list, 55 struct queue_pair_entry *entry); 56 static void queue_pair_list_remove_entry(struct queue_pair_list *qp_list, 57 struct queue_pair_entry *entry); 58 static struct queue_pair_entry *queue_pair_list_get_head( 59 struct queue_pair_list *qp_list); 60 static int queue_pair_notify_peer_local(bool attach, 61 struct vmci_handle handle); 62 static struct qp_guest_endpoint *qp_guest_endpoint_create( 63 struct vmci_handle handle, vmci_id peer, uint32_t flags, 64 uint64_t produce_size, uint64_t consume_size, 65 void *produce_q, void *consume_q); 66 static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry); 67 static int vmci_queue_pair_alloc_hypercall( 68 const struct qp_guest_endpoint *entry); 69 static int vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle, 70 struct vmci_queue **produce_q, uint64_t produce_size, 71 struct vmci_queue **consume_q, uint64_t consume_size, 72 vmci_id peer, uint32_t flags, 73 vmci_privilege_flags priv_flags); 74 static int vmci_queue_pair_detach_guest_work(struct vmci_handle handle); 75 static int vmci_queue_pair_detach_hypercall(struct vmci_handle handle); 76 77 extern int vmci_send_datagram(struct vmci_datagram *); 78 79 /* 80 *------------------------------------------------------------------------------ 81 * 82 * vmci_queue_pair_alloc -- 83 * 84 * Allocates a VMCI QueuePair. Only checks validity of input arguments. The 85 * real work is done in the host or guest specific function. 86 * 87 * Results: 88 * VMCI_SUCCESS on success, appropriate error code otherwise. 89 * 90 * Side effects: 91 * None. 92 * 93 *------------------------------------------------------------------------------ 94 */ 95 96 int 97 vmci_queue_pair_alloc(struct vmci_handle *handle, struct vmci_queue **produce_q, 98 uint64_t produce_size, struct vmci_queue **consume_q, uint64_t consume_size, 99 vmci_id peer, uint32_t flags, vmci_privilege_flags priv_flags) 100 { 101 102 if (!handle || !produce_q || !consume_q || 103 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS)) 104 return (VMCI_ERROR_INVALID_ARGS); 105 106 return (vmci_queue_pair_alloc_guest_work(handle, produce_q, 107 produce_size, consume_q, consume_size, peer, flags, priv_flags)); 108 } 109 110 /* 111 *------------------------------------------------------------------------------ 112 * 113 * vmci_queue_pair_detach -- 114 * 115 * Detaches from a VMCI QueuePair. Only checks validity of input argument. 116 * Real work is done in the host or guest specific function. 117 * 118 * Results: 119 * Success or failure. 120 * 121 * Side effects: 122 * Memory is freed. 123 * 124 *------------------------------------------------------------------------------ 125 */ 126 127 int 128 vmci_queue_pair_detach(struct vmci_handle handle) 129 { 130 131 if (VMCI_HANDLE_INVALID(handle)) 132 return (VMCI_ERROR_INVALID_ARGS); 133 134 return (vmci_queue_pair_detach_guest_work(handle)); 135 } 136 137 /* 138 *------------------------------------------------------------------------------ 139 * 140 * queue_pair_list_init -- 141 * 142 * Initializes the list of QueuePairs. 143 * 144 * Results: 145 * Success or failure. 146 * 147 * Side effects: 148 * None. 149 * 150 *------------------------------------------------------------------------------ 151 */ 152 153 static inline int 154 queue_pair_list_init(struct queue_pair_list *qp_list) 155 { 156 int ret; 157 158 vmci_list_init(&qp_list->head); 159 atomic_store_int(&qp_list->hibernate, 0); 160 ret = vmci_mutex_init(&qp_list->mutex, "VMCI QP List lock"); 161 return (ret); 162 } 163 164 /* 165 *------------------------------------------------------------------------------ 166 * 167 * queue_pair_list_destroy -- 168 * 169 * Destroy the list's mutex. 170 * 171 * Results: 172 * None. 173 * 174 * Side effects: 175 * None. 176 * 177 *------------------------------------------------------------------------------ 178 */ 179 180 static inline void 181 queue_pair_list_destroy(struct queue_pair_list *qp_list) 182 { 183 184 vmci_mutex_destroy(&qp_list->mutex); 185 vmci_list_init(&qp_list->head); 186 } 187 188 /* 189 *------------------------------------------------------------------------------ 190 * 191 * queue_pair_list_find_entry -- 192 * 193 * Finds the entry in the list corresponding to a given handle. Assumes that 194 * the list is locked. 195 * 196 * Results: 197 * Pointer to entry. 198 * 199 * Side effects: 200 * None. 201 * 202 *------------------------------------------------------------------------------ 203 */ 204 205 static struct queue_pair_entry * 206 queue_pair_list_find_entry(struct queue_pair_list *qp_list, 207 struct vmci_handle handle) 208 { 209 struct queue_pair_entry *next; 210 211 if (VMCI_HANDLE_INVALID(handle)) 212 return (NULL); 213 214 vmci_list_scan(next, &qp_list->head, list_item) { 215 if (VMCI_HANDLE_EQUAL(next->handle, handle)) 216 return (next); 217 } 218 219 return (NULL); 220 } 221 222 /* 223 *------------------------------------------------------------------------------ 224 * 225 * queue_pair_list_add_entry -- 226 * 227 * Adds the given entry to the list. Assumes that the list is locked. 228 * 229 * Results: 230 * None. 231 * 232 * Side effects: 233 * None. 234 * 235 *------------------------------------------------------------------------------ 236 */ 237 238 static void 239 queue_pair_list_add_entry(struct queue_pair_list *qp_list, 240 struct queue_pair_entry *entry) 241 { 242 243 if (entry) 244 vmci_list_insert(&qp_list->head, entry, list_item); 245 } 246 247 /* 248 *------------------------------------------------------------------------------ 249 * 250 * queue_pair_list_remove_entry -- 251 * 252 * Removes the given entry from the list. Assumes that the list is locked. 253 * 254 * Results: 255 * None. 256 * 257 * Side effects: 258 * None. 259 * 260 *------------------------------------------------------------------------------ 261 */ 262 263 static void 264 queue_pair_list_remove_entry(struct queue_pair_list *qp_list, 265 struct queue_pair_entry *entry) 266 { 267 268 if (entry) 269 vmci_list_remove(entry, list_item); 270 } 271 272 /* 273 *------------------------------------------------------------------------------ 274 * 275 * queue_pair_list_get_head -- 276 * 277 * Returns the entry from the head of the list. Assumes that the list is 278 * locked. 279 * 280 * Results: 281 * Pointer to entry. 282 * 283 * Side effects: 284 * None. 285 * 286 *------------------------------------------------------------------------------ 287 */ 288 289 static struct queue_pair_entry * 290 queue_pair_list_get_head(struct queue_pair_list *qp_list) 291 { 292 293 return (vmci_list_first(&qp_list->head)); 294 } 295 296 /* 297 *------------------------------------------------------------------------------ 298 * 299 * vmci_qp_guest_endpoints_init -- 300 * 301 * Initalizes data structure state keeping track of queue pair guest 302 * endpoints. 303 * 304 * Results: 305 * VMCI_SUCCESS on success and appropriate failure code otherwise. 306 * 307 * Side effects: 308 * None. 309 * 310 *------------------------------------------------------------------------------ 311 */ 312 313 int 314 vmci_qp_guest_endpoints_init(void) 315 { 316 317 return (queue_pair_list_init(&qp_guest_endpoints)); 318 } 319 320 /* 321 *------------------------------------------------------------------------------ 322 * 323 * vmci_qp_guest_endpoints_exit -- 324 * 325 * Destroys all guest queue pair endpoints. If active guest queue pairs 326 * still exist, hypercalls to attempt detach from these queue pairs will be 327 * made. Any failure to detach is silently ignored. 328 * 329 * Results: 330 * None. 331 * 332 * Side effects: 333 * None. 334 * 335 *------------------------------------------------------------------------------ 336 */ 337 338 void 339 vmci_qp_guest_endpoints_exit(void) 340 { 341 struct qp_guest_endpoint *entry; 342 343 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 344 345 while ((entry = 346 (struct qp_guest_endpoint *)queue_pair_list_get_head( 347 &qp_guest_endpoints)) != NULL) { 348 /* 349 * Don't make a hypercall for local QueuePairs. 350 */ 351 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL)) 352 vmci_queue_pair_detach_hypercall(entry->qp.handle); 353 /* 354 * We cannot fail the exit, so let's reset ref_count. 355 */ 356 entry->qp.ref_count = 0; 357 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp); 358 qp_guest_endpoint_destroy(entry); 359 } 360 361 atomic_store_int(&qp_guest_endpoints.hibernate, 0); 362 vmci_mutex_release(&qp_guest_endpoints.mutex); 363 queue_pair_list_destroy(&qp_guest_endpoints); 364 } 365 366 /* 367 *------------------------------------------------------------------------------ 368 * 369 * vmci_qp_guest_endpoints_sync -- 370 * 371 * Use this as a synchronization point when setting globals, for example, 372 * during device shutdown. 373 * 374 * Results: 375 * true. 376 * 377 * Side effects: 378 * None. 379 * 380 *------------------------------------------------------------------------------ 381 */ 382 383 void 384 vmci_qp_guest_endpoints_sync(void) 385 { 386 387 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 388 vmci_mutex_release(&qp_guest_endpoints.mutex); 389 } 390 391 /* 392 *------------------------------------------------------------------------------ 393 * 394 * qp_guest_endpoint_create -- 395 * 396 * Allocates and initializes a qp_guest_endpoint structure. Allocates a 397 * QueuePair rid (and handle) iff the given entry has an invalid handle. 398 * 0 through VMCI_RESERVED_RESOURCE_ID_MAX are reserved handles. Assumes 399 * that the QP list mutex is held by the caller. 400 * 401 * Results: 402 * Pointer to structure intialized. 403 * 404 * Side effects: 405 * None. 406 * 407 *------------------------------------------------------------------------------ 408 */ 409 410 struct qp_guest_endpoint * 411 qp_guest_endpoint_create(struct vmci_handle handle, vmci_id peer, 412 uint32_t flags, uint64_t produce_size, uint64_t consume_size, 413 void *produce_q, void *consume_q) 414 { 415 struct qp_guest_endpoint *entry; 416 static vmci_id queue_pair_rid; 417 const uint64_t num_ppns = CEILING(produce_size, PAGE_SIZE) + 418 CEILING(consume_size, PAGE_SIZE) + 419 2; /* One page each for the queue headers. */ 420 421 queue_pair_rid = VMCI_RESERVED_RESOURCE_ID_MAX + 1; 422 423 ASSERT((produce_size || consume_size) && produce_q && consume_q); 424 425 if (VMCI_HANDLE_INVALID(handle)) { 426 vmci_id context_id = vmci_get_context_id(); 427 vmci_id old_rid = queue_pair_rid; 428 429 /* 430 * Generate a unique QueuePair rid. Keep on trying until we 431 * wrap around in the RID space. 432 */ 433 ASSERT(old_rid > VMCI_RESERVED_RESOURCE_ID_MAX); 434 do { 435 handle = VMCI_MAKE_HANDLE(context_id, queue_pair_rid); 436 entry = 437 (struct qp_guest_endpoint *) 438 queue_pair_list_find_entry(&qp_guest_endpoints, 439 handle); 440 queue_pair_rid++; 441 if (UNLIKELY(!queue_pair_rid)) { 442 /* 443 * Skip the reserved rids. 444 */ 445 queue_pair_rid = 446 VMCI_RESERVED_RESOURCE_ID_MAX + 1; 447 } 448 } while (entry && queue_pair_rid != old_rid); 449 450 if (UNLIKELY(entry != NULL)) { 451 ASSERT(queue_pair_rid == old_rid); 452 /* 453 * We wrapped around --- no rids were free. 454 */ 455 return (NULL); 456 } 457 } 458 459 ASSERT(!VMCI_HANDLE_INVALID(handle) && 460 queue_pair_list_find_entry(&qp_guest_endpoints, handle) == NULL); 461 entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL); 462 if (entry) { 463 entry->qp.handle = handle; 464 entry->qp.peer = peer; 465 entry->qp.flags = flags; 466 entry->qp.produce_size = produce_size; 467 entry->qp.consume_size = consume_size; 468 entry->qp.ref_count = 0; 469 entry->num_ppns = num_ppns; 470 memset(&entry->ppn_set, 0, sizeof(entry->ppn_set)); 471 entry->produce_q = produce_q; 472 entry->consume_q = consume_q; 473 } 474 return (entry); 475 } 476 477 /* 478 *------------------------------------------------------------------------------ 479 * 480 * qp_guest_endpoint_destroy -- 481 * 482 * Frees a qp_guest_endpoint structure. 483 * 484 * Results: 485 * None. 486 * 487 * Side effects: 488 * None. 489 * 490 *------------------------------------------------------------------------------ 491 */ 492 493 void 494 qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry) 495 { 496 497 ASSERT(entry); 498 ASSERT(entry->qp.ref_count == 0); 499 500 vmci_free_ppn_set(&entry->ppn_set); 501 vmci_free_queue(entry->produce_q, entry->qp.produce_size); 502 vmci_free_queue(entry->consume_q, entry->qp.consume_size); 503 vmci_free_kernel_mem(entry, sizeof(*entry)); 504 } 505 506 /* 507 *------------------------------------------------------------------------------ 508 * 509 * vmci_queue_pair_alloc_hypercall -- 510 * 511 * Helper to make a QueuePairAlloc hypercall when the driver is 512 * supporting a guest device. 513 * 514 * Results: 515 * Result of the hypercall. 516 * 517 * Side effects: 518 * Memory is allocated & freed. 519 * 520 *------------------------------------------------------------------------------ 521 */ 522 static int 523 vmci_queue_pair_alloc_hypercall(const struct qp_guest_endpoint *entry) 524 { 525 struct vmci_queue_pair_alloc_msg *alloc_msg; 526 size_t msg_size; 527 int result; 528 529 if (!entry || entry->num_ppns <= 2) 530 return (VMCI_ERROR_INVALID_ARGS); 531 532 ASSERT(!(entry->qp.flags & VMCI_QPFLAG_LOCAL)); 533 534 msg_size = sizeof(*alloc_msg) + (size_t)entry->num_ppns * sizeof(PPN); 535 alloc_msg = vmci_alloc_kernel_mem(msg_size, VMCI_MEMORY_NORMAL); 536 if (!alloc_msg) 537 return (VMCI_ERROR_NO_MEM); 538 539 alloc_msg->hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 540 VMCI_QUEUEPAIR_ALLOC); 541 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE; 542 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE; 543 alloc_msg->handle = entry->qp.handle; 544 alloc_msg->peer = entry->qp.peer; 545 alloc_msg->flags = entry->qp.flags; 546 alloc_msg->produce_size = entry->qp.produce_size; 547 alloc_msg->consume_size = entry->qp.consume_size; 548 alloc_msg->num_ppns = entry->num_ppns; 549 result = vmci_populate_ppn_list((uint8_t *)alloc_msg + 550 sizeof(*alloc_msg), &entry->ppn_set); 551 if (result == VMCI_SUCCESS) 552 result = vmci_send_datagram((struct vmci_datagram *)alloc_msg); 553 vmci_free_kernel_mem(alloc_msg, msg_size); 554 555 return (result); 556 } 557 558 /* 559 *------------------------------------------------------------------------------ 560 * 561 * vmci_queue_pair_alloc_guest_work -- 562 * 563 * This functions handles the actual allocation of a VMCI queue pair guest 564 * endpoint. Allocates physical pages for the queue pair. It makes OS 565 * dependent calls through generic wrappers. 566 * 567 * Results: 568 * Success or failure. 569 * 570 * Side effects: 571 * Memory is allocated. 572 * 573 *------------------------------------------------------------------------------ 574 */ 575 576 static int 577 vmci_queue_pair_alloc_guest_work(struct vmci_handle *handle, 578 struct vmci_queue **produce_q, uint64_t produce_size, 579 struct vmci_queue **consume_q, uint64_t consume_size, vmci_id peer, 580 uint32_t flags, vmci_privilege_flags priv_flags) 581 { 582 struct qp_guest_endpoint *queue_pair_entry = NULL; 583 void *my_consume_q = NULL; 584 void *my_produce_q = NULL; 585 const uint64_t num_consume_pages = CEILING(consume_size, PAGE_SIZE) + 1; 586 const uint64_t num_produce_pages = CEILING(produce_size, PAGE_SIZE) + 1; 587 int result; 588 589 ASSERT(handle && produce_q && consume_q && 590 (produce_size || consume_size)); 591 592 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS) 593 return (VMCI_ERROR_NO_ACCESS); 594 595 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 596 597 if ((atomic_load_int(&qp_guest_endpoints.hibernate) == 1) && 598 !(flags & VMCI_QPFLAG_LOCAL)) { 599 /* 600 * While guest OS is in hibernate state, creating non-local 601 * queue pairs is not allowed after the point where the VMCI 602 * guest driver converted the existing queue pairs to local 603 * ones. 604 */ 605 606 result = VMCI_ERROR_UNAVAILABLE; 607 goto error; 608 } 609 610 if ((queue_pair_entry = 611 (struct qp_guest_endpoint *)queue_pair_list_find_entry( 612 &qp_guest_endpoints, *handle)) != NULL) { 613 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) { 614 /* Local attach case. */ 615 if (queue_pair_entry->qp.ref_count > 1) { 616 VMCI_LOG_DEBUG(LGPFX"Error attempting to " 617 "attach more than once.\n"); 618 result = VMCI_ERROR_UNAVAILABLE; 619 goto error_keep_entry; 620 } 621 622 if (queue_pair_entry->qp.produce_size != consume_size || 623 queue_pair_entry->qp.consume_size != produce_size || 624 queue_pair_entry->qp.flags != 625 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) { 626 VMCI_LOG_DEBUG(LGPFX"Error mismatched " 627 "queue pair in local attach.\n"); 628 result = VMCI_ERROR_QUEUEPAIR_MISMATCH; 629 goto error_keep_entry; 630 } 631 632 /* 633 * Do a local attach. We swap the consume and produce 634 * queues for the attacher and deliver an attach event. 635 */ 636 result = queue_pair_notify_peer_local(true, *handle); 637 if (result < VMCI_SUCCESS) 638 goto error_keep_entry; 639 my_produce_q = queue_pair_entry->consume_q; 640 my_consume_q = queue_pair_entry->produce_q; 641 goto out; 642 } 643 result = VMCI_ERROR_ALREADY_EXISTS; 644 goto error_keep_entry; 645 } 646 647 my_produce_q = vmci_alloc_queue(produce_size, flags); 648 if (!my_produce_q) { 649 VMCI_LOG_WARNING(LGPFX"Error allocating pages for produce " 650 "queue.\n"); 651 result = VMCI_ERROR_NO_MEM; 652 goto error; 653 } 654 655 my_consume_q = vmci_alloc_queue(consume_size, flags); 656 if (!my_consume_q) { 657 VMCI_LOG_WARNING(LGPFX"Error allocating pages for consume " 658 "queue.\n"); 659 result = VMCI_ERROR_NO_MEM; 660 goto error; 661 } 662 663 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags, 664 produce_size, consume_size, my_produce_q, my_consume_q); 665 if (!queue_pair_entry) { 666 VMCI_LOG_WARNING(LGPFX"Error allocating memory in %s.\n", 667 __FUNCTION__); 668 result = VMCI_ERROR_NO_MEM; 669 goto error; 670 } 671 672 result = vmci_alloc_ppn_set(my_produce_q, num_produce_pages, 673 my_consume_q, num_consume_pages, &queue_pair_entry->ppn_set); 674 if (result < VMCI_SUCCESS) { 675 VMCI_LOG_WARNING(LGPFX"vmci_alloc_ppn_set failed.\n"); 676 goto error; 677 } 678 679 /* 680 * It's only necessary to notify the host if this queue pair will be 681 * attached to from another context. 682 */ 683 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) { 684 /* Local create case. */ 685 vmci_id context_id = vmci_get_context_id(); 686 687 /* 688 * Enforce similar checks on local queue pairs as we do for 689 * regular ones. The handle's context must match the creator 690 * or attacher context id (here they are both the current 691 * context id) and the attach-only flag cannot exist during 692 * create. We also ensure specified peer is this context or 693 * an invalid one. 694 */ 695 if (queue_pair_entry->qp.handle.context != context_id || 696 (queue_pair_entry->qp.peer != VMCI_INVALID_ID && 697 queue_pair_entry->qp.peer != context_id)) { 698 result = VMCI_ERROR_NO_ACCESS; 699 goto error; 700 } 701 702 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) { 703 result = VMCI_ERROR_NOT_FOUND; 704 goto error; 705 } 706 } else { 707 result = vmci_queue_pair_alloc_hypercall(queue_pair_entry); 708 if (result < VMCI_SUCCESS) { 709 VMCI_LOG_WARNING( 710 LGPFX"vmci_queue_pair_alloc_hypercall result = " 711 "%d.\n", result); 712 goto error; 713 } 714 } 715 716 queue_pair_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp); 717 718 out: 719 queue_pair_entry->qp.ref_count++; 720 *handle = queue_pair_entry->qp.handle; 721 *produce_q = (struct vmci_queue *)my_produce_q; 722 *consume_q = (struct vmci_queue *)my_consume_q; 723 724 /* 725 * We should initialize the queue pair header pages on a local queue 726 * pair create. For non-local queue pairs, the hypervisor initializes 727 * the header pages in the create step. 728 */ 729 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) && 730 queue_pair_entry->qp.ref_count == 1) { 731 vmci_queue_header_init((*produce_q)->q_header, *handle); 732 vmci_queue_header_init((*consume_q)->q_header, *handle); 733 } 734 735 vmci_mutex_release(&qp_guest_endpoints.mutex); 736 737 return (VMCI_SUCCESS); 738 739 error: 740 vmci_mutex_release(&qp_guest_endpoints.mutex); 741 if (queue_pair_entry) { 742 /* The queues will be freed inside the destroy routine. */ 743 qp_guest_endpoint_destroy(queue_pair_entry); 744 } else { 745 if (my_produce_q) 746 vmci_free_queue(my_produce_q, produce_size); 747 if (my_consume_q) 748 vmci_free_queue(my_consume_q, consume_size); 749 } 750 return (result); 751 752 error_keep_entry: 753 /* This path should only be used when an existing entry was found. */ 754 ASSERT(queue_pair_entry->qp.ref_count > 0); 755 vmci_mutex_release(&qp_guest_endpoints.mutex); 756 return (result); 757 } 758 759 /* 760 *------------------------------------------------------------------------------ 761 * 762 * vmci_queue_pair_detach_hypercall -- 763 * 764 * Helper to make a QueuePairDetach hypercall when the driver is supporting 765 * a guest device. 766 * 767 * Results: 768 * Result of the hypercall. 769 * 770 * Side effects: 771 * None. 772 * 773 *------------------------------------------------------------------------------ 774 */ 775 776 int 777 vmci_queue_pair_detach_hypercall(struct vmci_handle handle) 778 { 779 struct vmci_queue_pair_detach_msg detach_msg; 780 781 detach_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 782 VMCI_QUEUEPAIR_DETACH); 783 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE; 784 detach_msg.hdr.payload_size = sizeof(handle); 785 detach_msg.handle = handle; 786 787 return (vmci_send_datagram((struct vmci_datagram *)&detach_msg)); 788 } 789 790 /* 791 *------------------------------------------------------------------------------ 792 * 793 * vmci_queue_pair_detach_guest_work -- 794 * 795 * Helper for VMCI QueuePair detach interface. Frees the physical pages for 796 * the queue pair. 797 * 798 * Results: 799 * Success or failure. 800 * 801 * Side effects: 802 * Memory may be freed. 803 * 804 *------------------------------------------------------------------------------ 805 */ 806 807 static int 808 vmci_queue_pair_detach_guest_work(struct vmci_handle handle) 809 { 810 struct qp_guest_endpoint *entry; 811 int result; 812 uint32_t ref_count; 813 814 ASSERT(!VMCI_HANDLE_INVALID(handle)); 815 816 vmci_mutex_acquire(&qp_guest_endpoints.mutex); 817 818 entry = (struct qp_guest_endpoint *)queue_pair_list_find_entry( 819 &qp_guest_endpoints, handle); 820 if (!entry) { 821 vmci_mutex_release(&qp_guest_endpoints.mutex); 822 return (VMCI_ERROR_NOT_FOUND); 823 } 824 825 ASSERT(entry->qp.ref_count >= 1); 826 827 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) { 828 result = VMCI_SUCCESS; 829 830 if (entry->qp.ref_count > 1) { 831 result = queue_pair_notify_peer_local(false, handle); 832 833 /* 834 * We can fail to notify a local queuepair because we 835 * can't allocate. We still want to release the entry 836 * if that happens, so don't bail out yet. 837 */ 838 } 839 } else { 840 result = vmci_queue_pair_detach_hypercall(handle); 841 if (entry->hibernate_failure) { 842 if (result == VMCI_ERROR_NOT_FOUND) { 843 844 /* 845 * If a queue pair detach failed when entering 846 * hibernation, the guest driver and the device 847 * may disagree on its existence when coming 848 * out of hibernation. The guest driver will 849 * regard it as a non-local queue pair, but 850 * the device state is gone, since the device 851 * has been powered off. In this case, we 852 * treat the queue pair as a local queue pair 853 * with no peer. 854 */ 855 856 ASSERT(entry->qp.ref_count == 1); 857 result = VMCI_SUCCESS; 858 } 859 } 860 if (result < VMCI_SUCCESS) { 861 862 /* 863 * We failed to notify a non-local queuepair. That other 864 * queuepair might still be accessing the shared 865 * memory, so don't release the entry yet. It will get 866 * cleaned up by vmci_queue_pair_Exit() if necessary 867 * (assuming we are going away, otherwise why did this 868 * fail?). 869 */ 870 871 vmci_mutex_release(&qp_guest_endpoints.mutex); 872 return (result); 873 } 874 } 875 876 /* 877 * If we get here then we either failed to notify a local queuepair, or 878 * we succeeded in all cases. Release the entry if required. 879 */ 880 881 entry->qp.ref_count--; 882 if (entry->qp.ref_count == 0) 883 queue_pair_list_remove_entry(&qp_guest_endpoints, &entry->qp); 884 885 /* If we didn't remove the entry, this could change once we unlock. */ 886 ref_count = entry ? entry->qp.ref_count : 887 0xffffffff; /* 888 * Value does not matter, silence the 889 * compiler. 890 */ 891 892 vmci_mutex_release(&qp_guest_endpoints.mutex); 893 894 if (ref_count == 0) 895 qp_guest_endpoint_destroy(entry); 896 return (result); 897 } 898 899 /* 900 *------------------------------------------------------------------------------ 901 * 902 * queue_pair_notify_peer_local -- 903 * 904 * Dispatches a queue pair event message directly into the local event 905 * queue. 906 * 907 * Results: 908 * VMCI_SUCCESS on success, error code otherwise 909 * 910 * Side effects: 911 * None. 912 * 913 *------------------------------------------------------------------------------ 914 */ 915 916 static int 917 queue_pair_notify_peer_local(bool attach, struct vmci_handle handle) 918 { 919 struct vmci_event_msg *e_msg; 920 struct vmci_event_payload_qp *e_payload; 921 /* buf is only 48 bytes. */ 922 vmci_id context_id; 923 context_id = vmci_get_context_id(); 924 char buf[sizeof(*e_msg) + sizeof(*e_payload)]; 925 926 e_msg = (struct vmci_event_msg *)buf; 927 e_payload = vmci_event_msg_payload(e_msg); 928 929 e_msg->hdr.dst = VMCI_MAKE_HANDLE(context_id, VMCI_EVENT_HANDLER); 930 e_msg->hdr.src = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, 931 VMCI_CONTEXT_RESOURCE_ID); 932 e_msg->hdr.payload_size = sizeof(*e_msg) + sizeof(*e_payload) - 933 sizeof(e_msg->hdr); 934 e_msg->event_data.event = attach ? VMCI_EVENT_QP_PEER_ATTACH : 935 VMCI_EVENT_QP_PEER_DETACH; 936 e_payload->peer_id = context_id; 937 e_payload->handle = handle; 938 939 return (vmci_event_dispatch((struct vmci_datagram *)e_msg)); 940 } 941