1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VMware VMCI Driver 4 * 5 * Copyright (C) 2012 VMware, Inc. All rights reserved. 6 */ 7 8 #include <linux/vmw_vmci_defs.h> 9 #include <linux/vmw_vmci_api.h> 10 #include <linux/moduleparam.h> 11 #include <linux/interrupt.h> 12 #include <linux/highmem.h> 13 #include <linux/kernel.h> 14 #include <linux/mm.h> 15 #include <linux/module.h> 16 #include <linux/processor.h> 17 #include <linux/sched.h> 18 #include <linux/slab.h> 19 #include <linux/init.h> 20 #include <linux/pci.h> 21 #include <linux/smp.h> 22 #include <linux/io.h> 23 #include <linux/vmalloc.h> 24 25 #include "vmci_datagram.h" 26 #include "vmci_doorbell.h" 27 #include "vmci_context.h" 28 #include "vmci_driver.h" 29 #include "vmci_event.h" 30 31 #define PCI_DEVICE_ID_VMWARE_VMCI 0x0740 32 33 #define VMCI_UTIL_NUM_RESOURCES 1 34 35 /* 36 * Datagram buffers for DMA send/receive must accommodate at least 37 * a maximum sized datagram and the header. 38 */ 39 #define VMCI_DMA_DG_BUFFER_SIZE (VMCI_MAX_DG_SIZE + PAGE_SIZE) 40 41 static bool vmci_disable_msi; 42 module_param_named(disable_msi, vmci_disable_msi, bool, 0); 43 MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)"); 44 45 static bool vmci_disable_msix; 46 module_param_named(disable_msix, vmci_disable_msix, bool, 0); 47 MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)"); 48 49 static u32 ctx_update_sub_id = VMCI_INVALID_ID; 50 static u32 vm_context_id = VMCI_INVALID_ID; 51 52 struct vmci_guest_device { 53 struct device *dev; /* PCI device we are attached to */ 54 void __iomem *iobase; 55 void __iomem *mmio_base; 56 57 bool exclusive_vectors; 58 59 struct wait_queue_head inout_wq; 60 61 void *data_buffer; 62 dma_addr_t data_buffer_base; 63 void *tx_buffer; 64 dma_addr_t tx_buffer_base; 65 void *notification_bitmap; 66 dma_addr_t notification_base; 67 }; 68 69 static bool use_ppn64; 70 71 bool vmci_use_ppn64(void) 72 { 73 return use_ppn64; 74 } 75 76 /* vmci_dev singleton device and supporting data*/ 77 struct pci_dev *vmci_pdev; 78 static struct vmci_guest_device *vmci_dev_g; 79 static DEFINE_SPINLOCK(vmci_dev_spinlock); 80 81 static atomic_t vmci_num_guest_devices = ATOMIC_INIT(0); 82 83 bool vmci_guest_code_active(void) 84 { 85 return atomic_read(&vmci_num_guest_devices) != 0; 86 } 87 88 u32 vmci_get_vm_context_id(void) 89 { 90 if (vm_context_id == VMCI_INVALID_ID) { 91 struct vmci_datagram get_cid_msg; 92 get_cid_msg.dst = 93 vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, 94 VMCI_GET_CONTEXT_ID); 95 get_cid_msg.src = VMCI_ANON_SRC_HANDLE; 96 get_cid_msg.payload_size = 0; 97 vm_context_id = vmci_send_datagram(&get_cid_msg); 98 } 99 return vm_context_id; 100 } 101 102 static unsigned int vmci_read_reg(struct vmci_guest_device *dev, u32 reg) 103 { 104 if (dev->mmio_base != NULL) 105 return readl(dev->mmio_base + reg); 106 return ioread32(dev->iobase + reg); 107 } 108 109 static void vmci_write_reg(struct vmci_guest_device *dev, u32 val, u32 reg) 110 { 111 if (dev->mmio_base != NULL) 112 writel(val, dev->mmio_base + reg); 113 else 114 iowrite32(val, dev->iobase + reg); 115 } 116 117 static void vmci_read_data(struct vmci_guest_device *vmci_dev, 118 void *dest, size_t size) 119 { 120 if (vmci_dev->mmio_base == NULL) 121 ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR, 122 dest, size); 123 else { 124 /* 125 * For DMA datagrams, the data_buffer will contain the header on the 126 * first page, followed by the incoming datagram(s) on the following 127 * pages. The header uses an S/G element immediately following the 128 * header on the first page to point to the data area. 129 */ 130 struct vmci_data_in_out_header *buffer_header = vmci_dev->data_buffer; 131 struct vmci_sg_elem *sg_array = (struct vmci_sg_elem *)(buffer_header + 1); 132 size_t buffer_offset = dest - vmci_dev->data_buffer; 133 134 buffer_header->opcode = 1; 135 buffer_header->size = 1; 136 buffer_header->busy = 0; 137 sg_array[0].addr = vmci_dev->data_buffer_base + buffer_offset; 138 sg_array[0].size = size; 139 140 vmci_write_reg(vmci_dev, lower_32_bits(vmci_dev->data_buffer_base), 141 VMCI_DATA_IN_LOW_ADDR); 142 143 wait_event(vmci_dev->inout_wq, buffer_header->busy == 1); 144 } 145 } 146 147 static int vmci_write_data(struct vmci_guest_device *dev, 148 struct vmci_datagram *dg) 149 { 150 int result; 151 152 if (dev->mmio_base != NULL) { 153 struct vmci_data_in_out_header *buffer_header = dev->tx_buffer; 154 u8 *dg_out_buffer = (u8 *)(buffer_header + 1); 155 156 if (VMCI_DG_SIZE(dg) > VMCI_MAX_DG_SIZE) 157 return VMCI_ERROR_INVALID_ARGS; 158 159 /* 160 * Initialize send buffer with outgoing datagram 161 * and set up header for inline data. Device will 162 * not access buffer asynchronously - only after 163 * the write to VMCI_DATA_OUT_LOW_ADDR. 164 */ 165 memcpy(dg_out_buffer, dg, VMCI_DG_SIZE(dg)); 166 buffer_header->opcode = 0; 167 buffer_header->size = VMCI_DG_SIZE(dg); 168 buffer_header->busy = 1; 169 170 vmci_write_reg(dev, lower_32_bits(dev->tx_buffer_base), 171 VMCI_DATA_OUT_LOW_ADDR); 172 173 /* Caller holds a spinlock, so cannot block. */ 174 spin_until_cond(buffer_header->busy == 0); 175 176 result = vmci_read_reg(vmci_dev_g, VMCI_RESULT_LOW_ADDR); 177 if (result == VMCI_SUCCESS) 178 result = (int)buffer_header->result; 179 } else { 180 iowrite8_rep(dev->iobase + VMCI_DATA_OUT_ADDR, 181 dg, VMCI_DG_SIZE(dg)); 182 result = vmci_read_reg(vmci_dev_g, VMCI_RESULT_LOW_ADDR); 183 } 184 185 return result; 186 } 187 188 /* 189 * VM to hypervisor call mechanism. We use the standard VMware naming 190 * convention since shared code is calling this function as well. 191 */ 192 int vmci_send_datagram(struct vmci_datagram *dg) 193 { 194 unsigned long flags; 195 int result; 196 197 /* Check args. */ 198 if (dg == NULL) 199 return VMCI_ERROR_INVALID_ARGS; 200 201 /* 202 * Need to acquire spinlock on the device because the datagram 203 * data may be spread over multiple pages and the monitor may 204 * interleave device user rpc calls from multiple 205 * VCPUs. Acquiring the spinlock precludes that 206 * possibility. Disabling interrupts to avoid incoming 207 * datagrams during a "rep out" and possibly landing up in 208 * this function. 209 */ 210 spin_lock_irqsave(&vmci_dev_spinlock, flags); 211 212 if (vmci_dev_g) { 213 vmci_write_data(vmci_dev_g, dg); 214 result = vmci_read_reg(vmci_dev_g, VMCI_RESULT_LOW_ADDR); 215 } else { 216 result = VMCI_ERROR_UNAVAILABLE; 217 } 218 219 spin_unlock_irqrestore(&vmci_dev_spinlock, flags); 220 221 return result; 222 } 223 EXPORT_SYMBOL_GPL(vmci_send_datagram); 224 225 /* 226 * Gets called with the new context id if updated or resumed. 227 * Context id. 228 */ 229 static void vmci_guest_cid_update(u32 sub_id, 230 const struct vmci_event_data *event_data, 231 void *client_data) 232 { 233 const struct vmci_event_payld_ctx *ev_payload = 234 vmci_event_data_const_payload(event_data); 235 236 if (sub_id != ctx_update_sub_id) { 237 pr_devel("Invalid subscriber (ID=0x%x)\n", sub_id); 238 return; 239 } 240 241 if (!event_data || ev_payload->context_id == VMCI_INVALID_ID) { 242 pr_devel("Invalid event data\n"); 243 return; 244 } 245 246 pr_devel("Updating context from (ID=0x%x) to (ID=0x%x) on event (type=%d)\n", 247 vm_context_id, ev_payload->context_id, event_data->event); 248 249 vm_context_id = ev_payload->context_id; 250 } 251 252 /* 253 * Verify that the host supports the hypercalls we need. If it does not, 254 * try to find fallback hypercalls and use those instead. Returns 0 if 255 * required hypercalls (or fallback hypercalls) are supported by the host, 256 * an error code otherwise. 257 */ 258 static int vmci_check_host_caps(struct pci_dev *pdev) 259 { 260 bool result; 261 struct vmci_resource_query_msg *msg; 262 u32 msg_size = sizeof(struct vmci_resource_query_hdr) + 263 VMCI_UTIL_NUM_RESOURCES * sizeof(u32); 264 struct vmci_datagram *check_msg; 265 266 check_msg = kzalloc(msg_size, GFP_KERNEL); 267 if (!check_msg) { 268 dev_err(&pdev->dev, "%s: Insufficient memory\n", __func__); 269 return -ENOMEM; 270 } 271 272 check_msg->dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, 273 VMCI_RESOURCES_QUERY); 274 check_msg->src = VMCI_ANON_SRC_HANDLE; 275 check_msg->payload_size = msg_size - VMCI_DG_HEADERSIZE; 276 msg = (struct vmci_resource_query_msg *)VMCI_DG_PAYLOAD(check_msg); 277 278 msg->num_resources = VMCI_UTIL_NUM_RESOURCES; 279 msg->resources[0] = VMCI_GET_CONTEXT_ID; 280 281 /* Checks that hyper calls are supported */ 282 result = vmci_send_datagram(check_msg) == 0x01; 283 kfree(check_msg); 284 285 dev_dbg(&pdev->dev, "%s: Host capability check: %s\n", 286 __func__, result ? "PASSED" : "FAILED"); 287 288 /* We need the vector. There are no fallbacks. */ 289 return result ? 0 : -ENXIO; 290 } 291 292 /* 293 * Reads datagrams from the device and dispatches them. For IO port 294 * based access to the device, we always start reading datagrams into 295 * only the first page of the datagram buffer. If the datagrams don't 296 * fit into one page, we use the maximum datagram buffer size for the 297 * remainder of the invocation. This is a simple heuristic for not 298 * penalizing small datagrams. For DMA-based datagrams, we always 299 * use the maximum datagram buffer size, since there is no performance 300 * penalty for doing so. 301 * 302 * This function assumes that it has exclusive access to the data 303 * in register(s) for the duration of the call. 304 */ 305 static void vmci_dispatch_dgs(struct vmci_guest_device *vmci_dev) 306 { 307 u8 *dg_in_buffer = vmci_dev->data_buffer; 308 struct vmci_datagram *dg; 309 size_t dg_in_buffer_size = VMCI_MAX_DG_SIZE; 310 size_t current_dg_in_buffer_size; 311 size_t remaining_bytes; 312 bool is_io_port = vmci_dev->mmio_base == NULL; 313 314 BUILD_BUG_ON(VMCI_MAX_DG_SIZE < PAGE_SIZE); 315 316 if (!is_io_port) { 317 /* For mmio, the first page is used for the header. */ 318 dg_in_buffer += PAGE_SIZE; 319 320 /* 321 * For DMA-based datagram operations, there is no performance 322 * penalty for reading the maximum buffer size. 323 */ 324 current_dg_in_buffer_size = VMCI_MAX_DG_SIZE; 325 } else { 326 current_dg_in_buffer_size = PAGE_SIZE; 327 } 328 vmci_read_data(vmci_dev, dg_in_buffer, current_dg_in_buffer_size); 329 dg = (struct vmci_datagram *)dg_in_buffer; 330 remaining_bytes = current_dg_in_buffer_size; 331 332 /* 333 * Read through the buffer until an invalid datagram header is 334 * encountered. The exit condition for datagrams read through 335 * VMCI_DATA_IN_ADDR is a bit more complicated, since a datagram 336 * can start on any page boundary in the buffer. 337 */ 338 while (dg->dst.resource != VMCI_INVALID_ID || 339 (is_io_port && remaining_bytes > PAGE_SIZE)) { 340 unsigned dg_in_size; 341 342 /* 343 * If using VMCI_DATA_IN_ADDR, skip to the next page 344 * as a datagram can start on any page boundary. 345 */ 346 if (dg->dst.resource == VMCI_INVALID_ID) { 347 dg = (struct vmci_datagram *)roundup( 348 (uintptr_t)dg + 1, PAGE_SIZE); 349 remaining_bytes = 350 (size_t)(dg_in_buffer + 351 current_dg_in_buffer_size - 352 (u8 *)dg); 353 continue; 354 } 355 356 dg_in_size = VMCI_DG_SIZE_ALIGNED(dg); 357 358 if (dg_in_size <= dg_in_buffer_size) { 359 int result; 360 361 /* 362 * If the remaining bytes in the datagram 363 * buffer doesn't contain the complete 364 * datagram, we first make sure we have enough 365 * room for it and then we read the reminder 366 * of the datagram and possibly any following 367 * datagrams. 368 */ 369 if (dg_in_size > remaining_bytes) { 370 if (remaining_bytes != 371 current_dg_in_buffer_size) { 372 373 /* 374 * We move the partial 375 * datagram to the front and 376 * read the reminder of the 377 * datagram and possibly 378 * following calls into the 379 * following bytes. 380 */ 381 memmove(dg_in_buffer, dg_in_buffer + 382 current_dg_in_buffer_size - 383 remaining_bytes, 384 remaining_bytes); 385 dg = (struct vmci_datagram *) 386 dg_in_buffer; 387 } 388 389 if (current_dg_in_buffer_size != 390 dg_in_buffer_size) 391 current_dg_in_buffer_size = 392 dg_in_buffer_size; 393 394 vmci_read_data(vmci_dev, 395 dg_in_buffer + 396 remaining_bytes, 397 current_dg_in_buffer_size - 398 remaining_bytes); 399 } 400 401 /* 402 * We special case event datagrams from the 403 * hypervisor. 404 */ 405 if (dg->src.context == VMCI_HYPERVISOR_CONTEXT_ID && 406 dg->dst.resource == VMCI_EVENT_HANDLER) { 407 result = vmci_event_dispatch(dg); 408 } else { 409 result = vmci_datagram_invoke_guest_handler(dg); 410 } 411 if (result < VMCI_SUCCESS) 412 dev_dbg(vmci_dev->dev, 413 "Datagram with resource (ID=0x%x) failed (err=%d)\n", 414 dg->dst.resource, result); 415 416 /* On to the next datagram. */ 417 dg = (struct vmci_datagram *)((u8 *)dg + 418 dg_in_size); 419 } else { 420 size_t bytes_to_skip; 421 422 /* 423 * Datagram doesn't fit in datagram buffer of maximal 424 * size. We drop it. 425 */ 426 dev_dbg(vmci_dev->dev, 427 "Failed to receive datagram (size=%u bytes)\n", 428 dg_in_size); 429 430 bytes_to_skip = dg_in_size - remaining_bytes; 431 if (current_dg_in_buffer_size != dg_in_buffer_size) 432 current_dg_in_buffer_size = dg_in_buffer_size; 433 434 for (;;) { 435 vmci_read_data(vmci_dev, dg_in_buffer, 436 current_dg_in_buffer_size); 437 if (bytes_to_skip <= current_dg_in_buffer_size) 438 break; 439 440 bytes_to_skip -= current_dg_in_buffer_size; 441 } 442 dg = (struct vmci_datagram *)(dg_in_buffer + 443 bytes_to_skip); 444 } 445 446 remaining_bytes = 447 (size_t) (dg_in_buffer + current_dg_in_buffer_size - 448 (u8 *)dg); 449 450 if (remaining_bytes < VMCI_DG_HEADERSIZE) { 451 /* Get the next batch of datagrams. */ 452 453 vmci_read_data(vmci_dev, dg_in_buffer, 454 current_dg_in_buffer_size); 455 dg = (struct vmci_datagram *)dg_in_buffer; 456 remaining_bytes = current_dg_in_buffer_size; 457 } 458 } 459 } 460 461 /* 462 * Scans the notification bitmap for raised flags, clears them 463 * and handles the notifications. 464 */ 465 static void vmci_process_bitmap(struct vmci_guest_device *dev) 466 { 467 if (!dev->notification_bitmap) { 468 dev_dbg(dev->dev, "No bitmap present in %s\n", __func__); 469 return; 470 } 471 472 vmci_dbell_scan_notification_entries(dev->notification_bitmap); 473 } 474 475 /* 476 * Interrupt handler for legacy or MSI interrupt, or for first MSI-X 477 * interrupt (vector VMCI_INTR_DATAGRAM). 478 */ 479 static irqreturn_t vmci_interrupt(int irq, void *_dev) 480 { 481 struct vmci_guest_device *dev = _dev; 482 483 /* 484 * If we are using MSI-X with exclusive vectors then we simply call 485 * vmci_dispatch_dgs(), since we know the interrupt was meant for us. 486 * Otherwise we must read the ICR to determine what to do. 487 */ 488 489 if (dev->exclusive_vectors) { 490 vmci_dispatch_dgs(dev); 491 } else { 492 unsigned int icr; 493 494 /* Acknowledge interrupt and determine what needs doing. */ 495 icr = vmci_read_reg(dev, VMCI_ICR_ADDR); 496 if (icr == 0 || icr == ~0) 497 return IRQ_NONE; 498 499 if (icr & VMCI_ICR_DATAGRAM) { 500 vmci_dispatch_dgs(dev); 501 icr &= ~VMCI_ICR_DATAGRAM; 502 } 503 504 if (icr & VMCI_ICR_NOTIFICATION) { 505 vmci_process_bitmap(dev); 506 icr &= ~VMCI_ICR_NOTIFICATION; 507 } 508 509 510 if (icr & VMCI_ICR_DMA_DATAGRAM) { 511 wake_up_all(&dev->inout_wq); 512 icr &= ~VMCI_ICR_DMA_DATAGRAM; 513 } 514 515 if (icr != 0) 516 dev_warn(dev->dev, 517 "Ignoring unknown interrupt cause (%d)\n", 518 icr); 519 } 520 521 return IRQ_HANDLED; 522 } 523 524 /* 525 * Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION, 526 * which is for the notification bitmap. Will only get called if we are 527 * using MSI-X with exclusive vectors. 528 */ 529 static irqreturn_t vmci_interrupt_bm(int irq, void *_dev) 530 { 531 struct vmci_guest_device *dev = _dev; 532 533 /* For MSI-X we can just assume it was meant for us. */ 534 vmci_process_bitmap(dev); 535 536 return IRQ_HANDLED; 537 } 538 539 /* 540 * Interrupt handler for MSI-X interrupt vector VMCI_INTR_DMA_DATAGRAM, 541 * which is for the completion of a DMA datagram send or receive operation. 542 * Will only get called if we are using MSI-X with exclusive vectors. 543 */ 544 static irqreturn_t vmci_interrupt_dma_datagram(int irq, void *_dev) 545 { 546 struct vmci_guest_device *dev = _dev; 547 548 wake_up_all(&dev->inout_wq); 549 550 return IRQ_HANDLED; 551 } 552 553 static void vmci_free_dg_buffers(struct vmci_guest_device *vmci_dev) 554 { 555 if (vmci_dev->mmio_base != NULL) { 556 if (vmci_dev->tx_buffer != NULL) 557 dma_free_coherent(vmci_dev->dev, 558 VMCI_DMA_DG_BUFFER_SIZE, 559 vmci_dev->tx_buffer, 560 vmci_dev->tx_buffer_base); 561 if (vmci_dev->data_buffer != NULL) 562 dma_free_coherent(vmci_dev->dev, 563 VMCI_DMA_DG_BUFFER_SIZE, 564 vmci_dev->data_buffer, 565 vmci_dev->data_buffer_base); 566 } else { 567 vfree(vmci_dev->data_buffer); 568 } 569 } 570 571 /* 572 * Most of the initialization at module load time is done here. 573 */ 574 static int vmci_guest_probe_device(struct pci_dev *pdev, 575 const struct pci_device_id *id) 576 { 577 struct vmci_guest_device *vmci_dev; 578 void __iomem *iobase = NULL; 579 void __iomem *mmio_base = NULL; 580 unsigned int num_irq_vectors; 581 unsigned int capabilities; 582 unsigned int caps_in_use; 583 unsigned long cmd; 584 int vmci_err; 585 int error; 586 587 dev_dbg(&pdev->dev, "Probing for vmci/PCI guest device\n"); 588 589 error = pcim_enable_device(pdev); 590 if (error) { 591 dev_err(&pdev->dev, 592 "Failed to enable VMCI device: %d\n", error); 593 return error; 594 } 595 596 /* 597 * The VMCI device with mmio access to registers requests 256KB 598 * for BAR1. If present, driver will use new VMCI device 599 * functionality for register access and datagram send/recv. 600 */ 601 602 if (pci_resource_len(pdev, 1) == VMCI_WITH_MMIO_ACCESS_BAR_SIZE) { 603 dev_info(&pdev->dev, "MMIO register access is available\n"); 604 mmio_base = pci_iomap_range(pdev, 1, VMCI_MMIO_ACCESS_OFFSET, 605 VMCI_MMIO_ACCESS_SIZE); 606 /* If the map fails, we fall back to IOIO access. */ 607 if (!mmio_base) 608 dev_warn(&pdev->dev, "Failed to map MMIO register access\n"); 609 } 610 611 if (!mmio_base) { 612 if (IS_ENABLED(CONFIG_ARM64)) { 613 dev_err(&pdev->dev, "MMIO base is invalid\n"); 614 return -ENXIO; 615 } 616 error = pcim_iomap_regions(pdev, BIT(0), KBUILD_MODNAME); 617 if (error) { 618 dev_err(&pdev->dev, "Failed to reserve/map IO regions\n"); 619 return error; 620 } 621 iobase = pcim_iomap_table(pdev)[0]; 622 } 623 624 vmci_dev = devm_kzalloc(&pdev->dev, sizeof(*vmci_dev), GFP_KERNEL); 625 if (!vmci_dev) { 626 dev_err(&pdev->dev, 627 "Can't allocate memory for VMCI device\n"); 628 error = -ENOMEM; 629 goto err_unmap_mmio_base; 630 } 631 632 vmci_dev->dev = &pdev->dev; 633 vmci_dev->exclusive_vectors = false; 634 vmci_dev->iobase = iobase; 635 vmci_dev->mmio_base = mmio_base; 636 637 init_waitqueue_head(&vmci_dev->inout_wq); 638 639 if (mmio_base != NULL) { 640 vmci_dev->tx_buffer = dma_alloc_coherent(&pdev->dev, VMCI_DMA_DG_BUFFER_SIZE, 641 &vmci_dev->tx_buffer_base, 642 GFP_KERNEL); 643 if (!vmci_dev->tx_buffer) { 644 dev_err(&pdev->dev, 645 "Can't allocate memory for datagram tx buffer\n"); 646 error = -ENOMEM; 647 goto err_unmap_mmio_base; 648 } 649 650 vmci_dev->data_buffer = dma_alloc_coherent(&pdev->dev, VMCI_DMA_DG_BUFFER_SIZE, 651 &vmci_dev->data_buffer_base, 652 GFP_KERNEL); 653 } else { 654 vmci_dev->data_buffer = vmalloc(VMCI_MAX_DG_SIZE); 655 } 656 if (!vmci_dev->data_buffer) { 657 dev_err(&pdev->dev, 658 "Can't allocate memory for datagram buffer\n"); 659 error = -ENOMEM; 660 goto err_free_data_buffers; 661 } 662 663 pci_set_master(pdev); /* To enable queue_pair functionality. */ 664 665 /* 666 * Verify that the VMCI Device supports the capabilities that 667 * we need. If the device is missing capabilities that we would 668 * like to use, check for fallback capabilities and use those 669 * instead (so we can run a new VM on old hosts). Fail the load if 670 * a required capability is missing and there is no fallback. 671 * 672 * Right now, we need datagrams. There are no fallbacks. 673 */ 674 capabilities = vmci_read_reg(vmci_dev, VMCI_CAPS_ADDR); 675 if (!(capabilities & VMCI_CAPS_DATAGRAM)) { 676 dev_err(&pdev->dev, "Device does not support datagrams\n"); 677 error = -ENXIO; 678 goto err_free_data_buffers; 679 } 680 caps_in_use = VMCI_CAPS_DATAGRAM; 681 682 /* 683 * Use 64-bit PPNs if the device supports. 684 * 685 * There is no check for the return value of dma_set_mask_and_coherent 686 * since this driver can handle the default mask values if 687 * dma_set_mask_and_coherent fails. 688 */ 689 if (capabilities & VMCI_CAPS_PPN64) { 690 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 691 use_ppn64 = true; 692 caps_in_use |= VMCI_CAPS_PPN64; 693 } else { 694 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); 695 use_ppn64 = false; 696 } 697 698 /* 699 * If the hardware supports notifications, we will use that as 700 * well. 701 */ 702 if (capabilities & VMCI_CAPS_NOTIFICATIONS) { 703 vmci_dev->notification_bitmap = dma_alloc_coherent( 704 &pdev->dev, PAGE_SIZE, &vmci_dev->notification_base, 705 GFP_KERNEL); 706 if (!vmci_dev->notification_bitmap) 707 dev_warn(&pdev->dev, 708 "Unable to allocate notification bitmap\n"); 709 else 710 caps_in_use |= VMCI_CAPS_NOTIFICATIONS; 711 } 712 713 if (mmio_base != NULL) { 714 if (capabilities & VMCI_CAPS_DMA_DATAGRAM) { 715 caps_in_use |= VMCI_CAPS_DMA_DATAGRAM; 716 } else { 717 dev_err(&pdev->dev, 718 "Missing capability: VMCI_CAPS_DMA_DATAGRAM\n"); 719 error = -ENXIO; 720 goto err_free_notification_bitmap; 721 } 722 } 723 724 dev_info(&pdev->dev, "Using capabilities 0x%x\n", caps_in_use); 725 726 /* Let the host know which capabilities we intend to use. */ 727 vmci_write_reg(vmci_dev, caps_in_use, VMCI_CAPS_ADDR); 728 729 if (caps_in_use & VMCI_CAPS_DMA_DATAGRAM) { 730 /* Let the device know the size for pages passed down. */ 731 vmci_write_reg(vmci_dev, PAGE_SHIFT, VMCI_GUEST_PAGE_SHIFT); 732 733 /* Configure the high order parts of the data in/out buffers. */ 734 vmci_write_reg(vmci_dev, upper_32_bits(vmci_dev->data_buffer_base), 735 VMCI_DATA_IN_HIGH_ADDR); 736 vmci_write_reg(vmci_dev, upper_32_bits(vmci_dev->tx_buffer_base), 737 VMCI_DATA_OUT_HIGH_ADDR); 738 } 739 740 /* Set up global device so that we can start sending datagrams */ 741 spin_lock_irq(&vmci_dev_spinlock); 742 vmci_dev_g = vmci_dev; 743 vmci_pdev = pdev; 744 spin_unlock_irq(&vmci_dev_spinlock); 745 746 /* 747 * Register notification bitmap with device if that capability is 748 * used. 749 */ 750 if (caps_in_use & VMCI_CAPS_NOTIFICATIONS) { 751 unsigned long bitmap_ppn = 752 vmci_dev->notification_base >> PAGE_SHIFT; 753 if (!vmci_dbell_register_notification_bitmap(bitmap_ppn)) { 754 dev_warn(&pdev->dev, 755 "VMCI device unable to register notification bitmap with PPN 0x%lx\n", 756 bitmap_ppn); 757 error = -ENXIO; 758 goto err_remove_vmci_dev_g; 759 } 760 } 761 762 /* Check host capabilities. */ 763 error = vmci_check_host_caps(pdev); 764 if (error) 765 goto err_remove_vmci_dev_g; 766 767 /* Enable device. */ 768 769 /* 770 * We subscribe to the VMCI_EVENT_CTX_ID_UPDATE here so we can 771 * update the internal context id when needed. 772 */ 773 vmci_err = vmci_event_subscribe(VMCI_EVENT_CTX_ID_UPDATE, 774 vmci_guest_cid_update, NULL, 775 &ctx_update_sub_id); 776 if (vmci_err < VMCI_SUCCESS) 777 dev_warn(&pdev->dev, 778 "Failed to subscribe to event (type=%d): %d\n", 779 VMCI_EVENT_CTX_ID_UPDATE, vmci_err); 780 781 /* 782 * Enable interrupts. Try MSI-X first, then MSI, and then fallback on 783 * legacy interrupts. 784 */ 785 if (vmci_dev->mmio_base != NULL) 786 num_irq_vectors = VMCI_MAX_INTRS; 787 else 788 num_irq_vectors = VMCI_MAX_INTRS_NOTIFICATION; 789 error = pci_alloc_irq_vectors(pdev, num_irq_vectors, num_irq_vectors, 790 PCI_IRQ_MSIX); 791 if (error < 0) { 792 error = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); 793 if (error < 0) 794 goto err_unsubscribe_event; 795 } else { 796 vmci_dev->exclusive_vectors = true; 797 } 798 799 /* 800 * Request IRQ for legacy or MSI interrupts, or for first 801 * MSI-X vector. 802 */ 803 error = request_threaded_irq(pci_irq_vector(pdev, 0), NULL, 804 vmci_interrupt, IRQF_SHARED, 805 KBUILD_MODNAME, vmci_dev); 806 if (error) { 807 dev_err(&pdev->dev, "Irq %u in use: %d\n", 808 pci_irq_vector(pdev, 0), error); 809 goto err_disable_msi; 810 } 811 812 /* 813 * For MSI-X with exclusive vectors we need to request an 814 * interrupt for each vector so that we get a separate 815 * interrupt handler routine. This allows us to distinguish 816 * between the vectors. 817 */ 818 if (vmci_dev->exclusive_vectors) { 819 error = request_threaded_irq(pci_irq_vector(pdev, 1), NULL, 820 vmci_interrupt_bm, 0, 821 KBUILD_MODNAME, vmci_dev); 822 if (error) { 823 dev_err(&pdev->dev, 824 "Failed to allocate irq %u: %d\n", 825 pci_irq_vector(pdev, 1), error); 826 goto err_free_irq; 827 } 828 if (caps_in_use & VMCI_CAPS_DMA_DATAGRAM) { 829 error = request_threaded_irq(pci_irq_vector(pdev, 2), 830 NULL, 831 vmci_interrupt_dma_datagram, 832 0, KBUILD_MODNAME, 833 vmci_dev); 834 if (error) { 835 dev_err(&pdev->dev, 836 "Failed to allocate irq %u: %d\n", 837 pci_irq_vector(pdev, 2), error); 838 goto err_free_bm_irq; 839 } 840 } 841 } 842 843 dev_dbg(&pdev->dev, "Registered device\n"); 844 845 atomic_inc(&vmci_num_guest_devices); 846 847 /* Enable specific interrupt bits. */ 848 cmd = VMCI_IMR_DATAGRAM; 849 if (caps_in_use & VMCI_CAPS_NOTIFICATIONS) 850 cmd |= VMCI_IMR_NOTIFICATION; 851 if (caps_in_use & VMCI_CAPS_DMA_DATAGRAM) 852 cmd |= VMCI_IMR_DMA_DATAGRAM; 853 vmci_write_reg(vmci_dev, cmd, VMCI_IMR_ADDR); 854 855 /* Enable interrupts. */ 856 vmci_write_reg(vmci_dev, VMCI_CONTROL_INT_ENABLE, VMCI_CONTROL_ADDR); 857 858 pci_set_drvdata(pdev, vmci_dev); 859 860 vmci_call_vsock_callback(false); 861 return 0; 862 863 err_free_bm_irq: 864 if (vmci_dev->exclusive_vectors) 865 free_irq(pci_irq_vector(pdev, 1), vmci_dev); 866 867 err_free_irq: 868 free_irq(pci_irq_vector(pdev, 0), vmci_dev); 869 870 err_disable_msi: 871 pci_free_irq_vectors(pdev); 872 873 err_unsubscribe_event: 874 vmci_err = vmci_event_unsubscribe(ctx_update_sub_id); 875 if (vmci_err < VMCI_SUCCESS) 876 dev_warn(&pdev->dev, 877 "Failed to unsubscribe from event (type=%d) with subscriber (ID=0x%x): %d\n", 878 VMCI_EVENT_CTX_ID_UPDATE, ctx_update_sub_id, vmci_err); 879 880 err_remove_vmci_dev_g: 881 spin_lock_irq(&vmci_dev_spinlock); 882 vmci_pdev = NULL; 883 vmci_dev_g = NULL; 884 spin_unlock_irq(&vmci_dev_spinlock); 885 886 err_free_notification_bitmap: 887 if (vmci_dev->notification_bitmap) { 888 vmci_write_reg(vmci_dev, VMCI_CONTROL_RESET, VMCI_CONTROL_ADDR); 889 dma_free_coherent(&pdev->dev, PAGE_SIZE, 890 vmci_dev->notification_bitmap, 891 vmci_dev->notification_base); 892 } 893 894 err_free_data_buffers: 895 vmci_free_dg_buffers(vmci_dev); 896 897 err_unmap_mmio_base: 898 if (mmio_base != NULL) 899 pci_iounmap(pdev, mmio_base); 900 901 /* The rest are managed resources and will be freed by PCI core */ 902 return error; 903 } 904 905 static void vmci_guest_remove_device(struct pci_dev *pdev) 906 { 907 struct vmci_guest_device *vmci_dev = pci_get_drvdata(pdev); 908 int vmci_err; 909 910 dev_dbg(&pdev->dev, "Removing device\n"); 911 912 atomic_dec(&vmci_num_guest_devices); 913 914 vmci_qp_guest_endpoints_exit(); 915 916 vmci_err = vmci_event_unsubscribe(ctx_update_sub_id); 917 if (vmci_err < VMCI_SUCCESS) 918 dev_warn(&pdev->dev, 919 "Failed to unsubscribe from event (type=%d) with subscriber (ID=0x%x): %d\n", 920 VMCI_EVENT_CTX_ID_UPDATE, ctx_update_sub_id, vmci_err); 921 922 spin_lock_irq(&vmci_dev_spinlock); 923 vmci_dev_g = NULL; 924 vmci_pdev = NULL; 925 spin_unlock_irq(&vmci_dev_spinlock); 926 927 dev_dbg(&pdev->dev, "Resetting vmci device\n"); 928 vmci_write_reg(vmci_dev, VMCI_CONTROL_RESET, VMCI_CONTROL_ADDR); 929 930 /* 931 * Free IRQ and then disable MSI/MSI-X as appropriate. For 932 * MSI-X, we might have multiple vectors, each with their own 933 * IRQ, which we must free too. 934 */ 935 if (vmci_dev->exclusive_vectors) { 936 free_irq(pci_irq_vector(pdev, 1), vmci_dev); 937 if (vmci_dev->mmio_base != NULL) 938 free_irq(pci_irq_vector(pdev, 2), vmci_dev); 939 } 940 free_irq(pci_irq_vector(pdev, 0), vmci_dev); 941 pci_free_irq_vectors(pdev); 942 943 if (vmci_dev->notification_bitmap) { 944 /* 945 * The device reset above cleared the bitmap state of the 946 * device, so we can safely free it here. 947 */ 948 949 dma_free_coherent(&pdev->dev, PAGE_SIZE, 950 vmci_dev->notification_bitmap, 951 vmci_dev->notification_base); 952 } 953 954 vmci_free_dg_buffers(vmci_dev); 955 956 if (vmci_dev->mmio_base != NULL) 957 pci_iounmap(pdev, vmci_dev->mmio_base); 958 959 /* The rest are managed resources and will be freed by PCI core */ 960 } 961 962 static const struct pci_device_id vmci_ids[] = { 963 { PCI_DEVICE(PCI_VENDOR_ID_VMWARE, PCI_DEVICE_ID_VMWARE_VMCI), }, 964 { 0 }, 965 }; 966 MODULE_DEVICE_TABLE(pci, vmci_ids); 967 968 static struct pci_driver vmci_guest_driver = { 969 .name = KBUILD_MODNAME, 970 .id_table = vmci_ids, 971 .probe = vmci_guest_probe_device, 972 .remove = vmci_guest_remove_device, 973 }; 974 975 int __init vmci_guest_init(void) 976 { 977 return pci_register_driver(&vmci_guest_driver); 978 } 979 980 void __exit vmci_guest_exit(void) 981 { 982 pci_unregister_driver(&vmci_guest_driver); 983 } 984