1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2024 Marvell. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/interrupt.h> 6 #include <linux/io-64-nonatomic-lo-hi.h> 7 #include <linux/module.h> 8 #include <linux/iommu.h> 9 #include "octep_vdpa.h" 10 11 #define OCTEP_VDPA_DRIVER_NAME "octep_vdpa" 12 #define OCTEP_VDPA_NAME_BUFSIZE 16 13 14 struct octep_pf { 15 u8 __iomem *base[PCI_STD_NUM_BARS]; 16 struct pci_dev *pdev; 17 struct resource res; 18 u64 vf_base; 19 int enabled_vfs; 20 u32 vf_stride; 21 u16 vf_devid; 22 }; 23 24 struct octep_vdpa_event_wk { 25 struct work_struct work; 26 void *ctxptr; 27 }; 28 29 struct octep_vdpa { 30 struct vdpa_device vdpa; 31 struct octep_hw *oct_hw; 32 struct pci_dev *pdev; 33 }; 34 35 struct octep_vdpa_mgmt_dev { 36 struct vdpa_mgmt_dev mdev; 37 struct octep_hw oct_hw; 38 struct pci_dev *pdev; 39 /* Work entry to handle device setup */ 40 struct work_struct setup_task; 41 /* Device status */ 42 atomic_t status; 43 struct octep_vdpa *oct_vdpa; 44 struct octep_vdpa_event_wk event_wk; 45 }; 46 47 static struct octep_hw *vdpa_to_octep_hw(struct vdpa_device *vdpa_dev) 48 { 49 struct octep_vdpa *oct_vdpa; 50 51 oct_vdpa = container_of(vdpa_dev, struct octep_vdpa, vdpa); 52 53 return oct_vdpa->oct_hw; 54 } 55 56 static inline void octep_vdpa_dev_event_schedule(struct octep_hw *oct_hw) 57 { 58 u8 __iomem *addr = oct_hw->base[OCTEP_HW_MBOX_BAR]; 59 struct octep_vdpa_mgmt_dev *mgmt_dev; 60 61 mgmt_dev = container_of(oct_hw, struct octep_vdpa_mgmt_dev, oct_hw); 62 writeb(OCTEP_VDPA_DEV_EVENT_ACTIVE, addr + OCTEP_VF_EVENT_STATE(0)); 63 schedule_work(&mgmt_dev->event_wk.work); 64 } 65 66 static irqreturn_t octep_vdpa_dev_event_handler(int irq, void *data) 67 { 68 struct octep_hw *oct_hw = data; 69 70 if (readb(oct_hw->base[OCTEP_HW_MBOX_BAR] + OCTEP_VF_EVENT_STATE(0)) == 71 OCTEP_VDPA_DEV_NEW_EVENT) 72 octep_vdpa_dev_event_schedule(oct_hw); 73 74 return IRQ_HANDLED; 75 } 76 77 static irqreturn_t octep_vdpa_intr_handler(int irq, void *data) 78 { 79 struct octep_hw *oct_hw = data; 80 int i, start_ring_idx = -1; 81 82 /* Each device has multiple interrupts (nb_irqs) shared among rings 83 * (nr_vring). Device interrupts are mapped to the rings in a 84 * round-robin fashion. 85 * 86 * For example, if nb_irqs = 8 and nr_vring = 64: 87 * 0 -> 0, 8, 16, 24, 32, 40, 48, 56; 88 * 1 -> 1, 9, 17, 25, 33, 41, 49, 57; 89 * ... 90 * 7 -> 7, 15, 23, 31, 39, 47, 55, 63; 91 */ 92 93 for (i = 0; i < oct_hw->nb_irqs; i++) { 94 if (oct_hw->irqs[i] == irq) { 95 start_ring_idx = i; 96 break; 97 } 98 } 99 if (start_ring_idx == -1) 100 return IRQ_NONE; 101 102 for (i = start_ring_idx; i < oct_hw->nr_vring; i += oct_hw->nb_irqs) { 103 if (ioread8(oct_hw->vqs[i].cb_notify_addr)) { 104 /* Acknowledge the per ring notification to the device */ 105 iowrite8(0, oct_hw->vqs[i].cb_notify_addr); 106 107 if (likely(oct_hw->vqs[i].cb.callback)) 108 oct_hw->vqs[i].cb.callback(oct_hw->vqs[i].cb.private); 109 break; 110 } 111 } 112 113 /* Check for config interrupt. Config uses the first interrupt */ 114 if (unlikely(irq == oct_hw->irqs[0])) { 115 if (ioread8(oct_hw->isr)) { 116 iowrite8(0, oct_hw->isr); 117 118 if (oct_hw->config_cb.callback) 119 oct_hw->config_cb.callback(oct_hw->config_cb.private); 120 } 121 octep_vdpa_dev_event_handler(irq, data); 122 } 123 124 return IRQ_HANDLED; 125 } 126 127 static void octep_free_irqs(struct octep_hw *oct_hw) 128 { 129 struct pci_dev *pdev = oct_hw->pdev; 130 int irq; 131 132 if (!oct_hw->irqs) 133 return; 134 135 for (irq = 0; irq < oct_hw->nb_irqs; irq++) { 136 if (!oct_hw->irqs[irq]) 137 break; 138 139 devm_free_irq(&pdev->dev, oct_hw->irqs[irq], oct_hw); 140 } 141 142 pci_free_irq_vectors(pdev); 143 devm_kfree(&pdev->dev, oct_hw->irqs); 144 oct_hw->irqs = NULL; 145 oct_hw->requested_irqs = 0; 146 } 147 148 static int octep_request_irqs(struct octep_hw *oct_hw, irqreturn_t (*irq_handler)(int, void *), 149 int nb_irqs) 150 { 151 struct pci_dev *pdev = oct_hw->pdev; 152 int ret, irq, idx; 153 154 if ((oct_hw->requested_irqs != nb_irqs) || (nb_irqs == 1)) 155 octep_free_irqs(oct_hw); 156 else 157 return 0; 158 159 oct_hw->irqs = devm_kcalloc(&pdev->dev, nb_irqs, sizeof(int), GFP_KERNEL); 160 if (!oct_hw->irqs) 161 return -ENOMEM; 162 163 ret = pci_alloc_irq_vectors(pdev, 1, nb_irqs, PCI_IRQ_MSIX); 164 if (ret < 0) { 165 dev_err(&pdev->dev, "Failed to alloc msix vector"); 166 return ret; 167 } 168 169 for (idx = 0; idx < nb_irqs; idx++) { 170 irq = pci_irq_vector(pdev, idx); 171 ret = devm_request_irq(&pdev->dev, irq, irq_handler, 0, dev_name(&pdev->dev), 172 oct_hw); 173 if (ret) 174 goto free_irqs; 175 oct_hw->irqs[idx] = irq; 176 } 177 oct_hw->requested_irqs = nb_irqs; 178 179 return 0; 180 181 free_irqs: 182 octep_free_irqs(oct_hw); 183 return ret; 184 } 185 186 static u64 octep_vdpa_get_device_features(struct vdpa_device *vdpa_dev) 187 { 188 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 189 190 return oct_hw->features; 191 } 192 193 static int octep_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features) 194 { 195 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 196 int ret; 197 198 pr_debug("Driver Features: %llx\n", features); 199 200 ret = octep_verify_features(features); 201 if (ret) { 202 dev_warn(&oct_hw->pdev->dev, 203 "Must negotiate minimum features 0x%llx for this device", 204 BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) | 205 BIT_ULL(VIRTIO_F_RING_PACKED)); 206 return ret; 207 } 208 octep_hw_set_drv_features(oct_hw, features); 209 210 return 0; 211 } 212 213 static u64 octep_vdpa_get_driver_features(struct vdpa_device *vdpa_dev) 214 { 215 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 216 217 return octep_hw_get_drv_features(oct_hw); 218 } 219 220 static u8 octep_vdpa_get_status(struct vdpa_device *vdpa_dev) 221 { 222 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 223 224 return octep_hw_get_status(oct_hw); 225 } 226 227 static void octep_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) 228 { 229 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 230 u8 status_old; 231 232 status_old = octep_hw_get_status(oct_hw); 233 234 if (status_old == status) 235 return; 236 237 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && 238 !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) { 239 if (octep_request_irqs(oct_hw, octep_vdpa_intr_handler, oct_hw->nb_irqs)) 240 status = status_old | VIRTIO_CONFIG_S_FAILED; 241 } 242 octep_hw_set_status(oct_hw, status); 243 } 244 245 static int octep_vdpa_reset(struct vdpa_device *vdpa_dev) 246 { 247 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 248 u8 status = octep_hw_get_status(oct_hw); 249 u16 qid; 250 251 if (status == 0) 252 return 0; 253 254 for (qid = 0; qid < oct_hw->nr_vring; qid++) { 255 oct_hw->vqs[qid].cb.callback = NULL; 256 oct_hw->vqs[qid].cb.private = NULL; 257 oct_hw->config_cb.callback = NULL; 258 oct_hw->config_cb.private = NULL; 259 } 260 octep_hw_reset(oct_hw); 261 262 if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 263 octep_free_irqs(oct_hw); 264 octep_request_irqs(oct_hw, octep_vdpa_dev_event_handler, 1); 265 } 266 267 return 0; 268 } 269 270 static u16 octep_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) 271 { 272 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 273 274 return octep_get_vq_size(oct_hw); 275 } 276 277 static int octep_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid, 278 struct vdpa_vq_state *state) 279 { 280 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 281 282 return octep_get_vq_state(oct_hw, qid, state); 283 } 284 285 static int octep_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid, 286 const struct vdpa_vq_state *state) 287 { 288 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 289 290 return octep_set_vq_state(oct_hw, qid, state); 291 } 292 293 static void octep_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, struct vdpa_callback *cb) 294 { 295 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 296 297 oct_hw->vqs[qid].cb = *cb; 298 } 299 300 static void octep_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready) 301 { 302 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 303 304 octep_set_vq_ready(oct_hw, qid, ready); 305 } 306 307 static bool octep_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid) 308 { 309 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 310 311 return octep_get_vq_ready(oct_hw, qid); 312 } 313 314 static void octep_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num) 315 { 316 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 317 318 octep_set_vq_num(oct_hw, qid, num); 319 } 320 321 static int octep_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, u64 desc_area, 322 u64 driver_area, u64 device_area) 323 { 324 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 325 326 pr_debug("qid[%d]: desc_area: %llx\n", qid, desc_area); 327 pr_debug("qid[%d]: driver_area: %llx\n", qid, driver_area); 328 pr_debug("qid[%d]: device_area: %llx\n\n", qid, device_area); 329 330 return octep_set_vq_address(oct_hw, qid, desc_area, driver_area, device_area); 331 } 332 333 static void octep_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid) 334 { 335 /* Not supported */ 336 } 337 338 static void octep_vdpa_kick_vq_with_data(struct vdpa_device *vdpa_dev, u32 data) 339 { 340 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 341 u16 idx = data & 0xFFFF; 342 343 vp_iowrite32(data, oct_hw->vqs[idx].notify_addr); 344 } 345 346 static u32 octep_vdpa_get_generation(struct vdpa_device *vdpa_dev) 347 { 348 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 349 350 return vp_ioread8(&oct_hw->common_cfg->config_generation); 351 } 352 353 static u32 octep_vdpa_get_device_id(struct vdpa_device *vdpa_dev) 354 { 355 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 356 357 return oct_hw->dev_id; 358 } 359 360 static u32 octep_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev) 361 { 362 return PCI_VENDOR_ID_CAVIUM; 363 } 364 365 static u32 octep_vdpa_get_vq_align(struct vdpa_device *vdpa_dev) 366 { 367 return PAGE_SIZE; 368 } 369 370 static size_t octep_vdpa_get_config_size(struct vdpa_device *vdpa_dev) 371 { 372 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 373 374 return oct_hw->config_size; 375 } 376 377 static void octep_vdpa_get_config(struct vdpa_device *vdpa_dev, unsigned int offset, void *buf, 378 unsigned int len) 379 { 380 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 381 382 octep_read_dev_config(oct_hw, offset, buf, len); 383 } 384 385 static void octep_vdpa_set_config(struct vdpa_device *vdpa_dev, unsigned int offset, 386 const void *buf, unsigned int len) 387 { 388 /* Not supported */ 389 } 390 391 static void octep_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, struct vdpa_callback *cb) 392 { 393 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 394 395 oct_hw->config_cb.callback = cb->callback; 396 oct_hw->config_cb.private = cb->private; 397 } 398 399 static struct vdpa_notification_area octep_get_vq_notification(struct vdpa_device *vdpa_dev, 400 u16 idx) 401 { 402 struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev); 403 struct vdpa_notification_area area; 404 405 area.addr = oct_hw->vqs[idx].notify_pa; 406 area.size = PAGE_SIZE; 407 408 return area; 409 } 410 411 static struct vdpa_config_ops octep_vdpa_ops = { 412 .get_device_features = octep_vdpa_get_device_features, 413 .set_driver_features = octep_vdpa_set_driver_features, 414 .get_driver_features = octep_vdpa_get_driver_features, 415 .get_status = octep_vdpa_get_status, 416 .set_status = octep_vdpa_set_status, 417 .reset = octep_vdpa_reset, 418 .get_vq_num_max = octep_vdpa_get_vq_num_max, 419 .get_vq_state = octep_vdpa_get_vq_state, 420 .set_vq_state = octep_vdpa_set_vq_state, 421 .set_vq_cb = octep_vdpa_set_vq_cb, 422 .set_vq_ready = octep_vdpa_set_vq_ready, 423 .get_vq_ready = octep_vdpa_get_vq_ready, 424 .set_vq_num = octep_vdpa_set_vq_num, 425 .set_vq_address = octep_vdpa_set_vq_address, 426 .get_vq_irq = NULL, 427 .kick_vq = octep_vdpa_kick_vq, 428 .kick_vq_with_data = octep_vdpa_kick_vq_with_data, 429 .get_generation = octep_vdpa_get_generation, 430 .get_device_id = octep_vdpa_get_device_id, 431 .get_vendor_id = octep_vdpa_get_vendor_id, 432 .get_vq_align = octep_vdpa_get_vq_align, 433 .get_config_size = octep_vdpa_get_config_size, 434 .get_config = octep_vdpa_get_config, 435 .set_config = octep_vdpa_set_config, 436 .set_config_cb = octep_vdpa_set_config_cb, 437 .get_vq_notification = octep_get_vq_notification, 438 }; 439 440 static int octep_iomap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar) 441 { 442 int ret; 443 444 ret = pci_request_region(pdev, bar, OCTEP_VDPA_DRIVER_NAME); 445 if (ret) { 446 dev_err(&pdev->dev, "Failed to request BAR:%u region\n", bar); 447 return ret; 448 } 449 450 tbl[bar] = pci_iomap(pdev, bar, pci_resource_len(pdev, bar)); 451 if (!tbl[bar]) { 452 dev_err(&pdev->dev, "Failed to iomap BAR:%u\n", bar); 453 pci_release_region(pdev, bar); 454 ret = -ENOMEM; 455 } 456 457 return ret; 458 } 459 460 static void octep_iounmap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar) 461 { 462 pci_iounmap(pdev, tbl[bar]); 463 pci_release_region(pdev, bar); 464 } 465 466 static void octep_vdpa_pf_bar_shrink(struct octep_pf *octpf) 467 { 468 struct pci_dev *pf_dev = octpf->pdev; 469 struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4; 470 struct pci_bus_region bus_region; 471 472 octpf->res.start = res->start; 473 octpf->res.end = res->end; 474 octpf->vf_base = res->start; 475 476 bus_region.start = res->start; 477 bus_region.end = res->start - 1; 478 479 pcibios_bus_to_resource(pf_dev->bus, res, &bus_region); 480 } 481 482 static void octep_vdpa_pf_bar_expand(struct octep_pf *octpf) 483 { 484 struct pci_dev *pf_dev = octpf->pdev; 485 struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4; 486 struct pci_bus_region bus_region; 487 488 bus_region.start = octpf->res.start; 489 bus_region.end = octpf->res.end; 490 491 pcibios_bus_to_resource(pf_dev->bus, res, &bus_region); 492 } 493 494 static void octep_vdpa_remove_pf(struct pci_dev *pdev) 495 { 496 struct octep_pf *octpf = pci_get_drvdata(pdev); 497 498 pci_disable_sriov(pdev); 499 500 if (octpf->base[OCTEP_HW_CAPS_BAR]) 501 octep_iounmap_region(pdev, octpf->base, OCTEP_HW_CAPS_BAR); 502 503 if (octpf->base[OCTEP_HW_MBOX_BAR]) 504 octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR); 505 506 octep_vdpa_pf_bar_expand(octpf); 507 508 /* The pf version does not use managed PCI. */ 509 pci_disable_device(pdev); 510 } 511 512 static void octep_vdpa_vf_bar_shrink(struct pci_dev *pdev) 513 { 514 struct resource *vf_res = pdev->resource + PCI_STD_RESOURCES + 4; 515 516 memset(vf_res, 0, sizeof(*vf_res)); 517 } 518 519 static void octep_vdpa_remove_vf(struct pci_dev *pdev) 520 { 521 struct octep_vdpa_mgmt_dev *mgmt_dev = pci_get_drvdata(pdev); 522 struct octep_hw *oct_hw; 523 int status; 524 525 oct_hw = &mgmt_dev->oct_hw; 526 status = atomic_read(&mgmt_dev->status); 527 atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_UNINIT); 528 529 cancel_work_sync(&mgmt_dev->setup_task); 530 if ((status == OCTEP_VDPA_DEV_STATUS_READY) || (status == OCTEP_VDPA_DEV_STATUS_ADDED) || 531 (status == OCTEP_VDPA_DEV_STATUS_REMOVED)) 532 vdpa_mgmtdev_unregister(&mgmt_dev->mdev); 533 534 if (oct_hw->base[OCTEP_HW_CAPS_BAR]) 535 octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR); 536 537 if (oct_hw->base[OCTEP_HW_MBOX_BAR]) 538 octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_MBOX_BAR); 539 540 octep_vdpa_vf_bar_shrink(pdev); 541 octep_free_irqs(oct_hw); 542 } 543 544 static void octep_vdpa_remove(struct pci_dev *pdev) 545 { 546 if (pdev->is_virtfn) 547 octep_vdpa_remove_vf(pdev); 548 else 549 octep_vdpa_remove_pf(pdev); 550 } 551 552 static int octep_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 553 const struct vdpa_dev_set_config *config) 554 { 555 struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(mdev, struct octep_vdpa_mgmt_dev, mdev); 556 struct octep_hw *oct_hw = &mgmt_dev->oct_hw; 557 struct pci_dev *pdev = oct_hw->pdev; 558 struct vdpa_device *vdpa_dev; 559 struct octep_vdpa *oct_vdpa; 560 u64 device_features; 561 int ret; 562 563 oct_vdpa = vdpa_alloc_device(struct octep_vdpa, vdpa, &pdev->dev, &octep_vdpa_ops, 564 NULL, 1, 1, NULL, false); 565 if (IS_ERR(oct_vdpa)) { 566 dev_err(&pdev->dev, "Failed to allocate vDPA structure for octep vdpa device"); 567 return PTR_ERR(oct_vdpa); 568 } 569 570 oct_vdpa->pdev = pdev; 571 oct_vdpa->vdpa.vmap.dma_dev = &pdev->dev; 572 oct_vdpa->vdpa.mdev = mdev; 573 oct_vdpa->oct_hw = oct_hw; 574 vdpa_dev = &oct_vdpa->vdpa; 575 mgmt_dev->oct_vdpa = oct_vdpa; 576 577 device_features = oct_hw->features; 578 if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { 579 if (config->device_features & ~device_features) { 580 dev_err(&pdev->dev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n", 581 config->device_features, device_features); 582 ret = -EINVAL; 583 goto vdpa_dev_put; 584 } 585 device_features &= config->device_features; 586 } 587 588 oct_hw->features = device_features; 589 dev_info(&pdev->dev, "Vdpa management device features : %llx\n", device_features); 590 591 ret = octep_verify_features(device_features); 592 if (ret) { 593 dev_warn(mdev->device, 594 "Must provision minimum features 0x%llx for this device", 595 BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | 596 BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) | BIT_ULL(VIRTIO_F_RING_PACKED)); 597 goto vdpa_dev_put; 598 } 599 if (name) 600 ret = dev_set_name(&vdpa_dev->dev, "%s", name); 601 else 602 ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index); 603 if (ret) 604 goto vdpa_dev_put; 605 606 ret = _vdpa_register_device(&oct_vdpa->vdpa, oct_hw->nr_vring); 607 if (ret) { 608 dev_err(&pdev->dev, "Failed to register to vDPA bus"); 609 goto vdpa_dev_put; 610 } 611 atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_ADDED); 612 return 0; 613 614 vdpa_dev_put: 615 put_device(&oct_vdpa->vdpa.dev); 616 return ret; 617 } 618 619 static void octep_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *vdpa_dev) 620 { 621 struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(mdev, struct octep_vdpa_mgmt_dev, mdev); 622 _vdpa_unregister_device(vdpa_dev); 623 atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_REMOVED); 624 } 625 626 static const struct vdpa_mgmtdev_ops octep_vdpa_mgmt_dev_ops = { 627 .dev_add = octep_vdpa_dev_add, 628 .dev_del = octep_vdpa_dev_del 629 }; 630 631 static bool get_device_ready_status(u8 __iomem *addr) 632 { 633 u32 signature = readl(addr + OCTEP_VF_MBOX_DATA(0)); 634 635 if (signature == OCTEP_DEV_READY_SIGNATURE) { 636 writel(0, addr + OCTEP_VF_MBOX_DATA(0)); 637 return true; 638 } 639 640 return false; 641 } 642 643 static struct virtio_device_id id_table[] = { 644 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 645 { 0 }, 646 }; 647 648 static void octep_event_work(struct work_struct *work) 649 { 650 struct octep_vdpa_event_wk *wk = container_of(work, struct octep_vdpa_event_wk, work); 651 struct octep_vdpa_mgmt_dev *mgmt_dev = (struct octep_vdpa_mgmt_dev *)wk->ctxptr; 652 u8 __iomem *addr = mgmt_dev->oct_hw.base[OCTEP_HW_MBOX_BAR]; 653 u8 event = readb(addr + OCTEP_VF_EVENT_REG(0)); 654 struct vdpa_dev_set_config config = {0}; 655 char name[OCTEP_VDPA_NAME_BUFSIZE]; 656 int ret = 0; 657 658 switch (event) { 659 case OCTEP_VDPA_DEV_ADD_EVENT: 660 if (atomic_read(&mgmt_dev->status) != OCTEP_VDPA_DEV_STATUS_ADDED) { 661 snprintf(name, sizeof(name), "%s-%x", "vdpa", mgmt_dev->pdev->devfn); 662 ret = octep_vdpa_dev_add(&mgmt_dev->mdev, name, &config); 663 } 664 break; 665 case OCTEP_VDPA_DEV_DEL_EVENT: 666 if (atomic_read(&mgmt_dev->status) == OCTEP_VDPA_DEV_STATUS_ADDED) 667 octep_vdpa_dev_del(&mgmt_dev->mdev, &mgmt_dev->oct_vdpa->vdpa); 668 break; 669 default: 670 break; 671 } 672 673 event = ret ? OCTEP_VDPA_DEV_EVENT_NACK : OCTEP_VDPA_DEV_EVENT_ACK; 674 writeb(event, addr + OCTEP_VF_EVENT_REG(0)); 675 writeb(OCTEP_VDPA_DEV_EVENT_DONE, addr + OCTEP_VF_EVENT_STATE(0)); 676 } 677 678 static void octep_vdpa_setup_task(struct work_struct *work) 679 { 680 struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(work, struct octep_vdpa_mgmt_dev, 681 setup_task); 682 struct pci_dev *pdev = mgmt_dev->pdev; 683 struct device *dev = &pdev->dev; 684 struct octep_hw *oct_hw; 685 unsigned long timeout; 686 u64 val; 687 int ret; 688 689 oct_hw = &mgmt_dev->oct_hw; 690 691 atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT); 692 693 /* Wait for a maximum of 5 sec */ 694 timeout = jiffies + msecs_to_jiffies(5000); 695 while (!time_after(jiffies, timeout)) { 696 if (get_device_ready_status(oct_hw->base[OCTEP_HW_MBOX_BAR])) { 697 atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_INIT); 698 break; 699 } 700 701 if (atomic_read(&mgmt_dev->status) >= OCTEP_VDPA_DEV_STATUS_READY) { 702 dev_info(dev, "Stopping vDPA setup task.\n"); 703 return; 704 } 705 706 usleep_range(1000, 1500); 707 } 708 709 if (atomic_read(&mgmt_dev->status) != OCTEP_VDPA_DEV_STATUS_INIT) { 710 dev_err(dev, "BAR initialization is timed out\n"); 711 return; 712 } 713 714 ret = octep_iomap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR); 715 if (ret) 716 return; 717 718 val = readq(oct_hw->base[OCTEP_HW_MBOX_BAR] + OCTEP_VF_IN_CTRL(0)); 719 oct_hw->nb_irqs = OCTEP_VF_IN_CTRL_RPVF(val); 720 if (!oct_hw->nb_irqs || oct_hw->nb_irqs > OCTEP_MAX_CB_INTR) { 721 dev_err(dev, "Invalid number of interrupts %d\n", oct_hw->nb_irqs); 722 goto unmap_region; 723 } 724 725 ret = octep_hw_caps_read(oct_hw, pdev); 726 if (ret < 0) 727 goto unmap_region; 728 729 mgmt_dev->mdev.ops = &octep_vdpa_mgmt_dev_ops; 730 mgmt_dev->mdev.id_table = id_table; 731 mgmt_dev->mdev.max_supported_vqs = oct_hw->nr_vring; 732 mgmt_dev->mdev.supported_features = oct_hw->features; 733 mgmt_dev->mdev.config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES); 734 mgmt_dev->mdev.device = dev; 735 736 ret = vdpa_mgmtdev_register(&mgmt_dev->mdev); 737 if (ret) { 738 dev_err(dev, "Failed to register vdpa management interface\n"); 739 goto unmap_region; 740 } 741 742 atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_READY); 743 INIT_WORK(&mgmt_dev->event_wk.work, octep_event_work); 744 mgmt_dev->event_wk.ctxptr = mgmt_dev; 745 octep_request_irqs(&mgmt_dev->oct_hw, octep_vdpa_dev_event_handler, 1); 746 747 return; 748 749 unmap_region: 750 octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR); 751 oct_hw->base[OCTEP_HW_CAPS_BAR] = NULL; 752 } 753 754 static int octep_vdpa_probe_vf(struct pci_dev *pdev) 755 { 756 struct octep_vdpa_mgmt_dev *mgmt_dev; 757 struct device *dev = &pdev->dev; 758 int ret; 759 760 ret = pcim_enable_device(pdev); 761 if (ret) { 762 dev_err(dev, "Failed to enable device\n"); 763 return ret; 764 } 765 766 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 767 if (ret) { 768 dev_err(dev, "No usable DMA configuration\n"); 769 return ret; 770 } 771 pci_set_master(pdev); 772 773 mgmt_dev = devm_kzalloc(dev, sizeof(struct octep_vdpa_mgmt_dev), GFP_KERNEL); 774 if (!mgmt_dev) 775 return -ENOMEM; 776 777 ret = octep_iomap_region(pdev, mgmt_dev->oct_hw.base, OCTEP_HW_MBOX_BAR); 778 if (ret) 779 return ret; 780 781 mgmt_dev->pdev = pdev; 782 pci_set_drvdata(pdev, mgmt_dev); 783 784 atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_ALLOC); 785 INIT_WORK(&mgmt_dev->setup_task, octep_vdpa_setup_task); 786 schedule_work(&mgmt_dev->setup_task); 787 dev_info(&pdev->dev, "octep vdpa mgmt device setup task is queued\n"); 788 789 return 0; 790 } 791 792 static void octep_vdpa_assign_barspace(struct pci_dev *vf_dev, struct pci_dev *pf_dev, u8 idx) 793 { 794 struct resource *vf_res = vf_dev->resource + PCI_STD_RESOURCES + 4; 795 struct resource *pf_res = pf_dev->resource + PCI_STD_RESOURCES + 4; 796 struct octep_pf *pf = pci_get_drvdata(pf_dev); 797 struct pci_bus_region bus_region; 798 799 vf_res->name = pci_name(vf_dev); 800 vf_res->flags = pf_res->flags; 801 vf_res->parent = (pf_dev->resource + PCI_STD_RESOURCES)->parent; 802 803 bus_region.start = pf->vf_base + idx * pf->vf_stride; 804 bus_region.end = bus_region.start + pf->vf_stride - 1; 805 pcibios_bus_to_resource(vf_dev->bus, vf_res, &bus_region); 806 } 807 808 static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs) 809 { 810 struct octep_pf *pf = pci_get_drvdata(pdev); 811 u8 __iomem *addr = pf->base[OCTEP_HW_MBOX_BAR]; 812 struct pci_dev *vf_pdev = NULL; 813 bool done = false; 814 int index = 0; 815 int ret, i; 816 u8 rpvf; 817 u64 val; 818 819 ret = pci_enable_sriov(pdev, num_vfs); 820 if (ret) 821 return ret; 822 823 pf->enabled_vfs = num_vfs; 824 825 while ((vf_pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_ANY_ID, vf_pdev))) { 826 if (vf_pdev->device != pf->vf_devid) 827 continue; 828 829 octep_vdpa_assign_barspace(vf_pdev, pdev, index); 830 if (++index == num_vfs) { 831 done = true; 832 pci_dev_put(vf_pdev); 833 break; 834 } 835 } 836 837 val = readq(addr + OCTEP_EPF_RINFO(0)); 838 rpvf = FIELD_GET(GENMASK_ULL(35, 32), val); 839 if (done) { 840 for (i = 0; i < pf->enabled_vfs; i++) 841 writel(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i * rpvf)); 842 } 843 844 return num_vfs; 845 } 846 847 static int octep_sriov_disable(struct pci_dev *pdev) 848 { 849 struct octep_pf *pf = pci_get_drvdata(pdev); 850 851 if (!pci_num_vf(pdev)) 852 return 0; 853 854 pci_disable_sriov(pdev); 855 pf->enabled_vfs = 0; 856 857 return 0; 858 } 859 860 static int octep_vdpa_sriov_configure(struct pci_dev *pdev, int num_vfs) 861 { 862 if (num_vfs > 0) 863 return octep_sriov_enable(pdev, num_vfs); 864 else 865 return octep_sriov_disable(pdev); 866 } 867 868 static u16 octep_get_vf_devid(struct pci_dev *pdev) 869 { 870 u16 did; 871 872 switch (pdev->device) { 873 case OCTEP_VDPA_DEVID_CN106K_PF: 874 did = OCTEP_VDPA_DEVID_CN106K_VF; 875 break; 876 case OCTEP_VDPA_DEVID_CN105K_PF: 877 did = OCTEP_VDPA_DEVID_CN105K_VF; 878 break; 879 case OCTEP_VDPA_DEVID_CN103K_PF: 880 did = OCTEP_VDPA_DEVID_CN103K_VF; 881 break; 882 default: 883 did = 0xFFFF; 884 break; 885 } 886 887 return did; 888 } 889 890 static int octep_vdpa_pf_setup(struct octep_pf *octpf) 891 { 892 u8 __iomem *addr = octpf->base[OCTEP_HW_MBOX_BAR]; 893 struct pci_dev *pdev = octpf->pdev; 894 int totalvfs; 895 size_t len; 896 u64 val; 897 898 totalvfs = pci_sriov_get_totalvfs(pdev); 899 if (unlikely(!totalvfs)) { 900 dev_info(&pdev->dev, "Total VFs are %d in PF sriov configuration\n", totalvfs); 901 return 0; 902 } 903 904 addr = octpf->base[OCTEP_HW_MBOX_BAR]; 905 val = readq(addr + OCTEP_EPF_RINFO(0)); 906 if (val == 0) { 907 dev_err(&pdev->dev, "Invalid device configuration\n"); 908 return -EINVAL; 909 } 910 911 len = pci_resource_len(pdev, OCTEP_HW_CAPS_BAR); 912 913 octpf->vf_stride = len / totalvfs; 914 octpf->vf_devid = octep_get_vf_devid(pdev); 915 916 octep_vdpa_pf_bar_shrink(octpf); 917 918 return 0; 919 } 920 921 static int octep_vdpa_probe_pf(struct pci_dev *pdev) 922 { 923 struct device *dev = &pdev->dev; 924 struct octep_pf *octpf; 925 int ret; 926 927 ret = pci_enable_device(pdev); 928 if (ret) { 929 dev_err(dev, "Failed to enable device\n"); 930 return ret; 931 } 932 933 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 934 if (ret) { 935 dev_err(dev, "No usable DMA configuration\n"); 936 goto disable_pci; 937 } 938 octpf = devm_kzalloc(dev, sizeof(*octpf), GFP_KERNEL); 939 if (!octpf) { 940 ret = -ENOMEM; 941 goto disable_pci; 942 } 943 944 ret = octep_iomap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR); 945 if (ret) 946 goto disable_pci; 947 948 pci_set_master(pdev); 949 pci_set_drvdata(pdev, octpf); 950 octpf->pdev = pdev; 951 952 ret = octep_vdpa_pf_setup(octpf); 953 if (ret) 954 goto unmap_region; 955 956 return 0; 957 958 unmap_region: 959 octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR); 960 disable_pci: 961 pci_disable_device(pdev); 962 return ret; 963 } 964 965 static int octep_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) 966 { 967 if (pdev->is_virtfn) 968 return octep_vdpa_probe_vf(pdev); 969 else 970 return octep_vdpa_probe_pf(pdev); 971 } 972 973 static struct pci_device_id octep_pci_vdpa_map[] = { 974 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_PF) }, 975 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_VF) }, 976 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_PF) }, 977 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_VF) }, 978 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_PF) }, 979 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_VF) }, 980 { 0 }, 981 }; 982 MODULE_DEVICE_TABLE(pci, octep_pci_vdpa_map); 983 984 static struct pci_driver octep_pci_vdpa = { 985 .name = OCTEP_VDPA_DRIVER_NAME, 986 .id_table = octep_pci_vdpa_map, 987 .probe = octep_vdpa_probe, 988 .remove = octep_vdpa_remove, 989 .sriov_configure = octep_vdpa_sriov_configure 990 }; 991 992 module_pci_driver(octep_pci_vdpa); 993 994 MODULE_AUTHOR("Marvell"); 995 MODULE_DESCRIPTION("Marvell Octeon PCIe endpoint vDPA driver"); 996 MODULE_LICENSE("GPL"); 997