1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6 #include <linux/pci.h> 7 8 #include <linux/pds/pds_common.h> 9 10 #include "core.h" 11 12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION); 13 MODULE_AUTHOR("Advanced Micro Devices, Inc"); 14 MODULE_LICENSE("GPL"); 15 16 /* Supported devices */ 17 static const struct pci_device_id pdsc_id_table[] = { 18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) }, 19 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) }, 20 { 0, } /* end of table */ 21 }; 22 MODULE_DEVICE_TABLE(pci, pdsc_id_table); 23 24 static void pdsc_stop_health_thread(struct pdsc *pdsc); 25 26 static void pdsc_wdtimer_cb(struct timer_list *t) 27 { 28 struct pdsc *pdsc = timer_container_of(pdsc, t, wdtimer); 29 30 dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies); 31 mod_timer(&pdsc->wdtimer, 32 round_jiffies(jiffies + pdsc->wdtimer_period)); 33 34 queue_work(pdsc->wq, &pdsc->health_work); 35 } 36 37 static void pdsc_unmap_bars(struct pdsc *pdsc) 38 { 39 struct pdsc_dev_bar *bars = pdsc->bars; 40 unsigned int i; 41 42 pdsc->info_regs = NULL; 43 pdsc->cmd_regs = NULL; 44 pdsc->intr_status = NULL; 45 pdsc->intr_ctrl = NULL; 46 47 for (i = 0; i < PDS_CORE_BARS_MAX; i++) { 48 if (bars[i].vaddr) 49 pci_iounmap(pdsc->pdev, bars[i].vaddr); 50 bars[i].vaddr = NULL; 51 } 52 } 53 54 static int pdsc_map_bars(struct pdsc *pdsc) 55 { 56 struct pdsc_dev_bar *bar = pdsc->bars; 57 struct pci_dev *pdev = pdsc->pdev; 58 struct device *dev = pdsc->dev; 59 struct pdsc_dev_bar *bars; 60 unsigned int i, j; 61 int num_bars = 0; 62 int err; 63 u32 sig; 64 65 bars = pdsc->bars; 66 67 /* Since the PCI interface in the hardware is configurable, 68 * we need to poke into all the bars to find the set we're 69 * expecting. 70 */ 71 for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) { 72 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 73 continue; 74 75 bars[j].len = pci_resource_len(pdev, i); 76 bars[j].bus_addr = pci_resource_start(pdev, i); 77 bars[j].res_index = i; 78 79 /* only map the whole bar 0 */ 80 if (j > 0) { 81 bars[j].vaddr = NULL; 82 } else { 83 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 84 if (!bars[j].vaddr) { 85 dev_err(dev, "Cannot map BAR %d, aborting\n", i); 86 return -ENODEV; 87 } 88 } 89 90 j++; 91 } 92 num_bars = j; 93 94 /* BAR0: dev_cmd and interrupts */ 95 if (num_bars < 1) { 96 dev_err(dev, "No bars found\n"); 97 err = -EFAULT; 98 goto err_out; 99 } 100 101 if (bar->len < PDS_CORE_BAR0_SIZE) { 102 dev_err(dev, "Resource bar size %lu too small\n", bar->len); 103 err = -EFAULT; 104 goto err_out; 105 } 106 107 pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET; 108 pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET; 109 pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET; 110 pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET; 111 112 sig = ioread32(&pdsc->info_regs->signature); 113 if (sig != PDS_CORE_DEV_INFO_SIGNATURE) { 114 dev_err(dev, "Incompatible firmware signature %x", sig); 115 err = -EFAULT; 116 goto err_out; 117 } 118 119 /* BAR1: doorbells */ 120 bar++; 121 if (num_bars < 2) { 122 dev_err(dev, "Doorbell bar missing\n"); 123 err = -EFAULT; 124 goto err_out; 125 } 126 127 pdsc->db_pages = bar->vaddr; 128 pdsc->phy_db_pages = bar->bus_addr; 129 130 return 0; 131 132 err_out: 133 pdsc_unmap_bars(pdsc); 134 return err; 135 } 136 137 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num) 138 { 139 return pci_iomap_range(pdsc->pdev, 140 pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index, 141 (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 142 } 143 144 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs) 145 { 146 struct pdsc *pdsc = pci_get_drvdata(pdev); 147 struct device *dev = pdsc->dev; 148 int ret = 0; 149 150 if (num_vfs > 0) { 151 pdsc->vfs = kzalloc_objs(struct pdsc_vf, num_vfs); 152 if (!pdsc->vfs) 153 return -ENOMEM; 154 pdsc->num_vfs = num_vfs; 155 156 ret = pci_enable_sriov(pdev, num_vfs); 157 if (ret) { 158 dev_err(dev, "Cannot enable SRIOV: %pe\n", 159 ERR_PTR(ret)); 160 goto no_vfs; 161 } 162 163 return num_vfs; 164 } 165 166 no_vfs: 167 pci_disable_sriov(pdev); 168 169 kfree(pdsc->vfs); 170 pdsc->vfs = NULL; 171 pdsc->num_vfs = 0; 172 173 return ret; 174 } 175 176 static int pdsc_init_vf(struct pdsc *vf) 177 { 178 struct devlink *dl; 179 struct pdsc *pf; 180 int err; 181 182 pf = pdsc_get_pf_struct(vf->pdev); 183 if (IS_ERR_OR_NULL(pf)) 184 return PTR_ERR(pf) ?: -1; 185 186 vf->vf_id = pci_iov_vf_id(vf->pdev); 187 188 dl = priv_to_devlink(vf); 189 devl_lock(dl); 190 devl_register(dl); 191 devl_unlock(dl); 192 193 pf->vfs[vf->vf_id].vf = vf; 194 err = pdsc_auxbus_dev_add(vf, pf, PDS_DEV_TYPE_VDPA, 195 &pf->vfs[vf->vf_id].padev); 196 if (err) { 197 devl_lock(dl); 198 devl_unregister(dl); 199 devl_unlock(dl); 200 } 201 202 return err; 203 } 204 205 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = { 206 .name = "fw", 207 .diagnose = pdsc_fw_reporter_diagnose, 208 }; 209 210 static const struct devlink_param pdsc_dl_params[] = { 211 DEVLINK_PARAM_GENERIC(ENABLE_VNET, 212 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 213 pdsc_dl_enable_get, 214 pdsc_dl_enable_set, 215 pdsc_dl_enable_validate), 216 }; 217 218 #define PDSC_WQ_NAME_LEN 24 219 220 static int pdsc_init_pf(struct pdsc *pdsc) 221 { 222 struct devlink_health_reporter *hr; 223 char wq_name[PDSC_WQ_NAME_LEN]; 224 struct devlink *dl; 225 int err; 226 227 pcie_print_link_status(pdsc->pdev); 228 229 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 230 if (err) { 231 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 232 ERR_PTR(err)); 233 return err; 234 } 235 236 err = pdsc_map_bars(pdsc); 237 if (err) 238 goto err_out_release_regions; 239 240 /* General workqueue and timer, but don't start timer yet */ 241 snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid); 242 pdsc->wq = create_singlethread_workqueue(wq_name); 243 if (!pdsc->wq) { 244 err = -ENOMEM; 245 goto err_out_unmap_bars; 246 } 247 INIT_WORK(&pdsc->health_work, pdsc_health_thread); 248 INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread); 249 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); 250 pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ; 251 252 mutex_init(&pdsc->devcmd_lock); 253 mutex_init(&pdsc->config_lock); 254 spin_lock_init(&pdsc->adminq_lock); 255 256 mutex_lock(&pdsc->config_lock); 257 set_bit(PDSC_S_FW_DEAD, &pdsc->state); 258 259 err = pdsc_setup(pdsc, PDSC_SETUP_INIT); 260 if (err) { 261 mutex_unlock(&pdsc->config_lock); 262 goto err_out_shutdown_timer; 263 } 264 265 err = pdsc_start(pdsc); 266 if (err) { 267 mutex_unlock(&pdsc->config_lock); 268 goto err_out_teardown; 269 } 270 271 mutex_unlock(&pdsc->config_lock); 272 273 err = pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL, &pdsc->padev); 274 if (err) 275 goto err_out_stop; 276 277 dl = priv_to_devlink(pdsc); 278 devl_lock(dl); 279 err = devl_params_register(dl, pdsc_dl_params, 280 ARRAY_SIZE(pdsc_dl_params)); 281 if (err) { 282 devl_unlock(dl); 283 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n", 284 ERR_PTR(err)); 285 goto err_out_del_dev; 286 } 287 288 hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, pdsc); 289 if (IS_ERR(hr)) { 290 devl_unlock(dl); 291 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr); 292 err = PTR_ERR(hr); 293 goto err_out_unreg_params; 294 } 295 pdsc->fw_reporter = hr; 296 297 devl_register(dl); 298 devl_unlock(dl); 299 300 /* Lastly, start the health check timer */ 301 mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period)); 302 303 return 0; 304 305 err_out_unreg_params: 306 devlink_params_unregister(dl, pdsc_dl_params, 307 ARRAY_SIZE(pdsc_dl_params)); 308 err_out_del_dev: 309 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); 310 err_out_stop: 311 pdsc_stop(pdsc); 312 err_out_teardown: 313 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 314 err_out_shutdown_timer: 315 timer_shutdown_sync(&pdsc->wdtimer); 316 if (pdsc->wq) 317 destroy_workqueue(pdsc->wq); 318 mutex_destroy(&pdsc->config_lock); 319 mutex_destroy(&pdsc->devcmd_lock); 320 pdsc_deferred_dma_free(pdsc); 321 pci_free_irq_vectors(pdsc->pdev); 322 err_out_unmap_bars: 323 pdsc_unmap_bars(pdsc); 324 err_out_release_regions: 325 pci_release_regions(pdsc->pdev); 326 327 return err; 328 } 329 330 static const struct devlink_ops pdsc_dl_ops = { 331 .info_get = pdsc_dl_info_get, 332 .flash_update = pdsc_dl_flash_update, 333 }; 334 335 static const struct devlink_ops pdsc_dl_vf_ops = { 336 }; 337 338 static DEFINE_IDA(pdsc_ida); 339 340 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 341 { 342 struct device *dev = &pdev->dev; 343 const struct devlink_ops *ops; 344 struct devlink *dl; 345 struct pdsc *pdsc; 346 bool is_pf; 347 int err; 348 349 is_pf = !pdev->is_virtfn; 350 ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops; 351 dl = devlink_alloc(ops, sizeof(struct pdsc), dev); 352 if (!dl) 353 return -ENOMEM; 354 pdsc = devlink_priv(dl); 355 356 pdsc->pdev = pdev; 357 pdsc->dev = &pdev->dev; 358 set_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 359 INIT_LIST_HEAD(&pdsc->deferred_dma_list); 360 spin_lock_init(&pdsc->deferred_dma_lock); 361 pci_set_drvdata(pdev, pdsc); 362 pdsc_debugfs_add_dev(pdsc); 363 364 err = ida_alloc(&pdsc_ida, GFP_KERNEL); 365 if (err < 0) { 366 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n", 367 __func__, ERR_PTR(err)); 368 goto err_out_free_devlink; 369 } 370 pdsc->uid = err; 371 372 /* Query system for DMA addressing limitation for the device. */ 373 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN)); 374 if (err) { 375 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n", 376 ERR_PTR(err)); 377 goto err_out_free_ida; 378 } 379 380 err = pci_enable_device(pdev); 381 if (err) { 382 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 383 goto err_out_free_ida; 384 } 385 pci_set_master(pdev); 386 387 if (is_pf) 388 err = pdsc_init_pf(pdsc); 389 else 390 err = pdsc_init_vf(pdsc); 391 if (err) { 392 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err)); 393 goto err_out_disable_device; 394 } 395 396 clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 397 return 0; 398 399 err_out_disable_device: 400 pci_disable_device(pdev); 401 err_out_free_ida: 402 ida_free(&pdsc_ida, pdsc->uid); 403 err_out_free_devlink: 404 pdsc_debugfs_del_dev(pdsc); 405 devlink_free(dl); 406 407 return err; 408 } 409 410 static void pdsc_remove(struct pci_dev *pdev) 411 { 412 struct pdsc *pdsc = pci_get_drvdata(pdev); 413 struct devlink *dl; 414 415 /* Unhook the registrations first to be sure there 416 * are no requests while we're stopping. 417 */ 418 dl = priv_to_devlink(pdsc); 419 devl_lock(dl); 420 devl_unregister(dl); 421 if (!pdev->is_virtfn) { 422 if (pdsc->fw_reporter) { 423 devl_health_reporter_destroy(pdsc->fw_reporter); 424 pdsc->fw_reporter = NULL; 425 } 426 devl_params_unregister(dl, pdsc_dl_params, 427 ARRAY_SIZE(pdsc_dl_params)); 428 } 429 devl_unlock(dl); 430 431 if (pdev->is_virtfn) { 432 struct pdsc *pf; 433 434 pf = pdsc_get_pf_struct(pdsc->pdev); 435 if (!IS_ERR(pf)) { 436 pdsc_auxbus_dev_del(pdsc, pf, &pf->vfs[pdsc->vf_id].padev); 437 pf->vfs[pdsc->vf_id].vf = NULL; 438 } 439 } else { 440 /* Remove the VFs and their aux_bus connections before other 441 * cleanup so that the clients can use the AdminQ to cleanly 442 * shut themselves down. 443 */ 444 pdsc_sriov_configure(pdev, 0); 445 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); 446 447 timer_shutdown_sync(&pdsc->wdtimer); 448 449 mutex_lock(&pdsc->config_lock); 450 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state); 451 452 pdsc_stop(pdsc); 453 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 454 mutex_unlock(&pdsc->config_lock); 455 456 if (pdsc->wq) 457 destroy_workqueue(pdsc->wq); 458 mutex_destroy(&pdsc->config_lock); 459 mutex_destroy(&pdsc->devcmd_lock); 460 461 pdsc_unmap_bars(pdsc); 462 pci_release_regions(pdev); 463 } 464 465 pci_disable_device(pdev); 466 pdsc_deferred_dma_free(pdsc); 467 468 ida_free(&pdsc_ida, pdsc->uid); 469 pdsc_debugfs_del_dev(pdsc); 470 devlink_free(dl); 471 } 472 473 static void pdsc_stop_health_thread(struct pdsc *pdsc) 474 { 475 if (pdsc->pdev->is_virtfn) 476 return; 477 478 timer_shutdown_sync(&pdsc->wdtimer); 479 if (pdsc->health_work.func && !pdsc->health_stopped) { 480 disable_work_sync(&pdsc->health_work); 481 pdsc->health_stopped = true; 482 } 483 } 484 485 static void pdsc_restart_health_thread(struct pdsc *pdsc) 486 { 487 if (pdsc->pdev->is_virtfn) 488 return; 489 490 if (pdsc->health_stopped) { 491 enable_work(&pdsc->health_work); 492 pdsc->health_stopped = false; 493 } 494 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); 495 mod_timer(&pdsc->wdtimer, jiffies + 1); 496 } 497 498 static void pdsc_reset_prepare(struct pci_dev *pdev) 499 { 500 struct pdsc *pdsc = pci_get_drvdata(pdev); 501 502 pdsc_stop_health_thread(pdsc); 503 pdsc_fw_down(pdsc); 504 505 if (pdev->is_virtfn) { 506 struct pdsc *pf; 507 508 pf = pdsc_get_pf_struct(pdsc->pdev); 509 if (!IS_ERR(pf)) 510 pdsc_auxbus_dev_del(pdsc, pf, 511 &pf->vfs[pdsc->vf_id].padev); 512 } else { 513 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); 514 } 515 516 if (!pdev->is_virtfn) { 517 mutex_lock(&pdsc->devcmd_lock); 518 pdsc_unmap_bars(pdsc); 519 mutex_unlock(&pdsc->devcmd_lock); 520 pci_release_regions(pdev); 521 } 522 if (pci_is_enabled(pdev)) 523 pci_disable_device(pdev); 524 pdsc_deferred_dma_free(pdsc); 525 } 526 527 static void pdsc_reset_done(struct pci_dev *pdev) 528 { 529 struct pdsc *pdsc = pci_get_drvdata(pdev); 530 struct device *dev = pdsc->dev; 531 int err; 532 533 err = pci_enable_device(pdev); 534 if (err) { 535 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 536 return; 537 } 538 pci_set_master(pdev); 539 540 if (!pdev->is_virtfn) { 541 pcie_print_link_status(pdsc->pdev); 542 543 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 544 if (err) { 545 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 546 ERR_PTR(err)); 547 return; 548 } 549 550 mutex_lock(&pdsc->devcmd_lock); 551 err = pdsc_map_bars(pdsc); 552 mutex_unlock(&pdsc->devcmd_lock); 553 if (err) 554 return; 555 } 556 557 pdsc_fw_up(pdsc); 558 pdsc_restart_health_thread(pdsc); 559 560 if (pdev->is_virtfn) { 561 struct pdsc *pf; 562 563 pf = pdsc_get_pf_struct(pdsc->pdev); 564 if (!IS_ERR(pf)) 565 pdsc_auxbus_dev_add(pdsc, pf, PDS_DEV_TYPE_VDPA, 566 &pf->vfs[pdsc->vf_id].padev); 567 } else { 568 pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL, 569 &pdsc->padev); 570 } 571 } 572 573 static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev, 574 pci_channel_state_t error) 575 { 576 if (error == pci_channel_io_frozen) { 577 struct pdsc *pdsc = pci_get_drvdata(pdev); 578 579 pdsc_reset_prepare(pdev); 580 if (!pdev->is_virtfn) 581 cancel_work_sync(&pdsc->pci_reset_work); 582 return PCI_ERS_RESULT_NEED_RESET; 583 } 584 585 return PCI_ERS_RESULT_NONE; 586 } 587 588 static void pdsc_pci_error_resume(struct pci_dev *pdev) 589 { 590 struct pdsc *pdsc = pci_get_drvdata(pdev); 591 592 if (test_bit(PDSC_S_FW_DEAD, &pdsc->state)) 593 pci_reset_function_locked(pdev); 594 } 595 596 static const struct pci_error_handlers pdsc_err_handler = { 597 /* FLR handling */ 598 .reset_prepare = pdsc_reset_prepare, 599 .reset_done = pdsc_reset_done, 600 601 /* AER handling */ 602 .error_detected = pdsc_pci_error_detected, 603 .resume = pdsc_pci_error_resume, 604 }; 605 606 static struct pci_driver pdsc_driver = { 607 .name = PDS_CORE_DRV_NAME, 608 .id_table = pdsc_id_table, 609 .probe = pdsc_probe, 610 .remove = pdsc_remove, 611 .sriov_configure = pdsc_sriov_configure, 612 .err_handler = &pdsc_err_handler, 613 }; 614 615 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev) 616 { 617 return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver); 618 } 619 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct); 620 621 static int __init pdsc_init_module(void) 622 { 623 if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME)) 624 return -EINVAL; 625 626 pdsc_debugfs_create(); 627 return pci_register_driver(&pdsc_driver); 628 } 629 630 static void __exit pdsc_cleanup_module(void) 631 { 632 pci_unregister_driver(&pdsc_driver); 633 pdsc_debugfs_destroy(); 634 } 635 636 module_init(pdsc_init_module); 637 module_exit(pdsc_cleanup_module); 638