1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/module.h> 5 #include <linux/netdevice.h> 6 #include <linux/etherdevice.h> 7 #include <linux/pci.h> 8 9 #include "ionic.h" 10 #include "ionic_bus.h" 11 #include "ionic_lif.h" 12 #include "ionic_aux.h" 13 #include "ionic_debugfs.h" 14 15 /* Supported devices */ 16 static const struct pci_device_id ionic_id_table[] = { 17 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF) }, 18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF) }, 19 { 0, } /* end of table */ 20 }; 21 MODULE_DEVICE_TABLE(pci, ionic_id_table); 22 23 int ionic_bus_get_irq(struct ionic *ionic, unsigned int num) 24 { 25 return pci_irq_vector(ionic->pdev, num); 26 } 27 28 const char *ionic_bus_info(struct ionic *ionic) 29 { 30 return pci_name(ionic->pdev); 31 } 32 33 int ionic_bus_alloc_irq_vectors(struct ionic *ionic, unsigned int nintrs) 34 { 35 return pci_alloc_irq_vectors(ionic->pdev, nintrs, nintrs, 36 PCI_IRQ_MSIX); 37 } 38 39 void ionic_bus_free_irq_vectors(struct ionic *ionic) 40 { 41 if (!ionic->nintrs) 42 return; 43 44 pci_free_irq_vectors(ionic->pdev); 45 } 46 47 static int ionic_map_bars(struct ionic *ionic) 48 { 49 struct pci_dev *pdev = ionic->pdev; 50 struct device *dev = ionic->dev; 51 struct ionic_dev_bar *bars; 52 unsigned int i, j; 53 54 bars = ionic->bars; 55 ionic->num_bars = 0; 56 57 for (i = 0, j = 0; i < IONIC_BARS_MAX; i++) { 58 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 59 continue; 60 bars[j].len = pci_resource_len(pdev, i); 61 62 /* only map the whole bar 0 */ 63 if (j > 0) { 64 bars[j].vaddr = NULL; 65 } else { 66 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 67 if (!bars[j].vaddr) { 68 dev_err(dev, 69 "Cannot memory-map BAR %d, aborting\n", 70 i); 71 return -ENODEV; 72 } 73 } 74 75 bars[j].bus_addr = pci_resource_start(pdev, i); 76 bars[j].res_index = i; 77 ionic->num_bars++; 78 j++; 79 } 80 81 return 0; 82 } 83 84 static void ionic_unmap_bars(struct ionic *ionic) 85 { 86 struct ionic_dev_bar *bars = ionic->bars; 87 unsigned int i; 88 89 for (i = 0; i < IONIC_BARS_MAX; i++) { 90 if (bars[i].vaddr) { 91 iounmap(bars[i].vaddr); 92 bars[i].bus_addr = 0; 93 bars[i].vaddr = NULL; 94 bars[i].len = 0; 95 } 96 } 97 ionic->num_bars = 0; 98 } 99 100 void __iomem *ionic_bus_map_dbpage(struct ionic *ionic, int page_num) 101 { 102 return pci_iomap_range(ionic->pdev, 103 ionic->bars[IONIC_PCI_BAR_DBELL].res_index, 104 (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 105 } 106 107 void ionic_bus_unmap_dbpage(struct ionic *ionic, void __iomem *page) 108 { 109 iounmap(page); 110 } 111 112 static void ionic_vf_dealloc_locked(struct ionic *ionic) 113 { 114 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR }; 115 struct ionic_vf *v; 116 int i; 117 118 if (!ionic->vfs) 119 return; 120 121 for (i = ionic->num_vfs - 1; i >= 0; i--) { 122 v = &ionic->vfs[i]; 123 124 if (v->stats_pa) { 125 vfc.stats_pa = 0; 126 ionic_set_vf_config(ionic, i, &vfc); 127 dma_unmap_single(ionic->dev, v->stats_pa, 128 sizeof(v->stats), DMA_FROM_DEVICE); 129 v->stats_pa = 0; 130 } 131 } 132 133 kfree(ionic->vfs); 134 ionic->vfs = NULL; 135 ionic->num_vfs = 0; 136 } 137 138 static void ionic_vf_dealloc(struct ionic *ionic) 139 { 140 down_write(&ionic->vf_op_lock); 141 ionic_vf_dealloc_locked(ionic); 142 up_write(&ionic->vf_op_lock); 143 } 144 145 static int ionic_vf_alloc(struct ionic *ionic, int num_vfs) 146 { 147 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR }; 148 struct ionic_vf *v; 149 int err = 0; 150 int i; 151 152 down_write(&ionic->vf_op_lock); 153 154 ionic->vfs = kzalloc_objs(struct ionic_vf, num_vfs); 155 if (!ionic->vfs) { 156 err = -ENOMEM; 157 goto out; 158 } 159 160 for (i = 0; i < num_vfs; i++) { 161 v = &ionic->vfs[i]; 162 v->stats_pa = dma_map_single(ionic->dev, &v->stats, 163 sizeof(v->stats), DMA_FROM_DEVICE); 164 if (dma_mapping_error(ionic->dev, v->stats_pa)) { 165 v->stats_pa = 0; 166 err = -ENODEV; 167 goto out; 168 } 169 170 ionic->num_vfs++; 171 172 /* ignore failures from older FW, we just won't get stats */ 173 vfc.stats_pa = cpu_to_le64(v->stats_pa); 174 ionic_set_vf_config(ionic, i, &vfc); 175 } 176 177 out: 178 if (err) 179 ionic_vf_dealloc_locked(ionic); 180 up_write(&ionic->vf_op_lock); 181 return err; 182 } 183 184 static int ionic_sriov_configure(struct pci_dev *pdev, int num_vfs) 185 { 186 struct ionic *ionic = pci_get_drvdata(pdev); 187 struct device *dev = ionic->dev; 188 int ret = 0; 189 190 if (ionic->lif && 191 test_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state)) 192 return -EBUSY; 193 194 if (num_vfs > 0) { 195 ret = pci_enable_sriov(pdev, num_vfs); 196 if (ret) { 197 dev_err(dev, "Cannot enable SRIOV: %d\n", ret); 198 goto out; 199 } 200 201 ret = ionic_vf_alloc(ionic, num_vfs); 202 if (ret) { 203 dev_err(dev, "Cannot alloc VFs: %d\n", ret); 204 pci_disable_sriov(pdev); 205 goto out; 206 } 207 208 ret = num_vfs; 209 } else { 210 pci_disable_sriov(pdev); 211 ionic_vf_dealloc(ionic); 212 } 213 214 out: 215 return ret; 216 } 217 218 static void ionic_clear_pci(struct ionic *ionic) 219 { 220 if (ionic->num_bars) { 221 ionic->idev.dev_info_regs = NULL; 222 ionic->idev.dev_cmd_regs = NULL; 223 ionic->idev.intr_status = NULL; 224 ionic->idev.intr_ctrl = NULL; 225 226 ionic_unmap_bars(ionic); 227 pci_release_regions(ionic->pdev); 228 } 229 230 if (pci_is_enabled(ionic->pdev)) 231 pci_disable_device(ionic->pdev); 232 } 233 234 static int ionic_setup_one(struct ionic *ionic) 235 { 236 struct pci_dev *pdev = ionic->pdev; 237 struct device *dev = ionic->dev; 238 int err; 239 240 ionic_debugfs_add_dev(ionic); 241 242 /* Setup PCI device */ 243 err = pci_enable_device_mem(pdev); 244 if (err) { 245 dev_err(dev, "Cannot enable PCI device: %d, aborting\n", err); 246 goto err_out_debugfs_del_dev; 247 } 248 249 err = pci_request_regions(pdev, IONIC_DRV_NAME); 250 if (err) { 251 dev_err(dev, "Cannot request PCI regions: %d, aborting\n", err); 252 goto err_out_clear_pci; 253 } 254 pcie_print_link_status(pdev); 255 256 err = ionic_map_bars(ionic); 257 if (err) 258 goto err_out_clear_pci; 259 260 /* Configure the device */ 261 err = ionic_setup(ionic); 262 if (err) { 263 dev_err(dev, "Cannot setup device: %d, aborting\n", err); 264 goto err_out_clear_pci; 265 } 266 pci_set_master(pdev); 267 268 err = ionic_identify(ionic); 269 if (err) { 270 dev_err(dev, "Cannot identify device: %d, aborting\n", err); 271 goto err_out_teardown; 272 } 273 ionic_debugfs_add_ident(ionic); 274 275 ionic_map_cmb(ionic); 276 277 err = ionic_init(ionic); 278 if (err) { 279 dev_err(dev, "Cannot init device: %d, aborting\n", err); 280 goto err_out_teardown; 281 } 282 283 /* Configure the port */ 284 err = ionic_port_identify(ionic); 285 if (err) { 286 dev_err(dev, "Cannot identify port: %d, aborting\n", err); 287 goto err_out_teardown; 288 } 289 290 err = ionic_port_init(ionic); 291 if (err) { 292 dev_err(dev, "Cannot init port: %d, aborting\n", err); 293 goto err_out_teardown; 294 } 295 296 return 0; 297 298 err_out_teardown: 299 ionic_dev_teardown(ionic); 300 err_out_clear_pci: 301 ionic_clear_pci(ionic); 302 err_out_debugfs_del_dev: 303 ionic_debugfs_del_dev(ionic); 304 305 return err; 306 } 307 308 static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 309 { 310 struct device *dev = &pdev->dev; 311 struct ionic *ionic; 312 int num_vfs; 313 int err; 314 315 ionic = ionic_devlink_alloc(dev); 316 if (!ionic) 317 return -ENOMEM; 318 319 ionic->pdev = pdev; 320 ionic->dev = dev; 321 pci_set_drvdata(pdev, ionic); 322 mutex_init(&ionic->dev_cmd_lock); 323 324 /* Query system for DMA addressing limitation for the device. */ 325 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(IONIC_ADDR_LEN)); 326 if (err) { 327 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting. err=%d\n", 328 err); 329 goto err_out; 330 } 331 332 #ifdef CONFIG_PPC64 333 /* Ensure MSI/MSI-X interrupts lie within addressable physical memory */ 334 pdev->msi_addr_mask = DMA_BIT_MASK(32); 335 #endif 336 337 err = ionic_setup_one(ionic); 338 if (err) 339 goto err_out; 340 341 /* Allocate and init the LIF */ 342 err = ionic_lif_size(ionic); 343 if (err) { 344 dev_err(dev, "Cannot size LIF: %d, aborting\n", err); 345 goto err_out_pci; 346 } 347 348 err = ionic_lif_alloc(ionic); 349 if (err) { 350 dev_err(dev, "Cannot allocate LIF: %d, aborting\n", err); 351 goto err_out_free_irqs; 352 } 353 354 err = ionic_lif_init(ionic->lif); 355 if (err) { 356 dev_err(dev, "Cannot init LIF: %d, aborting\n", err); 357 goto err_out_free_lifs; 358 } 359 360 init_rwsem(&ionic->vf_op_lock); 361 num_vfs = pci_num_vf(pdev); 362 if (num_vfs) { 363 dev_info(dev, "%d VFs found already enabled\n", num_vfs); 364 err = ionic_vf_alloc(ionic, num_vfs); 365 if (err) 366 dev_err(dev, "Cannot enable existing VFs: %d\n", err); 367 } 368 369 err = ionic_devlink_register(ionic); 370 if (err) { 371 dev_err(dev, "Cannot register devlink: %d\n", err); 372 goto err_out_deinit_lifs; 373 } 374 375 err = ionic_lif_register(ionic->lif); 376 if (err) { 377 dev_err(dev, "Cannot register LIF: %d, aborting\n", err); 378 goto err_out_deregister_devlink; 379 } 380 381 ionic_auxbus_register(ionic->lif); 382 383 mod_timer(&ionic->watchdog_timer, 384 round_jiffies(jiffies + ionic->watchdog_period)); 385 ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE); 386 387 return 0; 388 389 err_out_deregister_devlink: 390 ionic_devlink_unregister(ionic); 391 err_out_deinit_lifs: 392 ionic_vf_dealloc(ionic); 393 ionic_lif_deinit(ionic->lif); 394 err_out_free_lifs: 395 ionic_lif_free(ionic->lif); 396 ionic->lif = NULL; 397 err_out_free_irqs: 398 ionic_bus_free_irq_vectors(ionic); 399 err_out_pci: 400 ionic_port_reset(ionic); 401 ionic_dev_teardown(ionic); 402 ionic_clear_pci(ionic); 403 ionic_debugfs_del_dev(ionic); 404 err_out: 405 mutex_destroy(&ionic->dev_cmd_lock); 406 ionic_devlink_free(ionic); 407 408 return err; 409 } 410 411 static void ionic_remove(struct pci_dev *pdev) 412 { 413 struct ionic *ionic = pci_get_drvdata(pdev); 414 415 timer_shutdown_sync(&ionic->watchdog_timer); 416 417 if (ionic->lif) { 418 cancel_work_sync(&ionic->lif->deferred.work); 419 /* prevent adminq cmds if already known as down */ 420 if (test_and_clear_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state)) 421 set_bit(IONIC_LIF_F_FW_STOPPING, ionic->lif->state); 422 423 if (ionic->lif->doorbell_wa) 424 cancel_delayed_work_sync(&ionic->doorbell_check_dwork); 425 ionic_auxbus_unregister(ionic->lif); 426 ionic_lif_unregister(ionic->lif); 427 ionic_devlink_unregister(ionic); 428 ionic_vf_dealloc(ionic); 429 ionic_lif_deinit(ionic->lif); 430 ionic_lif_free(ionic->lif); 431 ionic->lif = NULL; 432 ionic_bus_free_irq_vectors(ionic); 433 } 434 435 ionic_port_reset(ionic); 436 ionic_reset(ionic); 437 ionic_dev_teardown(ionic); 438 ionic_clear_pci(ionic); 439 ionic_debugfs_del_dev(ionic); 440 mutex_destroy(&ionic->dev_cmd_lock); 441 ionic_devlink_free(ionic); 442 } 443 444 static void ionic_reset_prepare(struct pci_dev *pdev) 445 { 446 struct ionic *ionic = pci_get_drvdata(pdev); 447 struct ionic_lif *lif = ionic->lif; 448 449 dev_dbg(ionic->dev, "%s: device stopping\n", __func__); 450 451 set_bit(IONIC_LIF_F_FW_RESET, lif->state); 452 453 timer_delete_sync(&ionic->watchdog_timer); 454 cancel_work_sync(&lif->deferred.work); 455 456 ionic_auxbus_unregister(ionic->lif); 457 mutex_lock(&lif->queue_lock); 458 ionic_stop_queues_reconfig(lif); 459 ionic_txrx_free(lif); 460 ionic_lif_deinit(lif); 461 ionic_qcqs_free(lif); 462 ionic_debugfs_del_lif(lif); 463 mutex_unlock(&lif->queue_lock); 464 465 ionic_dev_teardown(ionic); 466 ionic_clear_pci(ionic); 467 ionic_debugfs_del_dev(ionic); 468 } 469 470 static void ionic_reset_done(struct pci_dev *pdev) 471 { 472 struct ionic *ionic = pci_get_drvdata(pdev); 473 struct ionic_lif *lif = ionic->lif; 474 int err; 475 476 err = ionic_setup_one(ionic); 477 if (err) 478 goto err_out; 479 480 ionic_debugfs_add_sizes(ionic); 481 ionic_debugfs_add_lif(ionic->lif); 482 483 err = ionic_restart_lif(lif); 484 if (err) 485 goto err_out; 486 487 mod_timer(&ionic->watchdog_timer, jiffies + 1); 488 489 err_out: 490 dev_dbg(ionic->dev, "%s: device recovery %s\n", 491 __func__, err ? "failed" : "done"); 492 } 493 494 static pci_ers_result_t ionic_pci_error_detected(struct pci_dev *pdev, 495 pci_channel_state_t error) 496 { 497 if (error == pci_channel_io_frozen) { 498 ionic_reset_prepare(pdev); 499 return PCI_ERS_RESULT_NEED_RESET; 500 } 501 502 return PCI_ERS_RESULT_NONE; 503 } 504 505 static void ionic_pci_error_resume(struct pci_dev *pdev) 506 { 507 struct ionic *ionic = pci_get_drvdata(pdev); 508 struct ionic_lif *lif = ionic->lif; 509 510 if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 511 pci_reset_function_locked(pdev); 512 } 513 514 static const struct pci_error_handlers ionic_err_handler = { 515 /* FLR handling */ 516 .reset_prepare = ionic_reset_prepare, 517 .reset_done = ionic_reset_done, 518 519 /* PCI bus error detected on this device */ 520 .error_detected = ionic_pci_error_detected, 521 .resume = ionic_pci_error_resume, 522 523 }; 524 525 static struct pci_driver ionic_driver = { 526 .name = IONIC_DRV_NAME, 527 .id_table = ionic_id_table, 528 .probe = ionic_probe, 529 .remove = ionic_remove, 530 .sriov_configure = ionic_sriov_configure, 531 .err_handler = &ionic_err_handler 532 }; 533 534 int ionic_bus_register_driver(void) 535 { 536 return pci_register_driver(&ionic_driver); 537 } 538 539 void ionic_bus_unregister_driver(void) 540 { 541 pci_unregister_driver(&ionic_driver); 542 } 543