1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/module.h> 5 #include <linux/netdevice.h> 6 #include <linux/etherdevice.h> 7 #include <linux/pci.h> 8 9 #include "ionic.h" 10 #include "ionic_bus.h" 11 #include "ionic_lif.h" 12 #include "ionic_debugfs.h" 13 14 /* Supported devices */ 15 static const struct pci_device_id ionic_id_table[] = { 16 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF) }, 17 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF) }, 18 { 0, } /* end of table */ 19 }; 20 MODULE_DEVICE_TABLE(pci, ionic_id_table); 21 22 int ionic_bus_get_irq(struct ionic *ionic, unsigned int num) 23 { 24 return pci_irq_vector(ionic->pdev, num); 25 } 26 27 const char *ionic_bus_info(struct ionic *ionic) 28 { 29 return pci_name(ionic->pdev); 30 } 31 32 int ionic_bus_alloc_irq_vectors(struct ionic *ionic, unsigned int nintrs) 33 { 34 return pci_alloc_irq_vectors(ionic->pdev, nintrs, nintrs, 35 PCI_IRQ_MSIX); 36 } 37 38 void ionic_bus_free_irq_vectors(struct ionic *ionic) 39 { 40 if (!ionic->nintrs) 41 return; 42 43 pci_free_irq_vectors(ionic->pdev); 44 } 45 46 static int ionic_map_bars(struct ionic *ionic) 47 { 48 struct pci_dev *pdev = ionic->pdev; 49 struct device *dev = ionic->dev; 50 struct ionic_dev_bar *bars; 51 unsigned int i, j; 52 53 bars = ionic->bars; 54 ionic->num_bars = 0; 55 56 for (i = 0, j = 0; i < IONIC_BARS_MAX; i++) { 57 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 58 continue; 59 bars[j].len = pci_resource_len(pdev, i); 60 61 /* only map the whole bar 0 */ 62 if (j > 0) { 63 bars[j].vaddr = NULL; 64 } else { 65 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 66 if (!bars[j].vaddr) { 67 dev_err(dev, 68 "Cannot memory-map BAR %d, aborting\n", 69 i); 70 return -ENODEV; 71 } 72 } 73 74 bars[j].bus_addr = pci_resource_start(pdev, i); 75 bars[j].res_index = i; 76 ionic->num_bars++; 77 j++; 78 } 79 80 return 0; 81 } 82 83 static void ionic_unmap_bars(struct ionic *ionic) 84 { 85 struct ionic_dev_bar *bars = ionic->bars; 86 unsigned int i; 87 88 for (i = 0; i < IONIC_BARS_MAX; i++) { 89 if (bars[i].vaddr) { 90 iounmap(bars[i].vaddr); 91 bars[i].bus_addr = 0; 92 bars[i].vaddr = NULL; 93 bars[i].len = 0; 94 } 95 } 96 ionic->num_bars = 0; 97 } 98 99 void __iomem *ionic_bus_map_dbpage(struct ionic *ionic, int page_num) 100 { 101 return pci_iomap_range(ionic->pdev, 102 ionic->bars[IONIC_PCI_BAR_DBELL].res_index, 103 (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 104 } 105 106 void ionic_bus_unmap_dbpage(struct ionic *ionic, void __iomem *page) 107 { 108 iounmap(page); 109 } 110 111 static void ionic_vf_dealloc_locked(struct ionic *ionic) 112 { 113 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR }; 114 struct ionic_vf *v; 115 int i; 116 117 if (!ionic->vfs) 118 return; 119 120 for (i = ionic->num_vfs - 1; i >= 0; i--) { 121 v = &ionic->vfs[i]; 122 123 if (v->stats_pa) { 124 vfc.stats_pa = 0; 125 ionic_set_vf_config(ionic, i, &vfc); 126 dma_unmap_single(ionic->dev, v->stats_pa, 127 sizeof(v->stats), DMA_FROM_DEVICE); 128 v->stats_pa = 0; 129 } 130 } 131 132 kfree(ionic->vfs); 133 ionic->vfs = NULL; 134 ionic->num_vfs = 0; 135 } 136 137 static void ionic_vf_dealloc(struct ionic *ionic) 138 { 139 down_write(&ionic->vf_op_lock); 140 ionic_vf_dealloc_locked(ionic); 141 up_write(&ionic->vf_op_lock); 142 } 143 144 static int ionic_vf_alloc(struct ionic *ionic, int num_vfs) 145 { 146 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR }; 147 struct ionic_vf *v; 148 int err = 0; 149 int i; 150 151 down_write(&ionic->vf_op_lock); 152 153 ionic->vfs = kcalloc(num_vfs, sizeof(struct ionic_vf), GFP_KERNEL); 154 if (!ionic->vfs) { 155 err = -ENOMEM; 156 goto out; 157 } 158 159 for (i = 0; i < num_vfs; i++) { 160 v = &ionic->vfs[i]; 161 v->stats_pa = dma_map_single(ionic->dev, &v->stats, 162 sizeof(v->stats), DMA_FROM_DEVICE); 163 if (dma_mapping_error(ionic->dev, v->stats_pa)) { 164 v->stats_pa = 0; 165 err = -ENODEV; 166 goto out; 167 } 168 169 ionic->num_vfs++; 170 171 /* ignore failures from older FW, we just won't get stats */ 172 vfc.stats_pa = cpu_to_le64(v->stats_pa); 173 ionic_set_vf_config(ionic, i, &vfc); 174 } 175 176 out: 177 if (err) 178 ionic_vf_dealloc_locked(ionic); 179 up_write(&ionic->vf_op_lock); 180 return err; 181 } 182 183 static int ionic_sriov_configure(struct pci_dev *pdev, int num_vfs) 184 { 185 struct ionic *ionic = pci_get_drvdata(pdev); 186 struct device *dev = ionic->dev; 187 int ret = 0; 188 189 if (ionic->lif && 190 test_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state)) 191 return -EBUSY; 192 193 if (num_vfs > 0) { 194 ret = pci_enable_sriov(pdev, num_vfs); 195 if (ret) { 196 dev_err(dev, "Cannot enable SRIOV: %d\n", ret); 197 goto out; 198 } 199 200 ret = ionic_vf_alloc(ionic, num_vfs); 201 if (ret) { 202 dev_err(dev, "Cannot alloc VFs: %d\n", ret); 203 pci_disable_sriov(pdev); 204 goto out; 205 } 206 207 ret = num_vfs; 208 } else { 209 pci_disable_sriov(pdev); 210 ionic_vf_dealloc(ionic); 211 } 212 213 out: 214 return ret; 215 } 216 217 static void ionic_clear_pci(struct ionic *ionic) 218 { 219 if (ionic->num_bars) { 220 ionic->idev.dev_info_regs = NULL; 221 ionic->idev.dev_cmd_regs = NULL; 222 ionic->idev.intr_status = NULL; 223 ionic->idev.intr_ctrl = NULL; 224 225 ionic_unmap_bars(ionic); 226 pci_release_regions(ionic->pdev); 227 } 228 229 if (pci_is_enabled(ionic->pdev)) 230 pci_disable_device(ionic->pdev); 231 } 232 233 static int ionic_setup_one(struct ionic *ionic) 234 { 235 struct pci_dev *pdev = ionic->pdev; 236 struct device *dev = ionic->dev; 237 int err; 238 239 ionic_debugfs_add_dev(ionic); 240 241 /* Setup PCI device */ 242 err = pci_enable_device_mem(pdev); 243 if (err) { 244 dev_err(dev, "Cannot enable PCI device: %d, aborting\n", err); 245 goto err_out_debugfs_del_dev; 246 } 247 248 err = pci_request_regions(pdev, IONIC_DRV_NAME); 249 if (err) { 250 dev_err(dev, "Cannot request PCI regions: %d, aborting\n", err); 251 goto err_out_clear_pci; 252 } 253 pcie_print_link_status(pdev); 254 255 err = ionic_map_bars(ionic); 256 if (err) 257 goto err_out_clear_pci; 258 259 /* Configure the device */ 260 err = ionic_setup(ionic); 261 if (err) { 262 dev_err(dev, "Cannot setup device: %d, aborting\n", err); 263 goto err_out_clear_pci; 264 } 265 pci_set_master(pdev); 266 267 err = ionic_identify(ionic); 268 if (err) { 269 dev_err(dev, "Cannot identify device: %d, aborting\n", err); 270 goto err_out_teardown; 271 } 272 ionic_debugfs_add_ident(ionic); 273 274 err = ionic_init(ionic); 275 if (err) { 276 dev_err(dev, "Cannot init device: %d, aborting\n", err); 277 goto err_out_teardown; 278 } 279 280 /* Configure the port */ 281 err = ionic_port_identify(ionic); 282 if (err) { 283 dev_err(dev, "Cannot identify port: %d, aborting\n", err); 284 goto err_out_teardown; 285 } 286 287 err = ionic_port_init(ionic); 288 if (err) { 289 dev_err(dev, "Cannot init port: %d, aborting\n", err); 290 goto err_out_teardown; 291 } 292 293 return 0; 294 295 err_out_teardown: 296 ionic_dev_teardown(ionic); 297 err_out_clear_pci: 298 ionic_clear_pci(ionic); 299 err_out_debugfs_del_dev: 300 ionic_debugfs_del_dev(ionic); 301 302 return err; 303 } 304 305 static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 306 { 307 struct device *dev = &pdev->dev; 308 struct ionic *ionic; 309 int num_vfs; 310 int err; 311 312 ionic = ionic_devlink_alloc(dev); 313 if (!ionic) 314 return -ENOMEM; 315 316 ionic->pdev = pdev; 317 ionic->dev = dev; 318 pci_set_drvdata(pdev, ionic); 319 mutex_init(&ionic->dev_cmd_lock); 320 321 /* Query system for DMA addressing limitation for the device. */ 322 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(IONIC_ADDR_LEN)); 323 if (err) { 324 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting. err=%d\n", 325 err); 326 goto err_out; 327 } 328 329 #ifdef CONFIG_PPC64 330 /* Ensure MSI/MSI-X interrupts lie within addressable physical memory */ 331 pdev->no_64bit_msi = 1; 332 #endif 333 334 err = ionic_setup_one(ionic); 335 if (err) 336 goto err_out; 337 338 /* Allocate and init the LIF */ 339 err = ionic_lif_size(ionic); 340 if (err) { 341 dev_err(dev, "Cannot size LIF: %d, aborting\n", err); 342 goto err_out_pci; 343 } 344 345 err = ionic_lif_alloc(ionic); 346 if (err) { 347 dev_err(dev, "Cannot allocate LIF: %d, aborting\n", err); 348 goto err_out_free_irqs; 349 } 350 351 err = ionic_lif_init(ionic->lif); 352 if (err) { 353 dev_err(dev, "Cannot init LIF: %d, aborting\n", err); 354 goto err_out_free_lifs; 355 } 356 357 init_rwsem(&ionic->vf_op_lock); 358 num_vfs = pci_num_vf(pdev); 359 if (num_vfs) { 360 dev_info(dev, "%d VFs found already enabled\n", num_vfs); 361 err = ionic_vf_alloc(ionic, num_vfs); 362 if (err) 363 dev_err(dev, "Cannot enable existing VFs: %d\n", err); 364 } 365 366 err = ionic_devlink_register(ionic); 367 if (err) { 368 dev_err(dev, "Cannot register devlink: %d\n", err); 369 goto err_out_deinit_lifs; 370 } 371 372 err = ionic_lif_register(ionic->lif); 373 if (err) { 374 dev_err(dev, "Cannot register LIF: %d, aborting\n", err); 375 goto err_out_deregister_devlink; 376 } 377 378 mod_timer(&ionic->watchdog_timer, 379 round_jiffies(jiffies + ionic->watchdog_period)); 380 ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE); 381 382 return 0; 383 384 err_out_deregister_devlink: 385 ionic_devlink_unregister(ionic); 386 err_out_deinit_lifs: 387 ionic_vf_dealloc(ionic); 388 ionic_lif_deinit(ionic->lif); 389 err_out_free_lifs: 390 ionic_lif_free(ionic->lif); 391 ionic->lif = NULL; 392 err_out_free_irqs: 393 ionic_bus_free_irq_vectors(ionic); 394 err_out_pci: 395 ionic_dev_teardown(ionic); 396 ionic_clear_pci(ionic); 397 ionic_debugfs_del_dev(ionic); 398 err_out: 399 mutex_destroy(&ionic->dev_cmd_lock); 400 ionic_devlink_free(ionic); 401 402 return err; 403 } 404 405 static void ionic_remove(struct pci_dev *pdev) 406 { 407 struct ionic *ionic = pci_get_drvdata(pdev); 408 409 timer_shutdown_sync(&ionic->watchdog_timer); 410 411 if (ionic->lif) { 412 /* prevent adminq cmds if already known as down */ 413 if (test_and_clear_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state)) 414 set_bit(IONIC_LIF_F_FW_STOPPING, ionic->lif->state); 415 416 if (ionic->lif->doorbell_wa) 417 cancel_delayed_work_sync(&ionic->doorbell_check_dwork); 418 ionic_lif_unregister(ionic->lif); 419 ionic_devlink_unregister(ionic); 420 ionic_lif_deinit(ionic->lif); 421 ionic_lif_free(ionic->lif); 422 ionic->lif = NULL; 423 ionic_bus_free_irq_vectors(ionic); 424 } 425 426 ionic_port_reset(ionic); 427 ionic_reset(ionic); 428 ionic_dev_teardown(ionic); 429 ionic_clear_pci(ionic); 430 ionic_debugfs_del_dev(ionic); 431 mutex_destroy(&ionic->dev_cmd_lock); 432 ionic_devlink_free(ionic); 433 } 434 435 static void ionic_reset_prepare(struct pci_dev *pdev) 436 { 437 struct ionic *ionic = pci_get_drvdata(pdev); 438 struct ionic_lif *lif = ionic->lif; 439 440 dev_dbg(ionic->dev, "%s: device stopping\n", __func__); 441 442 set_bit(IONIC_LIF_F_FW_RESET, lif->state); 443 444 del_timer_sync(&ionic->watchdog_timer); 445 cancel_work_sync(&lif->deferred.work); 446 447 mutex_lock(&lif->queue_lock); 448 ionic_stop_queues_reconfig(lif); 449 ionic_txrx_free(lif); 450 ionic_lif_deinit(lif); 451 ionic_qcqs_free(lif); 452 ionic_debugfs_del_lif(lif); 453 mutex_unlock(&lif->queue_lock); 454 455 ionic_dev_teardown(ionic); 456 ionic_clear_pci(ionic); 457 ionic_debugfs_del_dev(ionic); 458 } 459 460 static void ionic_reset_done(struct pci_dev *pdev) 461 { 462 struct ionic *ionic = pci_get_drvdata(pdev); 463 struct ionic_lif *lif = ionic->lif; 464 int err; 465 466 err = ionic_setup_one(ionic); 467 if (err) 468 goto err_out; 469 470 ionic_debugfs_add_sizes(ionic); 471 ionic_debugfs_add_lif(ionic->lif); 472 473 err = ionic_restart_lif(lif); 474 if (err) 475 goto err_out; 476 477 mod_timer(&ionic->watchdog_timer, jiffies + 1); 478 479 err_out: 480 dev_dbg(ionic->dev, "%s: device recovery %s\n", 481 __func__, err ? "failed" : "done"); 482 } 483 484 static pci_ers_result_t ionic_pci_error_detected(struct pci_dev *pdev, 485 pci_channel_state_t error) 486 { 487 if (error == pci_channel_io_frozen) { 488 ionic_reset_prepare(pdev); 489 return PCI_ERS_RESULT_NEED_RESET; 490 } 491 492 return PCI_ERS_RESULT_NONE; 493 } 494 495 static void ionic_pci_error_resume(struct pci_dev *pdev) 496 { 497 struct ionic *ionic = pci_get_drvdata(pdev); 498 struct ionic_lif *lif = ionic->lif; 499 500 if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 501 pci_reset_function_locked(pdev); 502 } 503 504 static const struct pci_error_handlers ionic_err_handler = { 505 /* FLR handling */ 506 .reset_prepare = ionic_reset_prepare, 507 .reset_done = ionic_reset_done, 508 509 /* PCI bus error detected on this device */ 510 .error_detected = ionic_pci_error_detected, 511 .resume = ionic_pci_error_resume, 512 513 }; 514 515 static struct pci_driver ionic_driver = { 516 .name = IONIC_DRV_NAME, 517 .id_table = ionic_id_table, 518 .probe = ionic_probe, 519 .remove = ionic_remove, 520 .sriov_configure = ionic_sriov_configure, 521 .err_handler = &ionic_err_handler 522 }; 523 524 int ionic_bus_register_driver(void) 525 { 526 return pci_register_driver(&ionic_driver); 527 } 528 529 void ionic_bus_unregister_driver(void) 530 { 531 pci_unregister_driver(&ionic_driver); 532 } 533