1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/module.h> 5 #include <linux/netdevice.h> 6 #include <linux/etherdevice.h> 7 #include <linux/pci.h> 8 9 #include "ionic.h" 10 #include "ionic_bus.h" 11 #include "ionic_lif.h" 12 #include "ionic_debugfs.h" 13 14 /* Supported devices */ 15 static const struct pci_device_id ionic_id_table[] = { 16 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF) }, 17 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF) }, 18 { 0, } /* end of table */ 19 }; 20 MODULE_DEVICE_TABLE(pci, ionic_id_table); 21 22 int ionic_bus_get_irq(struct ionic *ionic, unsigned int num) 23 { 24 return pci_irq_vector(ionic->pdev, num); 25 } 26 27 const char *ionic_bus_info(struct ionic *ionic) 28 { 29 return pci_name(ionic->pdev); 30 } 31 32 int ionic_bus_alloc_irq_vectors(struct ionic *ionic, unsigned int nintrs) 33 { 34 return pci_alloc_irq_vectors(ionic->pdev, nintrs, nintrs, 35 PCI_IRQ_MSIX); 36 } 37 38 void ionic_bus_free_irq_vectors(struct ionic *ionic) 39 { 40 if (!ionic->nintrs) 41 return; 42 43 pci_free_irq_vectors(ionic->pdev); 44 } 45 46 static int ionic_map_bars(struct ionic *ionic) 47 { 48 struct pci_dev *pdev = ionic->pdev; 49 struct device *dev = ionic->dev; 50 struct ionic_dev_bar *bars; 51 unsigned int i, j; 52 53 bars = ionic->bars; 54 ionic->num_bars = 0; 55 56 for (i = 0, j = 0; i < IONIC_BARS_MAX; i++) { 57 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 58 continue; 59 bars[j].len = pci_resource_len(pdev, i); 60 61 /* only map the whole bar 0 */ 62 if (j > 0) { 63 bars[j].vaddr = NULL; 64 } else { 65 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 66 if (!bars[j].vaddr) { 67 dev_err(dev, 68 "Cannot memory-map BAR %d, aborting\n", 69 i); 70 return -ENODEV; 71 } 72 } 73 74 bars[j].bus_addr = pci_resource_start(pdev, i); 75 bars[j].res_index = i; 76 ionic->num_bars++; 77 j++; 78 } 79 80 return 0; 81 } 82 83 static void ionic_unmap_bars(struct ionic *ionic) 84 { 85 struct ionic_dev_bar *bars = ionic->bars; 86 unsigned int i; 87 88 for (i = 0; i < IONIC_BARS_MAX; i++) { 89 if (bars[i].vaddr) { 90 iounmap(bars[i].vaddr); 91 bars[i].bus_addr = 0; 92 bars[i].vaddr = NULL; 93 bars[i].len = 0; 94 } 95 } 96 } 97 98 void __iomem *ionic_bus_map_dbpage(struct ionic *ionic, int page_num) 99 { 100 return pci_iomap_range(ionic->pdev, 101 ionic->bars[IONIC_PCI_BAR_DBELL].res_index, 102 (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 103 } 104 105 void ionic_bus_unmap_dbpage(struct ionic *ionic, void __iomem *page) 106 { 107 iounmap(page); 108 } 109 110 static void ionic_vf_dealloc_locked(struct ionic *ionic) 111 { 112 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR }; 113 struct ionic_vf *v; 114 int i; 115 116 if (!ionic->vfs) 117 return; 118 119 for (i = ionic->num_vfs - 1; i >= 0; i--) { 120 v = &ionic->vfs[i]; 121 122 if (v->stats_pa) { 123 vfc.stats_pa = 0; 124 ionic_set_vf_config(ionic, i, &vfc); 125 dma_unmap_single(ionic->dev, v->stats_pa, 126 sizeof(v->stats), DMA_FROM_DEVICE); 127 v->stats_pa = 0; 128 } 129 } 130 131 kfree(ionic->vfs); 132 ionic->vfs = NULL; 133 ionic->num_vfs = 0; 134 } 135 136 static void ionic_vf_dealloc(struct ionic *ionic) 137 { 138 down_write(&ionic->vf_op_lock); 139 ionic_vf_dealloc_locked(ionic); 140 up_write(&ionic->vf_op_lock); 141 } 142 143 static int ionic_vf_alloc(struct ionic *ionic, int num_vfs) 144 { 145 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR }; 146 struct ionic_vf *v; 147 int err = 0; 148 int i; 149 150 down_write(&ionic->vf_op_lock); 151 152 ionic->vfs = kcalloc(num_vfs, sizeof(struct ionic_vf), GFP_KERNEL); 153 if (!ionic->vfs) { 154 err = -ENOMEM; 155 goto out; 156 } 157 158 for (i = 0; i < num_vfs; i++) { 159 v = &ionic->vfs[i]; 160 v->stats_pa = dma_map_single(ionic->dev, &v->stats, 161 sizeof(v->stats), DMA_FROM_DEVICE); 162 if (dma_mapping_error(ionic->dev, v->stats_pa)) { 163 v->stats_pa = 0; 164 err = -ENODEV; 165 goto out; 166 } 167 168 ionic->num_vfs++; 169 170 /* ignore failures from older FW, we just won't get stats */ 171 vfc.stats_pa = cpu_to_le64(v->stats_pa); 172 ionic_set_vf_config(ionic, i, &vfc); 173 } 174 175 out: 176 if (err) 177 ionic_vf_dealloc_locked(ionic); 178 up_write(&ionic->vf_op_lock); 179 return err; 180 } 181 182 static int ionic_sriov_configure(struct pci_dev *pdev, int num_vfs) 183 { 184 struct ionic *ionic = pci_get_drvdata(pdev); 185 struct device *dev = ionic->dev; 186 int ret = 0; 187 188 if (ionic->lif && 189 test_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state)) 190 return -EBUSY; 191 192 if (num_vfs > 0) { 193 ret = pci_enable_sriov(pdev, num_vfs); 194 if (ret) { 195 dev_err(dev, "Cannot enable SRIOV: %d\n", ret); 196 goto out; 197 } 198 199 ret = ionic_vf_alloc(ionic, num_vfs); 200 if (ret) { 201 dev_err(dev, "Cannot alloc VFs: %d\n", ret); 202 pci_disable_sriov(pdev); 203 goto out; 204 } 205 206 ret = num_vfs; 207 } else { 208 pci_disable_sriov(pdev); 209 ionic_vf_dealloc(ionic); 210 } 211 212 out: 213 return ret; 214 } 215 216 static void ionic_clear_pci(struct ionic *ionic) 217 { 218 ionic->idev.dev_info_regs = NULL; 219 ionic->idev.dev_cmd_regs = NULL; 220 ionic->idev.intr_status = NULL; 221 ionic->idev.intr_ctrl = NULL; 222 223 ionic_unmap_bars(ionic); 224 pci_release_regions(ionic->pdev); 225 226 if (pci_is_enabled(ionic->pdev)) 227 pci_disable_device(ionic->pdev); 228 } 229 230 static int ionic_setup_one(struct ionic *ionic) 231 { 232 struct pci_dev *pdev = ionic->pdev; 233 struct device *dev = ionic->dev; 234 int err; 235 236 ionic_debugfs_add_dev(ionic); 237 238 /* Setup PCI device */ 239 err = pci_enable_device_mem(pdev); 240 if (err) { 241 dev_err(dev, "Cannot enable PCI device: %d, aborting\n", err); 242 goto err_out_debugfs_del_dev; 243 } 244 245 err = pci_request_regions(pdev, IONIC_DRV_NAME); 246 if (err) { 247 dev_err(dev, "Cannot request PCI regions: %d, aborting\n", err); 248 goto err_out_clear_pci; 249 } 250 pcie_print_link_status(pdev); 251 252 err = ionic_map_bars(ionic); 253 if (err) 254 goto err_out_clear_pci; 255 256 /* Configure the device */ 257 err = ionic_setup(ionic); 258 if (err) { 259 dev_err(dev, "Cannot setup device: %d, aborting\n", err); 260 goto err_out_clear_pci; 261 } 262 pci_set_master(pdev); 263 264 err = ionic_identify(ionic); 265 if (err) { 266 dev_err(dev, "Cannot identify device: %d, aborting\n", err); 267 goto err_out_teardown; 268 } 269 ionic_debugfs_add_ident(ionic); 270 271 err = ionic_init(ionic); 272 if (err) { 273 dev_err(dev, "Cannot init device: %d, aborting\n", err); 274 goto err_out_teardown; 275 } 276 277 /* Configure the port */ 278 err = ionic_port_identify(ionic); 279 if (err) { 280 dev_err(dev, "Cannot identify port: %d, aborting\n", err); 281 goto err_out_teardown; 282 } 283 284 err = ionic_port_init(ionic); 285 if (err) { 286 dev_err(dev, "Cannot init port: %d, aborting\n", err); 287 goto err_out_teardown; 288 } 289 290 return 0; 291 292 err_out_teardown: 293 ionic_dev_teardown(ionic); 294 err_out_clear_pci: 295 ionic_clear_pci(ionic); 296 err_out_debugfs_del_dev: 297 ionic_debugfs_del_dev(ionic); 298 299 return err; 300 } 301 302 static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 303 { 304 struct device *dev = &pdev->dev; 305 struct ionic *ionic; 306 int num_vfs; 307 int err; 308 309 ionic = ionic_devlink_alloc(dev); 310 if (!ionic) 311 return -ENOMEM; 312 313 ionic->pdev = pdev; 314 ionic->dev = dev; 315 pci_set_drvdata(pdev, ionic); 316 mutex_init(&ionic->dev_cmd_lock); 317 318 /* Query system for DMA addressing limitation for the device. */ 319 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(IONIC_ADDR_LEN)); 320 if (err) { 321 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting. err=%d\n", 322 err); 323 goto err_out; 324 } 325 326 err = ionic_setup_one(ionic); 327 if (err) 328 goto err_out; 329 330 /* Allocate and init the LIF */ 331 err = ionic_lif_size(ionic); 332 if (err) { 333 dev_err(dev, "Cannot size LIF: %d, aborting\n", err); 334 goto err_out_pci; 335 } 336 337 err = ionic_lif_alloc(ionic); 338 if (err) { 339 dev_err(dev, "Cannot allocate LIF: %d, aborting\n", err); 340 goto err_out_free_irqs; 341 } 342 343 err = ionic_lif_init(ionic->lif); 344 if (err) { 345 dev_err(dev, "Cannot init LIF: %d, aborting\n", err); 346 goto err_out_free_lifs; 347 } 348 349 init_rwsem(&ionic->vf_op_lock); 350 num_vfs = pci_num_vf(pdev); 351 if (num_vfs) { 352 dev_info(dev, "%d VFs found already enabled\n", num_vfs); 353 err = ionic_vf_alloc(ionic, num_vfs); 354 if (err) 355 dev_err(dev, "Cannot enable existing VFs: %d\n", err); 356 } 357 358 err = ionic_devlink_register(ionic); 359 if (err) { 360 dev_err(dev, "Cannot register devlink: %d\n", err); 361 goto err_out_deinit_lifs; 362 } 363 364 err = ionic_lif_register(ionic->lif); 365 if (err) { 366 dev_err(dev, "Cannot register LIF: %d, aborting\n", err); 367 goto err_out_deregister_devlink; 368 } 369 370 mod_timer(&ionic->watchdog_timer, 371 round_jiffies(jiffies + ionic->watchdog_period)); 372 373 return 0; 374 375 err_out_deregister_devlink: 376 ionic_devlink_unregister(ionic); 377 err_out_deinit_lifs: 378 ionic_vf_dealloc(ionic); 379 ionic_lif_deinit(ionic->lif); 380 err_out_free_lifs: 381 ionic_lif_free(ionic->lif); 382 ionic->lif = NULL; 383 err_out_free_irqs: 384 ionic_bus_free_irq_vectors(ionic); 385 err_out_pci: 386 ionic_dev_teardown(ionic); 387 ionic_clear_pci(ionic); 388 err_out: 389 mutex_destroy(&ionic->dev_cmd_lock); 390 ionic_devlink_free(ionic); 391 392 return err; 393 } 394 395 static void ionic_remove(struct pci_dev *pdev) 396 { 397 struct ionic *ionic = pci_get_drvdata(pdev); 398 399 timer_shutdown_sync(&ionic->watchdog_timer); 400 401 if (ionic->lif) { 402 /* prevent adminq cmds if already known as down */ 403 if (test_and_clear_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state)) 404 set_bit(IONIC_LIF_F_FW_STOPPING, ionic->lif->state); 405 406 ionic_lif_unregister(ionic->lif); 407 ionic_devlink_unregister(ionic); 408 ionic_lif_deinit(ionic->lif); 409 ionic_lif_free(ionic->lif); 410 ionic->lif = NULL; 411 ionic_bus_free_irq_vectors(ionic); 412 } 413 414 ionic_port_reset(ionic); 415 ionic_reset(ionic); 416 ionic_dev_teardown(ionic); 417 ionic_clear_pci(ionic); 418 ionic_debugfs_del_dev(ionic); 419 mutex_destroy(&ionic->dev_cmd_lock); 420 ionic_devlink_free(ionic); 421 } 422 423 static void ionic_reset_prepare(struct pci_dev *pdev) 424 { 425 struct ionic *ionic = pci_get_drvdata(pdev); 426 struct ionic_lif *lif = ionic->lif; 427 428 dev_dbg(ionic->dev, "%s: device stopping\n", __func__); 429 430 set_bit(IONIC_LIF_F_FW_RESET, lif->state); 431 432 del_timer_sync(&ionic->watchdog_timer); 433 cancel_work_sync(&lif->deferred.work); 434 435 mutex_lock(&lif->queue_lock); 436 ionic_stop_queues_reconfig(lif); 437 ionic_txrx_free(lif); 438 ionic_lif_deinit(lif); 439 ionic_qcqs_free(lif); 440 ionic_debugfs_del_lif(lif); 441 mutex_unlock(&lif->queue_lock); 442 443 ionic_dev_teardown(ionic); 444 ionic_clear_pci(ionic); 445 ionic_debugfs_del_dev(ionic); 446 } 447 448 static void ionic_reset_done(struct pci_dev *pdev) 449 { 450 struct ionic *ionic = pci_get_drvdata(pdev); 451 struct ionic_lif *lif = ionic->lif; 452 int err; 453 454 err = ionic_setup_one(ionic); 455 if (err) 456 goto err_out; 457 458 ionic_debugfs_add_sizes(ionic); 459 ionic_debugfs_add_lif(ionic->lif); 460 461 err = ionic_restart_lif(lif); 462 if (err) 463 goto err_out; 464 465 mod_timer(&ionic->watchdog_timer, jiffies + 1); 466 467 err_out: 468 dev_dbg(ionic->dev, "%s: device recovery %s\n", 469 __func__, err ? "failed" : "done"); 470 } 471 472 static pci_ers_result_t ionic_pci_error_detected(struct pci_dev *pdev, 473 pci_channel_state_t error) 474 { 475 if (error == pci_channel_io_frozen) { 476 ionic_reset_prepare(pdev); 477 return PCI_ERS_RESULT_NEED_RESET; 478 } 479 480 return PCI_ERS_RESULT_NONE; 481 } 482 483 static void ionic_pci_error_resume(struct pci_dev *pdev) 484 { 485 struct ionic *ionic = pci_get_drvdata(pdev); 486 struct ionic_lif *lif = ionic->lif; 487 488 if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 489 pci_reset_function_locked(pdev); 490 } 491 492 static const struct pci_error_handlers ionic_err_handler = { 493 /* FLR handling */ 494 .reset_prepare = ionic_reset_prepare, 495 .reset_done = ionic_reset_done, 496 497 /* PCI bus error detected on this device */ 498 .error_detected = ionic_pci_error_detected, 499 .resume = ionic_pci_error_resume, 500 501 }; 502 503 static struct pci_driver ionic_driver = { 504 .name = IONIC_DRV_NAME, 505 .id_table = ionic_id_table, 506 .probe = ionic_probe, 507 .remove = ionic_remove, 508 .sriov_configure = ionic_sriov_configure, 509 .err_handler = &ionic_err_handler 510 }; 511 512 int ionic_bus_register_driver(void) 513 { 514 return pci_register_driver(&ionic_driver); 515 } 516 517 void ionic_bus_unregister_driver(void) 518 { 519 pci_unregister_driver(&ionic_driver); 520 } 521