1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 #include <linux/module.h> 8 #include <net/addrconf.h> 9 #include <rdma/erdma-abi.h> 10 11 #include "erdma.h" 12 #include "erdma_cm.h" 13 #include "erdma_verbs.h" 14 15 MODULE_AUTHOR("Cheng Xu <chengyou@linux.alibaba.com>"); 16 MODULE_DESCRIPTION("Alibaba elasticRDMA adapter driver"); 17 MODULE_LICENSE("Dual BSD/GPL"); 18 19 static int erdma_netdev_event(struct notifier_block *nb, unsigned long event, 20 void *arg) 21 { 22 struct net_device *netdev = netdev_notifier_info_to_dev(arg); 23 struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb); 24 25 if (dev->netdev == NULL || dev->netdev != netdev) 26 goto done; 27 28 switch (event) { 29 case NETDEV_CHANGEMTU: 30 if (dev->mtu != netdev->mtu) { 31 erdma_set_mtu(dev, netdev->mtu); 32 dev->mtu = netdev->mtu; 33 } 34 break; 35 case NETDEV_REGISTER: 36 case NETDEV_UNREGISTER: 37 case NETDEV_CHANGEADDR: 38 case NETDEV_GOING_DOWN: 39 case NETDEV_CHANGE: 40 default: 41 break; 42 } 43 44 done: 45 return NOTIFY_OK; 46 } 47 48 static int erdma_enum_and_get_netdev(struct erdma_dev *dev) 49 { 50 struct net_device *netdev; 51 int ret = -EPROBE_DEFER; 52 53 /* Already binded to a net_device, so we skip. */ 54 if (dev->netdev) 55 return 0; 56 57 rtnl_lock(); 58 for_each_netdev(&init_net, netdev) { 59 /* 60 * In erdma, the paired netdev and ibdev should have the same 61 * MAC address. erdma can get the value from its PCIe bar 62 * registers. Since erdma can not get the paired netdev 63 * reference directly, we do a traverse here to get the paired 64 * netdev. 65 */ 66 if (ether_addr_equal_unaligned(netdev->perm_addr, 67 dev->attrs.peer_addr)) { 68 ret = ib_device_set_netdev(&dev->ibdev, netdev, 1); 69 if (ret) { 70 rtnl_unlock(); 71 ibdev_warn(&dev->ibdev, 72 "failed (%d) to link netdev", ret); 73 return ret; 74 } 75 76 dev->netdev = netdev; 77 break; 78 } 79 } 80 81 rtnl_unlock(); 82 83 return ret; 84 } 85 86 static int erdma_device_register(struct erdma_dev *dev) 87 { 88 struct ib_device *ibdev = &dev->ibdev; 89 int ret; 90 91 ret = erdma_enum_and_get_netdev(dev); 92 if (ret) 93 return ret; 94 95 dev->mtu = dev->netdev->mtu; 96 addrconf_addr_eui48((u8 *)&ibdev->node_guid, dev->netdev->dev_addr); 97 98 ret = ib_register_device(ibdev, "erdma_%d", &dev->pdev->dev); 99 if (ret) { 100 dev_err(&dev->pdev->dev, 101 "ib_register_device failed: ret = %d\n", ret); 102 return ret; 103 } 104 105 dev->netdev_nb.notifier_call = erdma_netdev_event; 106 ret = register_netdevice_notifier(&dev->netdev_nb); 107 if (ret) { 108 ibdev_err(&dev->ibdev, "failed to register notifier.\n"); 109 ib_unregister_device(ibdev); 110 } 111 112 return ret; 113 } 114 115 static irqreturn_t erdma_comm_irq_handler(int irq, void *data) 116 { 117 struct erdma_dev *dev = data; 118 119 erdma_cmdq_completion_handler(&dev->cmdq); 120 erdma_aeq_event_handler(dev); 121 122 return IRQ_HANDLED; 123 } 124 125 static int erdma_request_vectors(struct erdma_dev *dev) 126 { 127 int expect_irq_num = min(num_possible_cpus() + 1, ERDMA_NUM_MSIX_VEC); 128 int ret; 129 130 ret = pci_alloc_irq_vectors(dev->pdev, 1, expect_irq_num, PCI_IRQ_MSIX); 131 if (ret < 0) { 132 dev_err(&dev->pdev->dev, "request irq vectors failed(%d)\n", 133 ret); 134 return ret; 135 } 136 dev->attrs.irq_num = ret; 137 138 return 0; 139 } 140 141 static int erdma_comm_irq_init(struct erdma_dev *dev) 142 { 143 snprintf(dev->comm_irq.name, ERDMA_IRQNAME_SIZE, "erdma-common@pci:%s", 144 pci_name(dev->pdev)); 145 dev->comm_irq.msix_vector = 146 pci_irq_vector(dev->pdev, ERDMA_MSIX_VECTOR_CMDQ); 147 148 cpumask_set_cpu(cpumask_first(cpumask_of_pcibus(dev->pdev->bus)), 149 &dev->comm_irq.affinity_hint_mask); 150 irq_set_affinity_hint(dev->comm_irq.msix_vector, 151 &dev->comm_irq.affinity_hint_mask); 152 153 return request_irq(dev->comm_irq.msix_vector, erdma_comm_irq_handler, 0, 154 dev->comm_irq.name, dev); 155 } 156 157 static void erdma_comm_irq_uninit(struct erdma_dev *dev) 158 { 159 irq_set_affinity_hint(dev->comm_irq.msix_vector, NULL); 160 free_irq(dev->comm_irq.msix_vector, dev); 161 } 162 163 static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev) 164 { 165 int ret; 166 167 dev->proto = erdma_reg_read32(dev, ERDMA_REGS_DEV_PROTO_REG); 168 169 dev->resp_pool = dma_pool_create("erdma_resp_pool", &pdev->dev, 170 ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE, 171 0); 172 if (!dev->resp_pool) 173 return -ENOMEM; 174 175 dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev, 176 ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0); 177 if (!dev->db_pool) { 178 ret = -ENOMEM; 179 goto destroy_resp_pool; 180 } 181 182 ret = dma_set_mask_and_coherent(&pdev->dev, 183 DMA_BIT_MASK(ERDMA_PCI_WIDTH)); 184 if (ret) 185 goto destroy_db_pool; 186 187 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 188 189 return 0; 190 191 destroy_db_pool: 192 dma_pool_destroy(dev->db_pool); 193 194 destroy_resp_pool: 195 dma_pool_destroy(dev->resp_pool); 196 197 return ret; 198 } 199 200 static void erdma_device_uninit(struct erdma_dev *dev) 201 { 202 dma_pool_destroy(dev->db_pool); 203 dma_pool_destroy(dev->resp_pool); 204 } 205 206 static void erdma_hw_reset(struct erdma_dev *dev) 207 { 208 u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1); 209 210 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl); 211 } 212 213 static int erdma_wait_hw_init_done(struct erdma_dev *dev) 214 { 215 int i; 216 217 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, 218 FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1)); 219 220 for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) { 221 if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG, 222 ERDMA_REG_DEV_ST_INIT_DONE_MASK)) 223 break; 224 225 msleep(ERDMA_REG_ACCESS_WAIT_MS); 226 } 227 228 if (i == ERDMA_WAIT_DEV_DONE_CNT) { 229 dev_err(&dev->pdev->dev, "wait init done failed.\n"); 230 return -ETIMEDOUT; 231 } 232 233 return 0; 234 } 235 236 static const struct pci_device_id erdma_pci_tbl[] = { 237 { PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) }, 238 {} 239 }; 240 241 static int erdma_probe_dev(struct pci_dev *pdev) 242 { 243 struct erdma_dev *dev; 244 int bars, err; 245 u32 version; 246 247 err = pci_enable_device(pdev); 248 if (err) { 249 dev_err(&pdev->dev, "pci_enable_device failed(%d)\n", err); 250 return err; 251 } 252 253 pci_set_master(pdev); 254 255 dev = ib_alloc_device(erdma_dev, ibdev); 256 if (!dev) { 257 dev_err(&pdev->dev, "ib_alloc_device failed\n"); 258 err = -ENOMEM; 259 goto err_disable_device; 260 } 261 262 pci_set_drvdata(pdev, dev); 263 dev->pdev = pdev; 264 dev->attrs.numa_node = dev_to_node(&pdev->dev); 265 266 bars = pci_select_bars(pdev, IORESOURCE_MEM); 267 err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME); 268 if (bars != ERDMA_BAR_MASK || err) { 269 err = err ? err : -EINVAL; 270 goto err_ib_device_release; 271 } 272 273 dev->func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR); 274 dev->func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR); 275 276 dev->func_bar = 277 devm_ioremap(&pdev->dev, dev->func_bar_addr, dev->func_bar_len); 278 if (!dev->func_bar) { 279 dev_err(&pdev->dev, "devm_ioremap failed.\n"); 280 err = -EFAULT; 281 goto err_release_bars; 282 } 283 284 version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG); 285 if (version == 0) { 286 /* we knows that it is a non-functional function. */ 287 err = -ENODEV; 288 goto err_iounmap_func_bar; 289 } 290 291 err = erdma_device_init(dev, pdev); 292 if (err) 293 goto err_iounmap_func_bar; 294 295 err = erdma_request_vectors(dev); 296 if (err) 297 goto err_uninit_device; 298 299 err = erdma_comm_irq_init(dev); 300 if (err) 301 goto err_free_vectors; 302 303 err = erdma_aeq_init(dev); 304 if (err) 305 goto err_uninit_comm_irq; 306 307 err = erdma_cmdq_init(dev); 308 if (err) 309 goto err_uninit_aeq; 310 311 err = erdma_wait_hw_init_done(dev); 312 if (err) 313 goto err_uninit_cmdq; 314 315 err = erdma_ceqs_init(dev); 316 if (err) 317 goto err_reset_hw; 318 319 erdma_finish_cmdq_init(dev); 320 321 return 0; 322 323 err_reset_hw: 324 erdma_hw_reset(dev); 325 326 err_uninit_cmdq: 327 erdma_cmdq_destroy(dev); 328 329 err_uninit_aeq: 330 erdma_eq_destroy(dev, &dev->aeq); 331 332 err_uninit_comm_irq: 333 erdma_comm_irq_uninit(dev); 334 335 err_free_vectors: 336 pci_free_irq_vectors(dev->pdev); 337 338 err_uninit_device: 339 erdma_device_uninit(dev); 340 341 err_iounmap_func_bar: 342 devm_iounmap(&pdev->dev, dev->func_bar); 343 344 err_release_bars: 345 pci_release_selected_regions(pdev, bars); 346 347 err_ib_device_release: 348 ib_dealloc_device(&dev->ibdev); 349 350 err_disable_device: 351 pci_disable_device(pdev); 352 353 return err; 354 } 355 356 static void erdma_remove_dev(struct pci_dev *pdev) 357 { 358 struct erdma_dev *dev = pci_get_drvdata(pdev); 359 360 erdma_ceqs_uninit(dev); 361 erdma_hw_reset(dev); 362 erdma_cmdq_destroy(dev); 363 erdma_eq_destroy(dev, &dev->aeq); 364 erdma_comm_irq_uninit(dev); 365 pci_free_irq_vectors(dev->pdev); 366 erdma_device_uninit(dev); 367 368 devm_iounmap(&pdev->dev, dev->func_bar); 369 pci_release_selected_regions(pdev, ERDMA_BAR_MASK); 370 371 ib_dealloc_device(&dev->ibdev); 372 373 pci_disable_device(pdev); 374 } 375 376 #define ERDMA_GET_CAP(name, cap) FIELD_GET(ERDMA_CMD_DEV_CAP_##name##_MASK, cap) 377 378 static int erdma_dev_attrs_init(struct erdma_dev *dev) 379 { 380 int err; 381 u64 req_hdr, cap0, cap1; 382 383 erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_RDMA, 384 CMDQ_OPCODE_QUERY_DEVICE); 385 386 err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0, 387 &cap1, true); 388 if (err) 389 return err; 390 391 dev->attrs.max_cqe = 1 << ERDMA_GET_CAP(MAX_CQE, cap0); 392 dev->attrs.max_mr_size = 1ULL << ERDMA_GET_CAP(MAX_MR_SIZE, cap0); 393 dev->attrs.max_mw = 1 << ERDMA_GET_CAP(MAX_MW, cap1); 394 dev->attrs.max_recv_wr = 1 << ERDMA_GET_CAP(MAX_RECV_WR, cap0); 395 dev->attrs.max_gid = 1 << ERDMA_GET_CAP(MAX_GID, cap0); 396 dev->attrs.max_ah = 1 << ERDMA_GET_CAP(MAX_AH, cap0); 397 dev->attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1); 398 dev->attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1); 399 dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1); 400 dev->attrs.max_mr = dev->attrs.max_qp << 1; 401 dev->attrs.max_cq = dev->attrs.max_qp << 1; 402 dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0); 403 404 dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR; 405 dev->attrs.max_ord = ERDMA_MAX_ORD; 406 dev->attrs.max_ird = ERDMA_MAX_IRD; 407 dev->attrs.max_send_sge = ERDMA_MAX_SEND_SGE; 408 dev->attrs.max_recv_sge = ERDMA_MAX_RECV_SGE; 409 dev->attrs.max_sge_rd = ERDMA_MAX_SGE_RD; 410 dev->attrs.max_pd = ERDMA_MAX_PD; 411 412 dev->res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD; 413 dev->res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev->attrs.max_mr; 414 dev->res_cb[ERDMA_RES_TYPE_AH].max_cap = dev->attrs.max_ah; 415 416 erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_COMMON, 417 CMDQ_OPCODE_QUERY_FW_INFO); 418 419 err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0, 420 &cap1, true); 421 if (!err) 422 dev->attrs.fw_version = 423 FIELD_GET(ERDMA_CMD_INFO0_FW_VER_MASK, cap0); 424 425 return err; 426 } 427 428 static int erdma_device_config(struct erdma_dev *dev) 429 { 430 struct erdma_cmdq_config_device_req req = {}; 431 432 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_EXTEND_DB)) 433 return 0; 434 435 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 436 CMDQ_OPCODE_CONF_DEVICE); 437 438 req.cfg = FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK, PAGE_SHIFT) | 439 FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK, 1); 440 441 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 442 true); 443 } 444 445 static int erdma_res_cb_init(struct erdma_dev *dev) 446 { 447 int i, j; 448 449 for (i = 0; i < ERDMA_RES_CNT; i++) { 450 dev->res_cb[i].next_alloc_idx = 1; 451 spin_lock_init(&dev->res_cb[i].lock); 452 dev->res_cb[i].bitmap = 453 bitmap_zalloc(dev->res_cb[i].max_cap, GFP_KERNEL); 454 if (!dev->res_cb[i].bitmap) 455 goto err; 456 } 457 458 return 0; 459 460 err: 461 for (j = 0; j < i; j++) 462 bitmap_free(dev->res_cb[j].bitmap); 463 464 return -ENOMEM; 465 } 466 467 static void erdma_res_cb_free(struct erdma_dev *dev) 468 { 469 int i; 470 471 for (i = 0; i < ERDMA_RES_CNT; i++) 472 bitmap_free(dev->res_cb[i].bitmap); 473 } 474 475 static const struct ib_device_ops erdma_device_ops_rocev2 = { 476 .get_link_layer = erdma_get_link_layer, 477 .add_gid = erdma_add_gid, 478 .del_gid = erdma_del_gid, 479 .query_pkey = erdma_query_pkey, 480 .create_ah = erdma_create_ah, 481 .destroy_ah = erdma_destroy_ah, 482 .query_ah = erdma_query_ah, 483 484 INIT_RDMA_OBJ_SIZE(ib_ah, erdma_ah, ibah), 485 }; 486 487 static const struct ib_device_ops erdma_device_ops_iwarp = { 488 .iw_accept = erdma_accept, 489 .iw_add_ref = erdma_qp_get_ref, 490 .iw_connect = erdma_connect, 491 .iw_create_listen = erdma_create_listen, 492 .iw_destroy_listen = erdma_destroy_listen, 493 .iw_get_qp = erdma_get_ibqp, 494 .iw_reject = erdma_reject, 495 .iw_rem_ref = erdma_qp_put_ref, 496 }; 497 498 static const struct ib_device_ops erdma_device_ops = { 499 .owner = THIS_MODULE, 500 .driver_id = RDMA_DRIVER_ERDMA, 501 .uverbs_abi_ver = ERDMA_ABI_VERSION, 502 503 .alloc_hw_port_stats = erdma_alloc_hw_port_stats, 504 .alloc_mr = erdma_ib_alloc_mr, 505 .alloc_pd = erdma_alloc_pd, 506 .alloc_ucontext = erdma_alloc_ucontext, 507 .create_cq = erdma_create_cq, 508 .create_qp = erdma_create_qp, 509 .dealloc_pd = erdma_dealloc_pd, 510 .dealloc_ucontext = erdma_dealloc_ucontext, 511 .dereg_mr = erdma_dereg_mr, 512 .destroy_cq = erdma_destroy_cq, 513 .destroy_qp = erdma_destroy_qp, 514 .disassociate_ucontext = erdma_disassociate_ucontext, 515 .get_dma_mr = erdma_get_dma_mr, 516 .get_hw_stats = erdma_get_hw_stats, 517 .get_port_immutable = erdma_get_port_immutable, 518 .map_mr_sg = erdma_map_mr_sg, 519 .mmap = erdma_mmap, 520 .mmap_free = erdma_mmap_free, 521 .post_recv = erdma_post_recv, 522 .post_send = erdma_post_send, 523 .poll_cq = erdma_poll_cq, 524 .query_device = erdma_query_device, 525 .query_gid = erdma_query_gid, 526 .query_port = erdma_query_port, 527 .query_qp = erdma_query_qp, 528 .req_notify_cq = erdma_req_notify_cq, 529 .reg_user_mr = erdma_reg_user_mr, 530 .modify_qp = erdma_modify_qp, 531 532 INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq), 533 INIT_RDMA_OBJ_SIZE(ib_pd, erdma_pd, ibpd), 534 INIT_RDMA_OBJ_SIZE(ib_ucontext, erdma_ucontext, ibucontext), 535 INIT_RDMA_OBJ_SIZE(ib_qp, erdma_qp, ibqp), 536 }; 537 538 static int erdma_ib_device_add(struct pci_dev *pdev) 539 { 540 struct erdma_dev *dev = pci_get_drvdata(pdev); 541 struct ib_device *ibdev = &dev->ibdev; 542 u64 mac; 543 int ret; 544 545 ret = erdma_dev_attrs_init(dev); 546 if (ret) 547 return ret; 548 549 ret = erdma_device_config(dev); 550 if (ret) 551 return ret; 552 553 if (erdma_device_iwarp(dev)) { 554 ibdev->node_type = RDMA_NODE_RNIC; 555 ib_set_device_ops(ibdev, &erdma_device_ops_iwarp); 556 } else { 557 ibdev->node_type = RDMA_NODE_IB_CA; 558 ib_set_device_ops(ibdev, &erdma_device_ops_rocev2); 559 } 560 561 memcpy(ibdev->node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC)); 562 563 /* 564 * Current model (one-to-one device association): 565 * One ERDMA device per net_device or, equivalently, 566 * per physical port. 567 */ 568 ibdev->phys_port_cnt = 1; 569 ibdev->num_comp_vectors = dev->attrs.irq_num - 1; 570 571 ib_set_device_ops(ibdev, &erdma_device_ops); 572 573 INIT_LIST_HEAD(&dev->cep_list); 574 575 spin_lock_init(&dev->lock); 576 xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1); 577 xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1); 578 dev->next_alloc_cqn = 1; 579 dev->next_alloc_qpn = 1; 580 581 ret = erdma_res_cb_init(dev); 582 if (ret) 583 return ret; 584 585 atomic_set(&dev->num_ctx, 0); 586 587 mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG); 588 mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) << 32; 589 590 u64_to_ether_addr(mac, dev->attrs.peer_addr); 591 592 dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND, 593 WQ_UNBOUND_MAX_ACTIVE); 594 if (!dev->reflush_wq) { 595 ret = -ENOMEM; 596 goto err_alloc_workqueue; 597 } 598 599 ret = erdma_device_register(dev); 600 if (ret) 601 goto err_register; 602 603 return 0; 604 605 err_register: 606 destroy_workqueue(dev->reflush_wq); 607 err_alloc_workqueue: 608 xa_destroy(&dev->qp_xa); 609 xa_destroy(&dev->cq_xa); 610 611 erdma_res_cb_free(dev); 612 613 return ret; 614 } 615 616 static void erdma_ib_device_remove(struct pci_dev *pdev) 617 { 618 struct erdma_dev *dev = pci_get_drvdata(pdev); 619 620 unregister_netdevice_notifier(&dev->netdev_nb); 621 ib_unregister_device(&dev->ibdev); 622 623 destroy_workqueue(dev->reflush_wq); 624 erdma_res_cb_free(dev); 625 xa_destroy(&dev->qp_xa); 626 xa_destroy(&dev->cq_xa); 627 } 628 629 static int erdma_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 630 { 631 int ret; 632 633 ret = erdma_probe_dev(pdev); 634 if (ret) 635 return ret; 636 637 ret = erdma_ib_device_add(pdev); 638 if (ret) { 639 erdma_remove_dev(pdev); 640 return ret; 641 } 642 643 return 0; 644 } 645 646 static void erdma_remove(struct pci_dev *pdev) 647 { 648 erdma_ib_device_remove(pdev); 649 erdma_remove_dev(pdev); 650 } 651 652 static struct pci_driver erdma_pci_driver = { 653 .name = DRV_MODULE_NAME, 654 .id_table = erdma_pci_tbl, 655 .probe = erdma_probe, 656 .remove = erdma_remove 657 }; 658 659 MODULE_DEVICE_TABLE(pci, erdma_pci_tbl); 660 661 static __init int erdma_init_module(void) 662 { 663 int ret; 664 665 ret = erdma_cm_init(); 666 if (ret) 667 return ret; 668 669 ret = pci_register_driver(&erdma_pci_driver); 670 if (ret) 671 erdma_cm_exit(); 672 673 return ret; 674 } 675 676 static void __exit erdma_exit_module(void) 677 { 678 pci_unregister_driver(&erdma_pci_driver); 679 680 erdma_cm_exit(); 681 } 682 683 module_init(erdma_init_module); 684 module_exit(erdma_exit_module); 685