1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 #include <linux/module.h> 8 #include <net/addrconf.h> 9 #include <rdma/erdma-abi.h> 10 11 #include "erdma.h" 12 #include "erdma_cm.h" 13 #include "erdma_verbs.h" 14 15 MODULE_AUTHOR("Cheng Xu <chengyou@linux.alibaba.com>"); 16 MODULE_DESCRIPTION("Alibaba elasticRDMA adapter driver"); 17 MODULE_LICENSE("Dual BSD/GPL"); 18 19 static int erdma_netdev_event(struct notifier_block *nb, unsigned long event, 20 void *arg) 21 { 22 struct net_device *netdev = netdev_notifier_info_to_dev(arg); 23 struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb); 24 25 if (dev->netdev == NULL || dev->netdev != netdev) 26 goto done; 27 28 switch (event) { 29 case NETDEV_UP: 30 dev->state = IB_PORT_ACTIVE; 31 erdma_port_event(dev, IB_EVENT_PORT_ACTIVE); 32 break; 33 case NETDEV_DOWN: 34 dev->state = IB_PORT_DOWN; 35 erdma_port_event(dev, IB_EVENT_PORT_ERR); 36 break; 37 case NETDEV_CHANGEMTU: 38 if (dev->mtu != netdev->mtu) { 39 erdma_set_mtu(dev, netdev->mtu); 40 dev->mtu = netdev->mtu; 41 } 42 break; 43 case NETDEV_REGISTER: 44 case NETDEV_UNREGISTER: 45 case NETDEV_CHANGEADDR: 46 case NETDEV_GOING_DOWN: 47 case NETDEV_CHANGE: 48 default: 49 break; 50 } 51 52 done: 53 return NOTIFY_OK; 54 } 55 56 static int erdma_enum_and_get_netdev(struct erdma_dev *dev) 57 { 58 struct net_device *netdev; 59 int ret = -EPROBE_DEFER; 60 61 /* Already binded to a net_device, so we skip. */ 62 if (dev->netdev) 63 return 0; 64 65 rtnl_lock(); 66 for_each_netdev(&init_net, netdev) { 67 /* 68 * In erdma, the paired netdev and ibdev should have the same 69 * MAC address. erdma can get the value from its PCIe bar 70 * registers. Since erdma can not get the paired netdev 71 * reference directly, we do a traverse here to get the paired 72 * netdev. 73 */ 74 if (ether_addr_equal_unaligned(netdev->perm_addr, 75 dev->attrs.peer_addr)) { 76 ret = ib_device_set_netdev(&dev->ibdev, netdev, 1); 77 if (ret) { 78 rtnl_unlock(); 79 ibdev_warn(&dev->ibdev, 80 "failed (%d) to link netdev", ret); 81 return ret; 82 } 83 84 dev->netdev = netdev; 85 break; 86 } 87 } 88 89 rtnl_unlock(); 90 91 return ret; 92 } 93 94 static int erdma_device_register(struct erdma_dev *dev) 95 { 96 struct ib_device *ibdev = &dev->ibdev; 97 int ret; 98 99 ret = erdma_enum_and_get_netdev(dev); 100 if (ret) 101 return ret; 102 103 dev->mtu = dev->netdev->mtu; 104 addrconf_addr_eui48((u8 *)&ibdev->node_guid, dev->netdev->dev_addr); 105 106 ret = ib_register_device(ibdev, "erdma_%d", &dev->pdev->dev); 107 if (ret) { 108 dev_err(&dev->pdev->dev, 109 "ib_register_device failed: ret = %d\n", ret); 110 return ret; 111 } 112 113 dev->netdev_nb.notifier_call = erdma_netdev_event; 114 ret = register_netdevice_notifier(&dev->netdev_nb); 115 if (ret) { 116 ibdev_err(&dev->ibdev, "failed to register notifier.\n"); 117 ib_unregister_device(ibdev); 118 } 119 120 return ret; 121 } 122 123 static irqreturn_t erdma_comm_irq_handler(int irq, void *data) 124 { 125 struct erdma_dev *dev = data; 126 127 erdma_cmdq_completion_handler(&dev->cmdq); 128 erdma_aeq_event_handler(dev); 129 130 return IRQ_HANDLED; 131 } 132 133 static int erdma_request_vectors(struct erdma_dev *dev) 134 { 135 int expect_irq_num = min(num_possible_cpus() + 1, ERDMA_NUM_MSIX_VEC); 136 int ret; 137 138 ret = pci_alloc_irq_vectors(dev->pdev, 1, expect_irq_num, PCI_IRQ_MSIX); 139 if (ret < 0) { 140 dev_err(&dev->pdev->dev, "request irq vectors failed(%d)\n", 141 ret); 142 return ret; 143 } 144 dev->attrs.irq_num = ret; 145 146 return 0; 147 } 148 149 static int erdma_comm_irq_init(struct erdma_dev *dev) 150 { 151 snprintf(dev->comm_irq.name, ERDMA_IRQNAME_SIZE, "erdma-common@pci:%s", 152 pci_name(dev->pdev)); 153 dev->comm_irq.msix_vector = 154 pci_irq_vector(dev->pdev, ERDMA_MSIX_VECTOR_CMDQ); 155 156 cpumask_set_cpu(cpumask_first(cpumask_of_pcibus(dev->pdev->bus)), 157 &dev->comm_irq.affinity_hint_mask); 158 irq_set_affinity_hint(dev->comm_irq.msix_vector, 159 &dev->comm_irq.affinity_hint_mask); 160 161 return request_irq(dev->comm_irq.msix_vector, erdma_comm_irq_handler, 0, 162 dev->comm_irq.name, dev); 163 } 164 165 static void erdma_comm_irq_uninit(struct erdma_dev *dev) 166 { 167 irq_set_affinity_hint(dev->comm_irq.msix_vector, NULL); 168 free_irq(dev->comm_irq.msix_vector, dev); 169 } 170 171 static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev) 172 { 173 int ret; 174 175 dev->resp_pool = dma_pool_create("erdma_resp_pool", &pdev->dev, 176 ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE, 177 0); 178 if (!dev->resp_pool) 179 return -ENOMEM; 180 181 dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev, 182 ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0); 183 if (!dev->db_pool) { 184 ret = -ENOMEM; 185 goto destroy_resp_pool; 186 } 187 188 ret = dma_set_mask_and_coherent(&pdev->dev, 189 DMA_BIT_MASK(ERDMA_PCI_WIDTH)); 190 if (ret) 191 goto destroy_db_pool; 192 193 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 194 195 return 0; 196 197 destroy_db_pool: 198 dma_pool_destroy(dev->db_pool); 199 200 destroy_resp_pool: 201 dma_pool_destroy(dev->resp_pool); 202 203 return ret; 204 } 205 206 static void erdma_device_uninit(struct erdma_dev *dev) 207 { 208 dma_pool_destroy(dev->db_pool); 209 dma_pool_destroy(dev->resp_pool); 210 } 211 212 static void erdma_hw_reset(struct erdma_dev *dev) 213 { 214 u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1); 215 216 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl); 217 } 218 219 static int erdma_wait_hw_init_done(struct erdma_dev *dev) 220 { 221 int i; 222 223 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, 224 FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1)); 225 226 for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) { 227 if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG, 228 ERDMA_REG_DEV_ST_INIT_DONE_MASK)) 229 break; 230 231 msleep(ERDMA_REG_ACCESS_WAIT_MS); 232 } 233 234 if (i == ERDMA_WAIT_DEV_DONE_CNT) { 235 dev_err(&dev->pdev->dev, "wait init done failed.\n"); 236 return -ETIMEDOUT; 237 } 238 239 return 0; 240 } 241 242 static const struct pci_device_id erdma_pci_tbl[] = { 243 { PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) }, 244 {} 245 }; 246 247 static int erdma_probe_dev(struct pci_dev *pdev) 248 { 249 struct erdma_dev *dev; 250 int bars, err; 251 u32 version; 252 253 err = pci_enable_device(pdev); 254 if (err) { 255 dev_err(&pdev->dev, "pci_enable_device failed(%d)\n", err); 256 return err; 257 } 258 259 pci_set_master(pdev); 260 261 dev = ib_alloc_device(erdma_dev, ibdev); 262 if (!dev) { 263 dev_err(&pdev->dev, "ib_alloc_device failed\n"); 264 err = -ENOMEM; 265 goto err_disable_device; 266 } 267 268 pci_set_drvdata(pdev, dev); 269 dev->pdev = pdev; 270 dev->attrs.numa_node = dev_to_node(&pdev->dev); 271 272 bars = pci_select_bars(pdev, IORESOURCE_MEM); 273 err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME); 274 if (bars != ERDMA_BAR_MASK || err) { 275 err = err ? err : -EINVAL; 276 goto err_ib_device_release; 277 } 278 279 dev->func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR); 280 dev->func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR); 281 282 dev->func_bar = 283 devm_ioremap(&pdev->dev, dev->func_bar_addr, dev->func_bar_len); 284 if (!dev->func_bar) { 285 dev_err(&pdev->dev, "devm_ioremap failed.\n"); 286 err = -EFAULT; 287 goto err_release_bars; 288 } 289 290 version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG); 291 if (version == 0) { 292 /* we knows that it is a non-functional function. */ 293 err = -ENODEV; 294 goto err_iounmap_func_bar; 295 } 296 297 err = erdma_device_init(dev, pdev); 298 if (err) 299 goto err_iounmap_func_bar; 300 301 err = erdma_request_vectors(dev); 302 if (err) 303 goto err_uninit_device; 304 305 err = erdma_comm_irq_init(dev); 306 if (err) 307 goto err_free_vectors; 308 309 err = erdma_aeq_init(dev); 310 if (err) 311 goto err_uninit_comm_irq; 312 313 err = erdma_cmdq_init(dev); 314 if (err) 315 goto err_uninit_aeq; 316 317 err = erdma_wait_hw_init_done(dev); 318 if (err) 319 goto err_uninit_cmdq; 320 321 err = erdma_ceqs_init(dev); 322 if (err) 323 goto err_reset_hw; 324 325 erdma_finish_cmdq_init(dev); 326 327 return 0; 328 329 err_reset_hw: 330 erdma_hw_reset(dev); 331 332 err_uninit_cmdq: 333 erdma_cmdq_destroy(dev); 334 335 err_uninit_aeq: 336 erdma_eq_destroy(dev, &dev->aeq); 337 338 err_uninit_comm_irq: 339 erdma_comm_irq_uninit(dev); 340 341 err_free_vectors: 342 pci_free_irq_vectors(dev->pdev); 343 344 err_uninit_device: 345 erdma_device_uninit(dev); 346 347 err_iounmap_func_bar: 348 devm_iounmap(&pdev->dev, dev->func_bar); 349 350 err_release_bars: 351 pci_release_selected_regions(pdev, bars); 352 353 err_ib_device_release: 354 ib_dealloc_device(&dev->ibdev); 355 356 err_disable_device: 357 pci_disable_device(pdev); 358 359 return err; 360 } 361 362 static void erdma_remove_dev(struct pci_dev *pdev) 363 { 364 struct erdma_dev *dev = pci_get_drvdata(pdev); 365 366 erdma_ceqs_uninit(dev); 367 erdma_hw_reset(dev); 368 erdma_cmdq_destroy(dev); 369 erdma_eq_destroy(dev, &dev->aeq); 370 erdma_comm_irq_uninit(dev); 371 pci_free_irq_vectors(dev->pdev); 372 erdma_device_uninit(dev); 373 374 devm_iounmap(&pdev->dev, dev->func_bar); 375 pci_release_selected_regions(pdev, ERDMA_BAR_MASK); 376 377 ib_dealloc_device(&dev->ibdev); 378 379 pci_disable_device(pdev); 380 } 381 382 #define ERDMA_GET_CAP(name, cap) FIELD_GET(ERDMA_CMD_DEV_CAP_##name##_MASK, cap) 383 384 static int erdma_dev_attrs_init(struct erdma_dev *dev) 385 { 386 int err; 387 u64 req_hdr, cap0, cap1; 388 389 erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_RDMA, 390 CMDQ_OPCODE_QUERY_DEVICE); 391 392 err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0, 393 &cap1); 394 if (err) 395 return err; 396 397 dev->attrs.max_cqe = 1 << ERDMA_GET_CAP(MAX_CQE, cap0); 398 dev->attrs.max_mr_size = 1ULL << ERDMA_GET_CAP(MAX_MR_SIZE, cap0); 399 dev->attrs.max_mw = 1 << ERDMA_GET_CAP(MAX_MW, cap1); 400 dev->attrs.max_recv_wr = 1 << ERDMA_GET_CAP(MAX_RECV_WR, cap0); 401 dev->attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1); 402 dev->attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1); 403 dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1); 404 dev->attrs.max_mr = dev->attrs.max_qp << 1; 405 dev->attrs.max_cq = dev->attrs.max_qp << 1; 406 dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0); 407 408 dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR; 409 dev->attrs.max_ord = ERDMA_MAX_ORD; 410 dev->attrs.max_ird = ERDMA_MAX_IRD; 411 dev->attrs.max_send_sge = ERDMA_MAX_SEND_SGE; 412 dev->attrs.max_recv_sge = ERDMA_MAX_RECV_SGE; 413 dev->attrs.max_sge_rd = ERDMA_MAX_SGE_RD; 414 dev->attrs.max_pd = ERDMA_MAX_PD; 415 416 dev->res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD; 417 dev->res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev->attrs.max_mr; 418 419 erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_COMMON, 420 CMDQ_OPCODE_QUERY_FW_INFO); 421 422 err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0, 423 &cap1); 424 if (!err) 425 dev->attrs.fw_version = 426 FIELD_GET(ERDMA_CMD_INFO0_FW_VER_MASK, cap0); 427 428 return err; 429 } 430 431 static int erdma_device_config(struct erdma_dev *dev) 432 { 433 struct erdma_cmdq_config_device_req req = {}; 434 435 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_EXTEND_DB)) 436 return 0; 437 438 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 439 CMDQ_OPCODE_CONF_DEVICE); 440 441 req.cfg = FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK, PAGE_SHIFT) | 442 FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK, 1); 443 444 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 445 } 446 447 static int erdma_res_cb_init(struct erdma_dev *dev) 448 { 449 int i, j; 450 451 for (i = 0; i < ERDMA_RES_CNT; i++) { 452 dev->res_cb[i].next_alloc_idx = 1; 453 spin_lock_init(&dev->res_cb[i].lock); 454 dev->res_cb[i].bitmap = 455 bitmap_zalloc(dev->res_cb[i].max_cap, GFP_KERNEL); 456 if (!dev->res_cb[i].bitmap) 457 goto err; 458 } 459 460 return 0; 461 462 err: 463 for (j = 0; j < i; j++) 464 bitmap_free(dev->res_cb[j].bitmap); 465 466 return -ENOMEM; 467 } 468 469 static void erdma_res_cb_free(struct erdma_dev *dev) 470 { 471 int i; 472 473 for (i = 0; i < ERDMA_RES_CNT; i++) 474 bitmap_free(dev->res_cb[i].bitmap); 475 } 476 477 static const struct ib_device_ops erdma_device_ops = { 478 .owner = THIS_MODULE, 479 .driver_id = RDMA_DRIVER_ERDMA, 480 .uverbs_abi_ver = ERDMA_ABI_VERSION, 481 482 .alloc_hw_port_stats = erdma_alloc_hw_port_stats, 483 .alloc_mr = erdma_ib_alloc_mr, 484 .alloc_pd = erdma_alloc_pd, 485 .alloc_ucontext = erdma_alloc_ucontext, 486 .create_cq = erdma_create_cq, 487 .create_qp = erdma_create_qp, 488 .dealloc_pd = erdma_dealloc_pd, 489 .dealloc_ucontext = erdma_dealloc_ucontext, 490 .dereg_mr = erdma_dereg_mr, 491 .destroy_cq = erdma_destroy_cq, 492 .destroy_qp = erdma_destroy_qp, 493 .disassociate_ucontext = erdma_disassociate_ucontext, 494 .get_dma_mr = erdma_get_dma_mr, 495 .get_hw_stats = erdma_get_hw_stats, 496 .get_port_immutable = erdma_get_port_immutable, 497 .iw_accept = erdma_accept, 498 .iw_add_ref = erdma_qp_get_ref, 499 .iw_connect = erdma_connect, 500 .iw_create_listen = erdma_create_listen, 501 .iw_destroy_listen = erdma_destroy_listen, 502 .iw_get_qp = erdma_get_ibqp, 503 .iw_reject = erdma_reject, 504 .iw_rem_ref = erdma_qp_put_ref, 505 .map_mr_sg = erdma_map_mr_sg, 506 .mmap = erdma_mmap, 507 .mmap_free = erdma_mmap_free, 508 .modify_qp = erdma_modify_qp, 509 .post_recv = erdma_post_recv, 510 .post_send = erdma_post_send, 511 .poll_cq = erdma_poll_cq, 512 .query_device = erdma_query_device, 513 .query_gid = erdma_query_gid, 514 .query_port = erdma_query_port, 515 .query_qp = erdma_query_qp, 516 .req_notify_cq = erdma_req_notify_cq, 517 .reg_user_mr = erdma_reg_user_mr, 518 519 INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq), 520 INIT_RDMA_OBJ_SIZE(ib_pd, erdma_pd, ibpd), 521 INIT_RDMA_OBJ_SIZE(ib_ucontext, erdma_ucontext, ibucontext), 522 INIT_RDMA_OBJ_SIZE(ib_qp, erdma_qp, ibqp), 523 }; 524 525 static int erdma_ib_device_add(struct pci_dev *pdev) 526 { 527 struct erdma_dev *dev = pci_get_drvdata(pdev); 528 struct ib_device *ibdev = &dev->ibdev; 529 u64 mac; 530 int ret; 531 532 ret = erdma_dev_attrs_init(dev); 533 if (ret) 534 return ret; 535 536 ret = erdma_device_config(dev); 537 if (ret) 538 return ret; 539 540 ibdev->node_type = RDMA_NODE_RNIC; 541 memcpy(ibdev->node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC)); 542 543 /* 544 * Current model (one-to-one device association): 545 * One ERDMA device per net_device or, equivalently, 546 * per physical port. 547 */ 548 ibdev->phys_port_cnt = 1; 549 ibdev->num_comp_vectors = dev->attrs.irq_num - 1; 550 551 ib_set_device_ops(ibdev, &erdma_device_ops); 552 553 INIT_LIST_HEAD(&dev->cep_list); 554 555 spin_lock_init(&dev->lock); 556 xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1); 557 xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1); 558 dev->next_alloc_cqn = 1; 559 dev->next_alloc_qpn = 1; 560 561 ret = erdma_res_cb_init(dev); 562 if (ret) 563 return ret; 564 565 atomic_set(&dev->num_ctx, 0); 566 567 mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG); 568 mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) << 32; 569 570 u64_to_ether_addr(mac, dev->attrs.peer_addr); 571 572 dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND, 573 WQ_UNBOUND_MAX_ACTIVE); 574 if (!dev->reflush_wq) { 575 ret = -ENOMEM; 576 goto err_alloc_workqueue; 577 } 578 579 ret = erdma_device_register(dev); 580 if (ret) 581 goto err_register; 582 583 return 0; 584 585 err_register: 586 destroy_workqueue(dev->reflush_wq); 587 err_alloc_workqueue: 588 xa_destroy(&dev->qp_xa); 589 xa_destroy(&dev->cq_xa); 590 591 erdma_res_cb_free(dev); 592 593 return ret; 594 } 595 596 static void erdma_ib_device_remove(struct pci_dev *pdev) 597 { 598 struct erdma_dev *dev = pci_get_drvdata(pdev); 599 600 unregister_netdevice_notifier(&dev->netdev_nb); 601 ib_unregister_device(&dev->ibdev); 602 603 destroy_workqueue(dev->reflush_wq); 604 erdma_res_cb_free(dev); 605 xa_destroy(&dev->qp_xa); 606 xa_destroy(&dev->cq_xa); 607 } 608 609 static int erdma_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 610 { 611 int ret; 612 613 ret = erdma_probe_dev(pdev); 614 if (ret) 615 return ret; 616 617 ret = erdma_ib_device_add(pdev); 618 if (ret) { 619 erdma_remove_dev(pdev); 620 return ret; 621 } 622 623 return 0; 624 } 625 626 static void erdma_remove(struct pci_dev *pdev) 627 { 628 erdma_ib_device_remove(pdev); 629 erdma_remove_dev(pdev); 630 } 631 632 static struct pci_driver erdma_pci_driver = { 633 .name = DRV_MODULE_NAME, 634 .id_table = erdma_pci_tbl, 635 .probe = erdma_probe, 636 .remove = erdma_remove 637 }; 638 639 MODULE_DEVICE_TABLE(pci, erdma_pci_tbl); 640 641 static __init int erdma_init_module(void) 642 { 643 int ret; 644 645 ret = erdma_cm_init(); 646 if (ret) 647 return ret; 648 649 ret = pci_register_driver(&erdma_pci_driver); 650 if (ret) 651 erdma_cm_exit(); 652 653 return ret; 654 } 655 656 static void __exit erdma_exit_module(void) 657 { 658 pci_unregister_driver(&erdma_pci_driver); 659 660 erdma_cm_exit(); 661 } 662 663 module_init(erdma_init_module); 664 module_exit(erdma_exit_module); 665