1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #include <linux/module.h> 5 #include <linux/printk.h> 6 #include <linux/pci.h> 7 #include <linux/irq.h> 8 #include <net/addrconf.h> 9 #include <rdma/ib_addr.h> 10 #include <rdma/ib_mad.h> 11 12 #include "ionic_ibdev.h" 13 14 #define DRIVER_DESCRIPTION "AMD Pensando RoCE HCA driver" 15 #define DEVICE_DESCRIPTION "AMD Pensando RoCE HCA" 16 17 MODULE_AUTHOR("Allen Hubbe <allen.hubbe@amd.com>"); 18 MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 19 MODULE_LICENSE("GPL"); 20 MODULE_IMPORT_NS("NET_IONIC"); 21 22 static int ionic_query_device(struct ib_device *ibdev, 23 struct ib_device_attr *attr, 24 struct ib_udata *udata) 25 { 26 struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); 27 struct net_device *ndev; 28 int err; 29 30 err = ib_no_udata_io(udata); 31 if (err) 32 return err; 33 34 ndev = ib_device_get_netdev(ibdev, 1); 35 addrconf_ifid_eui48((u8 *)&attr->sys_image_guid, ndev); 36 dev_put(ndev); 37 attr->max_mr_size = dev->lif_cfg.npts_per_lif * PAGE_SIZE / 2; 38 attr->page_size_cap = dev->lif_cfg.page_size_supported; 39 40 attr->vendor_id = to_pci_dev(dev->lif_cfg.hwdev)->vendor; 41 attr->vendor_part_id = to_pci_dev(dev->lif_cfg.hwdev)->device; 42 43 attr->hw_ver = ionic_lif_asic_rev(dev->lif_cfg.lif); 44 attr->fw_ver = 0; 45 attr->max_qp = dev->lif_cfg.qp_count; 46 attr->max_qp_wr = IONIC_MAX_DEPTH; 47 attr->device_cap_flags = 48 IB_DEVICE_MEM_WINDOW | 49 IB_DEVICE_MEM_MGT_EXTENSIONS | 50 IB_DEVICE_MEM_WINDOW_TYPE_2B | 51 0; 52 attr->max_send_sge = 53 min(ionic_v1_send_wqe_max_sge(dev->lif_cfg.max_stride, 0, false), 54 IONIC_SPEC_HIGH); 55 attr->max_recv_sge = 56 min(ionic_v1_recv_wqe_max_sge(dev->lif_cfg.max_stride, 0, false), 57 IONIC_SPEC_HIGH); 58 attr->max_sge_rd = attr->max_send_sge; 59 attr->max_cq = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count; 60 attr->max_cqe = IONIC_MAX_CQ_DEPTH - IONIC_CQ_GRACE; 61 attr->max_mr = dev->lif_cfg.nmrs_per_lif; 62 attr->max_pd = IONIC_MAX_PD; 63 attr->max_qp_rd_atom = IONIC_MAX_RD_ATOM; 64 attr->max_ee_rd_atom = 0; 65 attr->max_res_rd_atom = IONIC_MAX_RD_ATOM; 66 attr->max_qp_init_rd_atom = IONIC_MAX_RD_ATOM; 67 attr->max_ee_init_rd_atom = 0; 68 attr->atomic_cap = IB_ATOMIC_GLOB; 69 attr->masked_atomic_cap = IB_ATOMIC_GLOB; 70 attr->max_mw = dev->lif_cfg.nmrs_per_lif; 71 attr->max_mcast_grp = 0; 72 attr->max_mcast_qp_attach = 0; 73 attr->max_ah = dev->lif_cfg.nahs_per_lif; 74 attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2; 75 attr->max_pkeys = IONIC_PKEY_TBL_LEN; 76 if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) { 77 attr->max_srq = dev->lif_cfg.srq_count; 78 attr->max_srq_wr = IONIC_MAX_SRQ_DEPTH; 79 attr->max_srq_sge = IONIC_MAX_SRQ_SGES; 80 } 81 82 return 0; 83 } 84 85 static int ionic_query_port(struct ib_device *ibdev, u32 port, 86 struct ib_port_attr *attr) 87 { 88 struct net_device *ndev; 89 90 if (port != 1) 91 return -EINVAL; 92 93 ndev = ib_device_get_netdev(ibdev, port); 94 if (!ndev) 95 return -ENODEV; 96 97 if (netif_running(ndev) && netif_carrier_ok(ndev)) { 98 attr->state = IB_PORT_ACTIVE; 99 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 100 } else if (netif_running(ndev)) { 101 attr->state = IB_PORT_DOWN; 102 attr->phys_state = IB_PORT_PHYS_STATE_POLLING; 103 } else { 104 attr->state = IB_PORT_DOWN; 105 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 106 } 107 108 attr->max_mtu = iboe_get_mtu(ndev->max_mtu); 109 attr->active_mtu = min(attr->max_mtu, iboe_get_mtu(ndev->mtu)); 110 attr->gid_tbl_len = IONIC_GID_TBL_LEN; 111 attr->ip_gids = true; 112 attr->port_cap_flags = 0; 113 attr->max_msg_sz = 0x80000000; 114 attr->pkey_tbl_len = IONIC_PKEY_TBL_LEN; 115 attr->max_vl_num = 1; 116 attr->subnet_prefix = 0xfe80000000000000ull; 117 118 dev_put(ndev); 119 120 return ib_get_eth_speed(ibdev, port, 121 &attr->active_speed, 122 &attr->active_width); 123 } 124 125 static enum rdma_link_layer ionic_get_link_layer(struct ib_device *ibdev, 126 u32 port) 127 { 128 return IB_LINK_LAYER_ETHERNET; 129 } 130 131 static int ionic_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 132 u16 *pkey) 133 { 134 if (port != 1) 135 return -EINVAL; 136 137 if (index != 0) 138 return -EINVAL; 139 140 *pkey = IB_DEFAULT_PKEY_FULL; 141 142 return 0; 143 } 144 145 static int ionic_modify_device(struct ib_device *ibdev, int mask, 146 struct ib_device_modify *attr) 147 { 148 struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); 149 150 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) 151 return -EOPNOTSUPP; 152 153 if (mask & IB_DEVICE_MODIFY_NODE_DESC) 154 memcpy(dev->ibdev.node_desc, attr->node_desc, 155 IB_DEVICE_NODE_DESC_MAX); 156 157 return 0; 158 } 159 160 static int ionic_get_port_immutable(struct ib_device *ibdev, u32 port, 161 struct ib_port_immutable *attr) 162 { 163 if (port != 1) 164 return -EINVAL; 165 166 attr->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 167 168 attr->pkey_tbl_len = IONIC_PKEY_TBL_LEN; 169 attr->gid_tbl_len = IONIC_GID_TBL_LEN; 170 attr->max_mad_size = IB_MGMT_MAD_SIZE; 171 172 return 0; 173 } 174 175 static void ionic_get_dev_fw_str(struct ib_device *ibdev, char *str) 176 { 177 struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); 178 179 ionic_lif_fw_version(dev->lif_cfg.lif, str, IB_FW_VERSION_NAME_MAX); 180 } 181 182 static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr, 183 char *buf) 184 { 185 struct ionic_ibdev *dev = 186 rdma_device_to_drv_device(device, struct ionic_ibdev, ibdev); 187 188 return sysfs_emit(buf, "0x%x\n", ionic_lif_asic_rev(dev->lif_cfg.lif)); 189 } 190 static DEVICE_ATTR_RO(hw_rev); 191 192 static ssize_t hca_type_show(struct device *device, 193 struct device_attribute *attr, char *buf) 194 { 195 struct ionic_ibdev *dev = 196 rdma_device_to_drv_device(device, struct ionic_ibdev, ibdev); 197 198 return sysfs_emit(buf, "%.64s\n", dev->ibdev.node_desc); 199 } 200 static DEVICE_ATTR_RO(hca_type); 201 202 static struct attribute *ionic_rdma_attributes[] = { 203 &dev_attr_hw_rev.attr, 204 &dev_attr_hca_type.attr, 205 NULL 206 }; 207 208 static const struct attribute_group ionic_rdma_attr_group = { 209 .attrs = ionic_rdma_attributes, 210 }; 211 212 static void ionic_disassociate_ucontext(struct ib_ucontext *ibcontext) 213 { 214 /* 215 * Dummy define disassociate_ucontext so that it does not 216 * wait for user context before cleaning up hw resources. 217 */ 218 } 219 220 static const struct ib_device_ops ionic_dev_ops = { 221 .owner = THIS_MODULE, 222 .driver_id = RDMA_DRIVER_IONIC, 223 .uverbs_abi_ver = IONIC_ABI_VERSION, 224 .uverbs_robust_udata = true, 225 226 .alloc_ucontext = ionic_alloc_ucontext, 227 .dealloc_ucontext = ionic_dealloc_ucontext, 228 .mmap = ionic_mmap, 229 .mmap_free = ionic_mmap_free, 230 .alloc_pd = ionic_alloc_pd, 231 .dealloc_pd = ionic_dealloc_pd, 232 .create_ah = ionic_create_ah, 233 .query_ah = ionic_query_ah, 234 .destroy_ah = ionic_destroy_ah, 235 .create_user_ah = ionic_create_ah, 236 .get_dma_mr = ionic_get_dma_mr, 237 .reg_user_mr = ionic_reg_user_mr, 238 .reg_user_mr_dmabuf = ionic_reg_user_mr_dmabuf, 239 .dereg_mr = ionic_dereg_mr, 240 .alloc_mr = ionic_alloc_mr, 241 .map_mr_sg = ionic_map_mr_sg, 242 .alloc_mw = ionic_alloc_mw, 243 .dealloc_mw = ionic_dealloc_mw, 244 .create_cq = ionic_create_cq, 245 .destroy_cq = ionic_destroy_cq, 246 .create_qp = ionic_create_qp, 247 .modify_qp = ionic_modify_qp, 248 .query_qp = ionic_query_qp, 249 .destroy_qp = ionic_destroy_qp, 250 251 .post_send = ionic_post_send, 252 .post_recv = ionic_post_recv, 253 .poll_cq = ionic_poll_cq, 254 .req_notify_cq = ionic_req_notify_cq, 255 256 .query_device = ionic_query_device, 257 .query_port = ionic_query_port, 258 .get_link_layer = ionic_get_link_layer, 259 .query_pkey = ionic_query_pkey, 260 .modify_device = ionic_modify_device, 261 .get_port_immutable = ionic_get_port_immutable, 262 .get_dev_fw_str = ionic_get_dev_fw_str, 263 .device_group = &ionic_rdma_attr_group, 264 .disassociate_ucontext = ionic_disassociate_ucontext, 265 266 INIT_RDMA_OBJ_SIZE(ib_ucontext, ionic_ctx, ibctx), 267 INIT_RDMA_OBJ_SIZE(ib_pd, ionic_pd, ibpd), 268 INIT_RDMA_OBJ_SIZE(ib_ah, ionic_ah, ibah), 269 INIT_RDMA_OBJ_SIZE(ib_cq, ionic_vcq, ibcq), 270 INIT_RDMA_OBJ_SIZE(ib_qp, ionic_qp, ibqp), 271 INIT_RDMA_OBJ_SIZE(ib_mw, ionic_mr, ibmw), 272 }; 273 274 static const struct ib_device_ops ionic_srq_ops = { 275 .create_srq = ionic_create_srq, 276 .modify_srq = ionic_modify_srq, 277 .query_srq = ionic_query_srq, 278 .destroy_srq = ionic_destroy_srq, 279 280 INIT_RDMA_OBJ_SIZE(ib_srq, ionic_srq, ibsrq), 281 }; 282 283 static void ionic_init_resids(struct ionic_ibdev *dev) 284 { 285 ionic_resid_init(&dev->inuse_cqid, dev->lif_cfg.cq_count); 286 dev->half_cqid_udma_shift = 287 order_base_2(dev->lif_cfg.cq_count / dev->lif_cfg.udma_count); 288 ionic_resid_init(&dev->inuse_pdid, IONIC_MAX_PD); 289 ionic_resid_init(&dev->inuse_ahid, dev->lif_cfg.nahs_per_lif); 290 ionic_resid_init(&dev->inuse_mrid, dev->lif_cfg.nmrs_per_lif); 291 /* skip reserved lkey */ 292 dev->next_mrkey = 1; 293 ionic_resid_init(&dev->inuse_qpid, dev->lif_cfg.qp_count); 294 /* skip reserved SMI and GSI qpids */ 295 dev->half_qpid_udma_shift = 296 order_base_2(dev->lif_cfg.qp_count / dev->lif_cfg.udma_count); 297 ionic_resid_init(&dev->inuse_dbid, dev->lif_cfg.dbid_count); 298 } 299 300 static void ionic_destroy_resids(struct ionic_ibdev *dev) 301 { 302 ionic_resid_destroy(&dev->inuse_cqid); 303 ionic_resid_destroy(&dev->inuse_pdid); 304 ionic_resid_destroy(&dev->inuse_ahid); 305 ionic_resid_destroy(&dev->inuse_mrid); 306 ionic_resid_destroy(&dev->inuse_qpid); 307 ionic_resid_destroy(&dev->inuse_dbid); 308 } 309 310 static void ionic_destroy_ibdev(struct ionic_ibdev *dev) 311 { 312 ionic_kill_rdma_admin(dev, false); 313 ib_unregister_device(&dev->ibdev); 314 ionic_stats_cleanup(dev); 315 ionic_destroy_rdma_admin(dev); 316 ionic_destroy_resids(dev); 317 WARN_ON(!xa_empty(&dev->qp_tbl)); 318 xa_destroy(&dev->qp_tbl); 319 WARN_ON(!xa_empty(&dev->cq_tbl)); 320 xa_destroy(&dev->cq_tbl); 321 WARN_ON(!xa_empty(&dev->srq_tbl)); 322 xa_destroy(&dev->srq_tbl); 323 ib_dealloc_device(&dev->ibdev); 324 } 325 326 static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) 327 { 328 struct ib_device *ibdev; 329 struct ionic_ibdev *dev; 330 struct net_device *ndev; 331 int rc; 332 333 dev = ib_alloc_device(ionic_ibdev, ibdev); 334 if (!dev) 335 return ERR_PTR(-EINVAL); 336 337 ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg); 338 339 xa_init_flags(&dev->qp_tbl, XA_FLAGS_LOCK_IRQ); 340 xa_init_flags(&dev->cq_tbl, XA_FLAGS_LOCK_IRQ); 341 xa_init_flags(&dev->srq_tbl, XA_FLAGS_LOCK_IRQ); 342 343 ionic_init_resids(dev); 344 345 rc = ionic_rdma_reset_devcmd(dev); 346 if (rc) 347 goto err_reset; 348 349 rc = ionic_create_rdma_admin(dev); 350 if (rc) 351 goto err_admin; 352 353 ibdev = &dev->ibdev; 354 ibdev->dev.parent = dev->lif_cfg.hwdev; 355 356 strscpy(ibdev->name, "ionic_%d", IB_DEVICE_NAME_MAX); 357 strscpy(ibdev->node_desc, DEVICE_DESCRIPTION, IB_DEVICE_NODE_DESC_MAX); 358 359 ibdev->node_type = RDMA_NODE_IB_CA; 360 ibdev->phys_port_cnt = 1; 361 362 /* the first two eq are reserved for async events */ 363 ibdev->num_comp_vectors = dev->lif_cfg.eq_count - 2; 364 365 ndev = ionic_lif_netdev(ionic_adev->lif); 366 addrconf_ifid_eui48((u8 *)&ibdev->node_guid, ndev); 367 rc = ib_device_set_netdev(ibdev, ndev, 1); 368 /* ionic_lif_netdev() returns ndev with refcount held */ 369 dev_put(ndev); 370 if (rc) 371 goto err_admin; 372 373 ib_set_device_ops(&dev->ibdev, &ionic_dev_ops); 374 if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) 375 ib_set_device_ops(&dev->ibdev, &ionic_srq_ops); 376 377 ionic_stats_init(dev); 378 379 rc = ib_register_device(ibdev, "ionic_%d", ibdev->dev.parent); 380 if (rc) 381 goto err_register; 382 383 return dev; 384 385 err_register: 386 ionic_stats_cleanup(dev); 387 err_admin: 388 ionic_kill_rdma_admin(dev, false); 389 ionic_destroy_rdma_admin(dev); 390 err_reset: 391 ionic_destroy_resids(dev); 392 xa_destroy(&dev->qp_tbl); 393 xa_destroy(&dev->cq_tbl); 394 xa_destroy(&dev->srq_tbl); 395 ib_dealloc_device(&dev->ibdev); 396 397 return ERR_PTR(rc); 398 } 399 400 static int ionic_aux_probe(struct auxiliary_device *adev, 401 const struct auxiliary_device_id *id) 402 { 403 struct ionic_aux_dev *ionic_adev; 404 struct ionic_ibdev *dev; 405 406 ionic_adev = container_of(adev, struct ionic_aux_dev, adev); 407 dev = ionic_create_ibdev(ionic_adev); 408 if (IS_ERR(dev)) 409 return dev_err_probe(&adev->dev, PTR_ERR(dev), 410 "Failed to register ibdev\n"); 411 412 auxiliary_set_drvdata(adev, dev); 413 ibdev_dbg(&dev->ibdev, "registered\n"); 414 415 return 0; 416 } 417 418 static void ionic_aux_remove(struct auxiliary_device *adev) 419 { 420 struct ionic_ibdev *dev = auxiliary_get_drvdata(adev); 421 422 dev_dbg(&adev->dev, "unregister ibdev\n"); 423 ionic_destroy_ibdev(dev); 424 dev_dbg(&adev->dev, "unregistered\n"); 425 } 426 427 static const struct auxiliary_device_id ionic_aux_id_table[] = { 428 { .name = "ionic.rdma", }, 429 {}, 430 }; 431 432 MODULE_DEVICE_TABLE(auxiliary, ionic_aux_id_table); 433 434 static struct auxiliary_driver ionic_aux_r_driver = { 435 .name = "rdma", 436 .probe = ionic_aux_probe, 437 .remove = ionic_aux_remove, 438 .id_table = ionic_aux_id_table, 439 }; 440 441 static int __init ionic_mod_init(void) 442 { 443 int rc; 444 445 ionic_evt_workq = create_workqueue(KBUILD_MODNAME "-evt"); 446 if (!ionic_evt_workq) 447 return -ENOMEM; 448 449 rc = auxiliary_driver_register(&ionic_aux_r_driver); 450 if (rc) 451 goto err_aux; 452 453 return 0; 454 455 err_aux: 456 destroy_workqueue(ionic_evt_workq); 457 458 return rc; 459 } 460 461 static void __exit ionic_mod_exit(void) 462 { 463 auxiliary_driver_unregister(&ionic_aux_r_driver); 464 destroy_workqueue(ionic_evt_workq); 465 } 466 467 module_init(ionic_mod_init); 468 module_exit(ionic_mod_exit); 469