1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev, 9 struct net_device *ndev, 10 mana_handle_t default_rxobj, 11 mana_handle_t ind_table[], 12 u32 log_ind_tbl_size, u32 rx_hash_key_len, 13 u8 *rx_hash_key) 14 { 15 struct mana_port_context *mpc = netdev_priv(ndev); 16 struct mana_cfg_rx_steer_req_v2 *req; 17 struct mana_cfg_rx_steer_resp resp = {}; 18 struct gdma_context *gc; 19 u32 req_buf_size; 20 int i, err; 21 22 gc = mdev_to_gc(dev); 23 24 req_buf_size = struct_size(req, indir_tab, MANA_INDIRECT_TABLE_SIZE); 25 req = kzalloc(req_buf_size, GFP_KERNEL); 26 if (!req) 27 return -ENOMEM; 28 29 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 30 sizeof(resp)); 31 32 req->hdr.req.msg_version = GDMA_MESSAGE_V2; 33 34 req->vport = mpc->port_handle; 35 req->rx_enable = 1; 36 req->update_default_rxobj = 1; 37 req->default_rxobj = default_rxobj; 38 req->hdr.dev_id = gc->mana.dev_id; 39 40 /* If there are more than 1 entries in indirection table, enable RSS */ 41 if (log_ind_tbl_size) 42 req->rss_enable = true; 43 44 req->num_indir_entries = MANA_INDIRECT_TABLE_SIZE; 45 req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, 46 indir_tab); 47 req->update_indir_tab = true; 48 req->cqe_coalescing_enable = 1; 49 50 /* The ind table passed to the hardware must have 51 * MANA_INDIRECT_TABLE_SIZE entries. Adjust the verb 52 * ind_table to MANA_INDIRECT_TABLE_SIZE if required 53 */ 54 ibdev_dbg(&dev->ib_dev, "ind table size %u\n", 1 << log_ind_tbl_size); 55 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) { 56 req->indir_tab[i] = ind_table[i % (1 << log_ind_tbl_size)]; 57 ibdev_dbg(&dev->ib_dev, "index %u handle 0x%llx\n", i, 58 req->indir_tab[i]); 59 } 60 61 req->update_hashkey = true; 62 if (rx_hash_key_len) 63 memcpy(req->hashkey, rx_hash_key, rx_hash_key_len); 64 else 65 netdev_rss_key_fill(req->hashkey, MANA_HASH_KEY_SIZE); 66 67 ibdev_dbg(&dev->ib_dev, "vport handle %llu default_rxobj 0x%llx\n", 68 req->vport, default_rxobj); 69 70 err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &resp); 71 if (err) { 72 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); 73 goto out; 74 } 75 76 if (resp.hdr.status) { 77 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", 78 resp.hdr.status); 79 err = -EPROTO; 80 goto out; 81 } 82 83 netdev_info(ndev, "Configured steering vPort %llu log_entries %u\n", 84 mpc->port_handle, log_ind_tbl_size); 85 86 out: 87 kfree(req); 88 return err; 89 } 90 91 static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, 92 struct ib_qp_init_attr *attr, 93 struct ib_udata *udata) 94 { 95 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 96 struct mana_ib_dev *mdev = 97 container_of(pd->device, struct mana_ib_dev, ib_dev); 98 struct gdma_context *gc = mdev_to_gc(mdev); 99 struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl; 100 struct mana_ib_create_qp_rss_resp resp = {}; 101 struct mana_ib_create_qp_rss ucmd = {}; 102 struct gdma_queue **gdma_cq_allocated; 103 mana_handle_t *mana_ind_table; 104 struct mana_port_context *mpc; 105 unsigned int ind_tbl_size; 106 struct net_device *ndev; 107 struct mana_ib_cq *cq; 108 struct mana_ib_wq *wq; 109 struct mana_eq *eq; 110 struct ib_cq *ibcq; 111 struct ib_wq *ibwq; 112 int i = 0; 113 u32 port; 114 int ret; 115 116 if (!udata || udata->inlen < sizeof(ucmd)) 117 return -EINVAL; 118 119 ret = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); 120 if (ret) { 121 ibdev_dbg(&mdev->ib_dev, 122 "Failed copy from udata for create rss-qp, err %d\n", 123 ret); 124 return ret; 125 } 126 127 if (attr->cap.max_recv_wr > mdev->adapter_caps.max_qp_wr) { 128 ibdev_dbg(&mdev->ib_dev, 129 "Requested max_recv_wr %d exceeding limit\n", 130 attr->cap.max_recv_wr); 131 return -EINVAL; 132 } 133 134 if (attr->cap.max_recv_sge > MAX_RX_WQE_SGL_ENTRIES) { 135 ibdev_dbg(&mdev->ib_dev, 136 "Requested max_recv_sge %d exceeding limit\n", 137 attr->cap.max_recv_sge); 138 return -EINVAL; 139 } 140 141 ind_tbl_size = 1 << ind_tbl->log_ind_tbl_size; 142 if (ind_tbl_size > MANA_INDIRECT_TABLE_SIZE) { 143 ibdev_dbg(&mdev->ib_dev, 144 "Indirect table size %d exceeding limit\n", 145 ind_tbl_size); 146 return -EINVAL; 147 } 148 149 if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) { 150 ibdev_dbg(&mdev->ib_dev, 151 "RX Hash function is not supported, %d\n", 152 ucmd.rx_hash_function); 153 return -EINVAL; 154 } 155 156 /* IB ports start with 1, MANA start with 0 */ 157 port = ucmd.port; 158 ndev = mana_ib_get_netdev(pd->device, port); 159 if (!ndev) { 160 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n", 161 port); 162 return -EINVAL; 163 } 164 mpc = netdev_priv(ndev); 165 166 ibdev_dbg(&mdev->ib_dev, "rx_hash_function %d port %d\n", 167 ucmd.rx_hash_function, port); 168 169 mana_ind_table = kcalloc(ind_tbl_size, sizeof(mana_handle_t), 170 GFP_KERNEL); 171 if (!mana_ind_table) { 172 ret = -ENOMEM; 173 goto fail; 174 } 175 176 gdma_cq_allocated = kcalloc(ind_tbl_size, sizeof(*gdma_cq_allocated), 177 GFP_KERNEL); 178 if (!gdma_cq_allocated) { 179 ret = -ENOMEM; 180 goto fail; 181 } 182 183 qp->port = port; 184 185 for (i = 0; i < ind_tbl_size; i++) { 186 struct mana_obj_spec wq_spec = {}; 187 struct mana_obj_spec cq_spec = {}; 188 189 ibwq = ind_tbl->ind_tbl[i]; 190 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 191 192 ibcq = ibwq->cq; 193 cq = container_of(ibcq, struct mana_ib_cq, ibcq); 194 195 wq_spec.gdma_region = wq->gdma_region; 196 wq_spec.queue_size = wq->wq_buf_size; 197 198 cq_spec.gdma_region = cq->gdma_region; 199 cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE; 200 cq_spec.modr_ctx_id = 0; 201 eq = &mpc->ac->eqs[cq->comp_vector % gc->max_num_queues]; 202 cq_spec.attached_eq = eq->eq->id; 203 204 ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ, 205 &wq_spec, &cq_spec, &wq->rx_object); 206 if (ret) { 207 /* Do cleanup starting with index i-1 */ 208 i--; 209 goto fail; 210 } 211 212 /* The GDMA regions are now owned by the WQ object */ 213 wq->gdma_region = GDMA_INVALID_DMA_REGION; 214 cq->gdma_region = GDMA_INVALID_DMA_REGION; 215 216 wq->id = wq_spec.queue_index; 217 cq->id = cq_spec.queue_index; 218 219 ibdev_dbg(&mdev->ib_dev, 220 "ret %d rx_object 0x%llx wq id %llu cq id %llu\n", 221 ret, wq->rx_object, wq->id, cq->id); 222 223 resp.entries[i].cqid = cq->id; 224 resp.entries[i].wqid = wq->id; 225 226 mana_ind_table[i] = wq->rx_object; 227 228 /* Create CQ table entry */ 229 ret = mana_ib_install_cq_cb(mdev, cq); 230 if (ret) 231 goto fail; 232 233 gdma_cq_allocated[i] = gc->cq_table[cq->id]; 234 } 235 resp.num_entries = i; 236 237 ret = mana_ib_cfg_vport_steering(mdev, ndev, wq->rx_object, 238 mana_ind_table, 239 ind_tbl->log_ind_tbl_size, 240 ucmd.rx_hash_key_len, 241 ucmd.rx_hash_key); 242 if (ret) 243 goto fail; 244 245 ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); 246 if (ret) { 247 ibdev_dbg(&mdev->ib_dev, 248 "Failed to copy to udata create rss-qp, %d\n", 249 ret); 250 goto fail; 251 } 252 253 kfree(gdma_cq_allocated); 254 kfree(mana_ind_table); 255 256 return 0; 257 258 fail: 259 while (i-- > 0) { 260 ibwq = ind_tbl->ind_tbl[i]; 261 ibcq = ibwq->cq; 262 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 263 cq = container_of(ibcq, struct mana_ib_cq, ibcq); 264 265 gc->cq_table[cq->id] = NULL; 266 kfree(gdma_cq_allocated[i]); 267 268 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 269 } 270 271 kfree(gdma_cq_allocated); 272 kfree(mana_ind_table); 273 274 return ret; 275 } 276 277 static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, 278 struct ib_qp_init_attr *attr, 279 struct ib_udata *udata) 280 { 281 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 282 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 283 struct mana_ib_dev *mdev = 284 container_of(ibpd->device, struct mana_ib_dev, ib_dev); 285 struct mana_ib_cq *send_cq = 286 container_of(attr->send_cq, struct mana_ib_cq, ibcq); 287 struct mana_ib_ucontext *mana_ucontext = 288 rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 289 ibucontext); 290 struct gdma_context *gc = mdev_to_gc(mdev); 291 struct mana_ib_create_qp_resp resp = {}; 292 struct mana_ib_create_qp ucmd = {}; 293 struct gdma_queue *gdma_cq = NULL; 294 struct mana_obj_spec wq_spec = {}; 295 struct mana_obj_spec cq_spec = {}; 296 struct mana_port_context *mpc; 297 struct net_device *ndev; 298 struct ib_umem *umem; 299 struct mana_eq *eq; 300 int eq_vec; 301 u32 port; 302 int err; 303 304 if (!mana_ucontext || udata->inlen < sizeof(ucmd)) 305 return -EINVAL; 306 307 err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); 308 if (err) { 309 ibdev_dbg(&mdev->ib_dev, 310 "Failed to copy from udata create qp-raw, %d\n", err); 311 return err; 312 } 313 314 if (attr->cap.max_send_wr > mdev->adapter_caps.max_qp_wr) { 315 ibdev_dbg(&mdev->ib_dev, 316 "Requested max_send_wr %d exceeding limit\n", 317 attr->cap.max_send_wr); 318 return -EINVAL; 319 } 320 321 if (attr->cap.max_send_sge > MAX_TX_WQE_SGL_ENTRIES) { 322 ibdev_dbg(&mdev->ib_dev, 323 "Requested max_send_sge %d exceeding limit\n", 324 attr->cap.max_send_sge); 325 return -EINVAL; 326 } 327 328 port = ucmd.port; 329 ndev = mana_ib_get_netdev(ibpd->device, port); 330 if (!ndev) { 331 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n", 332 port); 333 return -EINVAL; 334 } 335 mpc = netdev_priv(ndev); 336 ibdev_dbg(&mdev->ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc); 337 338 err = mana_ib_cfg_vport(mdev, port, pd, mana_ucontext->doorbell); 339 if (err) 340 return -ENODEV; 341 342 qp->port = port; 343 344 ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", 345 ucmd.sq_buf_addr, ucmd.port); 346 347 umem = ib_umem_get(ibpd->device, ucmd.sq_buf_addr, ucmd.sq_buf_size, 348 IB_ACCESS_LOCAL_WRITE); 349 if (IS_ERR(umem)) { 350 err = PTR_ERR(umem); 351 ibdev_dbg(&mdev->ib_dev, 352 "Failed to get umem for create qp-raw, err %d\n", 353 err); 354 goto err_free_vport; 355 } 356 qp->sq_umem = umem; 357 358 err = mana_ib_create_zero_offset_dma_region(mdev, qp->sq_umem, 359 &qp->sq_gdma_region); 360 if (err) { 361 ibdev_dbg(&mdev->ib_dev, 362 "Failed to create dma region for create qp-raw, %d\n", 363 err); 364 goto err_release_umem; 365 } 366 367 ibdev_dbg(&mdev->ib_dev, 368 "create_dma_region ret %d gdma_region 0x%llx\n", 369 err, qp->sq_gdma_region); 370 371 /* Create a WQ on the same port handle used by the Ethernet */ 372 wq_spec.gdma_region = qp->sq_gdma_region; 373 wq_spec.queue_size = ucmd.sq_buf_size; 374 375 cq_spec.gdma_region = send_cq->gdma_region; 376 cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE; 377 cq_spec.modr_ctx_id = 0; 378 eq_vec = send_cq->comp_vector % gc->max_num_queues; 379 eq = &mpc->ac->eqs[eq_vec]; 380 cq_spec.attached_eq = eq->eq->id; 381 382 err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec, 383 &cq_spec, &qp->tx_object); 384 if (err) { 385 ibdev_dbg(&mdev->ib_dev, 386 "Failed to create wq for create raw-qp, err %d\n", 387 err); 388 goto err_destroy_dma_region; 389 } 390 391 /* The GDMA regions are now owned by the WQ object */ 392 qp->sq_gdma_region = GDMA_INVALID_DMA_REGION; 393 send_cq->gdma_region = GDMA_INVALID_DMA_REGION; 394 395 qp->sq_id = wq_spec.queue_index; 396 send_cq->id = cq_spec.queue_index; 397 398 /* Create CQ table entry */ 399 err = mana_ib_install_cq_cb(mdev, send_cq); 400 if (err) 401 goto err_destroy_wq_obj; 402 403 ibdev_dbg(&mdev->ib_dev, 404 "ret %d qp->tx_object 0x%llx sq id %llu cq id %llu\n", err, 405 qp->tx_object, qp->sq_id, send_cq->id); 406 407 resp.sqid = qp->sq_id; 408 resp.cqid = send_cq->id; 409 resp.tx_vp_offset = pd->tx_vp_offset; 410 411 err = ib_copy_to_udata(udata, &resp, sizeof(resp)); 412 if (err) { 413 ibdev_dbg(&mdev->ib_dev, 414 "Failed copy udata for create qp-raw, %d\n", 415 err); 416 goto err_release_gdma_cq; 417 } 418 419 return 0; 420 421 err_release_gdma_cq: 422 kfree(gdma_cq); 423 gc->cq_table[send_cq->id] = NULL; 424 425 err_destroy_wq_obj: 426 mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object); 427 428 err_destroy_dma_region: 429 mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region); 430 431 err_release_umem: 432 ib_umem_release(umem); 433 434 err_free_vport: 435 mana_ib_uncfg_vport(mdev, pd, port); 436 437 return err; 438 } 439 440 int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 441 struct ib_udata *udata) 442 { 443 switch (attr->qp_type) { 444 case IB_QPT_RAW_PACKET: 445 /* When rwq_ind_tbl is used, it's for creating WQs for RSS */ 446 if (attr->rwq_ind_tbl) 447 return mana_ib_create_qp_rss(ibqp, ibqp->pd, attr, 448 udata); 449 450 return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata); 451 default: 452 /* Creating QP other than IB_QPT_RAW_PACKET is not supported */ 453 ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n", 454 attr->qp_type); 455 } 456 457 return -EINVAL; 458 } 459 460 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 461 int attr_mask, struct ib_udata *udata) 462 { 463 /* modify_qp is not supported by this version of the driver */ 464 return -EOPNOTSUPP; 465 } 466 467 static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, 468 struct ib_rwq_ind_table *ind_tbl, 469 struct ib_udata *udata) 470 { 471 struct mana_ib_dev *mdev = 472 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 473 struct mana_port_context *mpc; 474 struct net_device *ndev; 475 struct mana_ib_wq *wq; 476 struct ib_wq *ibwq; 477 int i; 478 479 ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); 480 mpc = netdev_priv(ndev); 481 482 for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) { 483 ibwq = ind_tbl->ind_tbl[i]; 484 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 485 ibdev_dbg(&mdev->ib_dev, "destroying wq->rx_object %llu\n", 486 wq->rx_object); 487 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 488 } 489 490 return 0; 491 } 492 493 static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata) 494 { 495 struct mana_ib_dev *mdev = 496 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 497 struct ib_pd *ibpd = qp->ibqp.pd; 498 struct mana_port_context *mpc; 499 struct net_device *ndev; 500 struct mana_ib_pd *pd; 501 502 ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); 503 mpc = netdev_priv(ndev); 504 pd = container_of(ibpd, struct mana_ib_pd, ibpd); 505 506 mana_destroy_wq_obj(mpc, GDMA_SQ, qp->tx_object); 507 508 if (qp->sq_umem) { 509 mana_ib_gd_destroy_dma_region(mdev, qp->sq_gdma_region); 510 ib_umem_release(qp->sq_umem); 511 } 512 513 mana_ib_uncfg_vport(mdev, pd, qp->port); 514 515 return 0; 516 } 517 518 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 519 { 520 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 521 522 switch (ibqp->qp_type) { 523 case IB_QPT_RAW_PACKET: 524 if (ibqp->rwq_ind_tbl) 525 return mana_ib_destroy_qp_rss(qp, ibqp->rwq_ind_tbl, 526 udata); 527 528 return mana_ib_destroy_qp_raw(qp, udata); 529 530 default: 531 ibdev_dbg(ibqp->device, "Unexpected QP type %u\n", 532 ibqp->qp_type); 533 } 534 535 return -ENOENT; 536 } 537