1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev, 9 struct net_device *ndev, 10 mana_handle_t default_rxobj, 11 mana_handle_t ind_table[], 12 u32 log_ind_tbl_size, u32 rx_hash_key_len, 13 u8 *rx_hash_key) 14 { 15 struct mana_port_context *mpc = netdev_priv(ndev); 16 struct mana_cfg_rx_steer_req_v2 *req; 17 struct mana_cfg_rx_steer_resp resp = {}; 18 struct gdma_context *gc; 19 u32 req_buf_size; 20 int i, err; 21 22 gc = mdev_to_gc(dev); 23 24 req_buf_size = struct_size(req, indir_tab, MANA_INDIRECT_TABLE_DEF_SIZE); 25 req = kzalloc(req_buf_size, GFP_KERNEL); 26 if (!req) 27 return -ENOMEM; 28 29 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 30 sizeof(resp)); 31 32 req->hdr.req.msg_version = GDMA_MESSAGE_V2; 33 34 req->vport = mpc->port_handle; 35 req->rx_enable = 1; 36 req->update_default_rxobj = 1; 37 req->default_rxobj = default_rxobj; 38 req->hdr.dev_id = gc->mana.dev_id; 39 40 /* If there are more than 1 entries in indirection table, enable RSS */ 41 if (log_ind_tbl_size) 42 req->rss_enable = true; 43 44 req->num_indir_entries = MANA_INDIRECT_TABLE_DEF_SIZE; 45 req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, 46 indir_tab); 47 req->update_indir_tab = true; 48 req->cqe_coalescing_enable = 1; 49 50 /* The ind table passed to the hardware must have 51 * MANA_INDIRECT_TABLE_DEF_SIZE entries. Adjust the verb 52 * ind_table to MANA_INDIRECT_TABLE_SIZE if required 53 */ 54 ibdev_dbg(&dev->ib_dev, "ind table size %u\n", 1 << log_ind_tbl_size); 55 for (i = 0; i < MANA_INDIRECT_TABLE_DEF_SIZE; i++) { 56 req->indir_tab[i] = ind_table[i % (1 << log_ind_tbl_size)]; 57 ibdev_dbg(&dev->ib_dev, "index %u handle 0x%llx\n", i, 58 req->indir_tab[i]); 59 } 60 61 req->update_hashkey = true; 62 if (rx_hash_key_len) 63 memcpy(req->hashkey, rx_hash_key, rx_hash_key_len); 64 else 65 netdev_rss_key_fill(req->hashkey, MANA_HASH_KEY_SIZE); 66 67 ibdev_dbg(&dev->ib_dev, "vport handle %llu default_rxobj 0x%llx\n", 68 req->vport, default_rxobj); 69 70 err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &resp); 71 if (err) { 72 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); 73 goto out; 74 } 75 76 if (resp.hdr.status) { 77 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", 78 resp.hdr.status); 79 err = -EPROTO; 80 goto out; 81 } 82 83 netdev_info(ndev, "Configured steering vPort %llu log_entries %u\n", 84 mpc->port_handle, log_ind_tbl_size); 85 86 out: 87 kfree(req); 88 return err; 89 } 90 91 static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, 92 struct ib_qp_init_attr *attr, 93 struct ib_udata *udata) 94 { 95 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 96 struct mana_ib_dev *mdev = 97 container_of(pd->device, struct mana_ib_dev, ib_dev); 98 struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl; 99 struct mana_ib_create_qp_rss_resp resp = {}; 100 struct mana_ib_create_qp_rss ucmd = {}; 101 mana_handle_t *mana_ind_table; 102 struct mana_port_context *mpc; 103 unsigned int ind_tbl_size; 104 struct net_device *ndev; 105 struct mana_ib_cq *cq; 106 struct mana_ib_wq *wq; 107 struct mana_eq *eq; 108 struct ib_cq *ibcq; 109 struct ib_wq *ibwq; 110 int i = 0; 111 u32 port; 112 int ret; 113 114 if (!udata || udata->inlen < sizeof(ucmd)) 115 return -EINVAL; 116 117 ret = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); 118 if (ret) { 119 ibdev_dbg(&mdev->ib_dev, 120 "Failed copy from udata for create rss-qp, err %d\n", 121 ret); 122 return ret; 123 } 124 125 if (attr->cap.max_recv_wr > mdev->adapter_caps.max_qp_wr) { 126 ibdev_dbg(&mdev->ib_dev, 127 "Requested max_recv_wr %d exceeding limit\n", 128 attr->cap.max_recv_wr); 129 return -EINVAL; 130 } 131 132 if (attr->cap.max_recv_sge > MAX_RX_WQE_SGL_ENTRIES) { 133 ibdev_dbg(&mdev->ib_dev, 134 "Requested max_recv_sge %d exceeding limit\n", 135 attr->cap.max_recv_sge); 136 return -EINVAL; 137 } 138 139 ind_tbl_size = 1 << ind_tbl->log_ind_tbl_size; 140 if (ind_tbl_size > MANA_INDIRECT_TABLE_DEF_SIZE) { 141 ibdev_dbg(&mdev->ib_dev, 142 "Indirect table size %d exceeding limit\n", 143 ind_tbl_size); 144 return -EINVAL; 145 } 146 147 if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) { 148 ibdev_dbg(&mdev->ib_dev, 149 "RX Hash function is not supported, %d\n", 150 ucmd.rx_hash_function); 151 return -EINVAL; 152 } 153 154 /* IB ports start with 1, MANA start with 0 */ 155 port = ucmd.port; 156 ndev = mana_ib_get_netdev(pd->device, port); 157 if (!ndev) { 158 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n", 159 port); 160 return -EINVAL; 161 } 162 mpc = netdev_priv(ndev); 163 164 ibdev_dbg(&mdev->ib_dev, "rx_hash_function %d port %d\n", 165 ucmd.rx_hash_function, port); 166 167 mana_ind_table = kcalloc(ind_tbl_size, sizeof(mana_handle_t), 168 GFP_KERNEL); 169 if (!mana_ind_table) { 170 ret = -ENOMEM; 171 goto fail; 172 } 173 174 qp->port = port; 175 176 for (i = 0; i < ind_tbl_size; i++) { 177 struct mana_obj_spec wq_spec = {}; 178 struct mana_obj_spec cq_spec = {}; 179 180 ibwq = ind_tbl->ind_tbl[i]; 181 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 182 183 ibcq = ibwq->cq; 184 cq = container_of(ibcq, struct mana_ib_cq, ibcq); 185 186 wq_spec.gdma_region = wq->queue.gdma_region; 187 wq_spec.queue_size = wq->wq_buf_size; 188 189 cq_spec.gdma_region = cq->queue.gdma_region; 190 cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE; 191 cq_spec.modr_ctx_id = 0; 192 eq = &mpc->ac->eqs[cq->comp_vector]; 193 cq_spec.attached_eq = eq->eq->id; 194 195 ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ, 196 &wq_spec, &cq_spec, &wq->rx_object); 197 if (ret) { 198 /* Do cleanup starting with index i-1 */ 199 i--; 200 goto fail; 201 } 202 203 /* The GDMA regions are now owned by the WQ object */ 204 wq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 205 cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 206 207 wq->queue.id = wq_spec.queue_index; 208 cq->queue.id = cq_spec.queue_index; 209 210 ibdev_dbg(&mdev->ib_dev, 211 "rx_object 0x%llx wq id %llu cq id %llu\n", 212 wq->rx_object, wq->queue.id, cq->queue.id); 213 214 resp.entries[i].cqid = cq->queue.id; 215 resp.entries[i].wqid = wq->queue.id; 216 217 mana_ind_table[i] = wq->rx_object; 218 219 /* Create CQ table entry */ 220 ret = mana_ib_install_cq_cb(mdev, cq); 221 if (ret) 222 goto fail; 223 } 224 resp.num_entries = i; 225 226 ret = mana_ib_cfg_vport_steering(mdev, ndev, wq->rx_object, 227 mana_ind_table, 228 ind_tbl->log_ind_tbl_size, 229 ucmd.rx_hash_key_len, 230 ucmd.rx_hash_key); 231 if (ret) 232 goto fail; 233 234 ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); 235 if (ret) { 236 ibdev_dbg(&mdev->ib_dev, 237 "Failed to copy to udata create rss-qp, %d\n", 238 ret); 239 goto fail; 240 } 241 242 kfree(mana_ind_table); 243 244 return 0; 245 246 fail: 247 while (i-- > 0) { 248 ibwq = ind_tbl->ind_tbl[i]; 249 ibcq = ibwq->cq; 250 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 251 cq = container_of(ibcq, struct mana_ib_cq, ibcq); 252 253 mana_ib_remove_cq_cb(mdev, cq); 254 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 255 } 256 257 kfree(mana_ind_table); 258 259 return ret; 260 } 261 262 static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, 263 struct ib_qp_init_attr *attr, 264 struct ib_udata *udata) 265 { 266 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 267 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 268 struct mana_ib_dev *mdev = 269 container_of(ibpd->device, struct mana_ib_dev, ib_dev); 270 struct mana_ib_cq *send_cq = 271 container_of(attr->send_cq, struct mana_ib_cq, ibcq); 272 struct mana_ib_ucontext *mana_ucontext = 273 rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 274 ibucontext); 275 struct mana_ib_create_qp_resp resp = {}; 276 struct mana_ib_create_qp ucmd = {}; 277 struct mana_obj_spec wq_spec = {}; 278 struct mana_obj_spec cq_spec = {}; 279 struct mana_port_context *mpc; 280 struct net_device *ndev; 281 struct mana_eq *eq; 282 int eq_vec; 283 u32 port; 284 int err; 285 286 if (!mana_ucontext || udata->inlen < sizeof(ucmd)) 287 return -EINVAL; 288 289 err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); 290 if (err) { 291 ibdev_dbg(&mdev->ib_dev, 292 "Failed to copy from udata create qp-raw, %d\n", err); 293 return err; 294 } 295 296 if (attr->cap.max_send_wr > mdev->adapter_caps.max_qp_wr) { 297 ibdev_dbg(&mdev->ib_dev, 298 "Requested max_send_wr %d exceeding limit\n", 299 attr->cap.max_send_wr); 300 return -EINVAL; 301 } 302 303 if (attr->cap.max_send_sge > MAX_TX_WQE_SGL_ENTRIES) { 304 ibdev_dbg(&mdev->ib_dev, 305 "Requested max_send_sge %d exceeding limit\n", 306 attr->cap.max_send_sge); 307 return -EINVAL; 308 } 309 310 port = ucmd.port; 311 ndev = mana_ib_get_netdev(ibpd->device, port); 312 if (!ndev) { 313 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n", 314 port); 315 return -EINVAL; 316 } 317 mpc = netdev_priv(ndev); 318 ibdev_dbg(&mdev->ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc); 319 320 err = mana_ib_cfg_vport(mdev, port, pd, mana_ucontext->doorbell); 321 if (err) 322 return -ENODEV; 323 324 qp->port = port; 325 326 ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", 327 ucmd.sq_buf_addr, ucmd.port); 328 329 err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq); 330 if (err) { 331 ibdev_dbg(&mdev->ib_dev, 332 "Failed to create queue for create qp-raw, err %d\n", err); 333 goto err_free_vport; 334 } 335 336 /* Create a WQ on the same port handle used by the Ethernet */ 337 wq_spec.gdma_region = qp->raw_sq.gdma_region; 338 wq_spec.queue_size = ucmd.sq_buf_size; 339 340 cq_spec.gdma_region = send_cq->queue.gdma_region; 341 cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE; 342 cq_spec.modr_ctx_id = 0; 343 eq_vec = send_cq->comp_vector; 344 eq = &mpc->ac->eqs[eq_vec]; 345 cq_spec.attached_eq = eq->eq->id; 346 347 err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec, 348 &cq_spec, &qp->qp_handle); 349 if (err) { 350 ibdev_dbg(&mdev->ib_dev, 351 "Failed to create wq for create raw-qp, err %d\n", 352 err); 353 goto err_destroy_queue; 354 } 355 356 /* The GDMA regions are now owned by the WQ object */ 357 qp->raw_sq.gdma_region = GDMA_INVALID_DMA_REGION; 358 send_cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 359 360 qp->raw_sq.id = wq_spec.queue_index; 361 send_cq->queue.id = cq_spec.queue_index; 362 363 /* Create CQ table entry */ 364 err = mana_ib_install_cq_cb(mdev, send_cq); 365 if (err) 366 goto err_destroy_wq_obj; 367 368 ibdev_dbg(&mdev->ib_dev, 369 "qp->qp_handle 0x%llx sq id %llu cq id %llu\n", 370 qp->qp_handle, qp->raw_sq.id, send_cq->queue.id); 371 372 resp.sqid = qp->raw_sq.id; 373 resp.cqid = send_cq->queue.id; 374 resp.tx_vp_offset = pd->tx_vp_offset; 375 376 err = ib_copy_to_udata(udata, &resp, sizeof(resp)); 377 if (err) { 378 ibdev_dbg(&mdev->ib_dev, 379 "Failed copy udata for create qp-raw, %d\n", 380 err); 381 goto err_remove_cq_cb; 382 } 383 384 return 0; 385 386 err_remove_cq_cb: 387 mana_ib_remove_cq_cb(mdev, send_cq); 388 389 err_destroy_wq_obj: 390 mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); 391 392 err_destroy_queue: 393 mana_ib_destroy_queue(mdev, &qp->raw_sq); 394 395 err_free_vport: 396 mana_ib_uncfg_vport(mdev, pd, port); 397 398 return err; 399 } 400 401 static u32 mana_ib_wqe_size(u32 sge, u32 oob_size) 402 { 403 u32 wqe_size = sge * sizeof(struct gdma_sge) + sizeof(struct gdma_wqe) + oob_size; 404 405 return ALIGN(wqe_size, GDMA_WQE_BU_SIZE); 406 } 407 408 static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type) 409 { 410 u32 queue_size; 411 412 switch (attr->qp_type) { 413 case IB_QPT_UD: 414 case IB_QPT_GSI: 415 if (queue_type == MANA_UD_SEND_QUEUE) 416 queue_size = attr->cap.max_send_wr * 417 mana_ib_wqe_size(attr->cap.max_send_sge, INLINE_OOB_LARGE_SIZE); 418 else 419 queue_size = attr->cap.max_recv_wr * 420 mana_ib_wqe_size(attr->cap.max_recv_sge, INLINE_OOB_SMALL_SIZE); 421 break; 422 default: 423 return 0; 424 } 425 426 return MANA_PAGE_ALIGN(roundup_pow_of_two(queue_size)); 427 } 428 429 static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32 queue_type) 430 { 431 enum gdma_queue_type type; 432 433 switch (attr->qp_type) { 434 case IB_QPT_UD: 435 case IB_QPT_GSI: 436 if (queue_type == MANA_UD_SEND_QUEUE) 437 type = GDMA_SQ; 438 else 439 type = GDMA_RQ; 440 break; 441 default: 442 type = GDMA_INVALID_QUEUE; 443 } 444 return type; 445 } 446 447 static int mana_table_store_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 448 { 449 return xa_insert_irq(&mdev->qp_table_wq, qp->ibqp.qp_num, qp, 450 GFP_KERNEL); 451 } 452 453 static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 454 { 455 xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num); 456 } 457 458 static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 459 { 460 u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; 461 u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 462 int err; 463 464 err = xa_insert_irq(&mdev->qp_table_wq, qids, qp, GFP_KERNEL); 465 if (err) 466 return err; 467 468 err = xa_insert_irq(&mdev->qp_table_wq, qidr, qp, GFP_KERNEL); 469 if (err) 470 goto remove_sq; 471 472 return 0; 473 474 remove_sq: 475 xa_erase_irq(&mdev->qp_table_wq, qids); 476 return err; 477 } 478 479 static void mana_table_remove_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 480 { 481 u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; 482 u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 483 484 xa_erase_irq(&mdev->qp_table_wq, qids); 485 xa_erase_irq(&mdev->qp_table_wq, qidr); 486 } 487 488 static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 489 { 490 refcount_set(&qp->refcount, 1); 491 init_completion(&qp->free); 492 493 switch (qp->ibqp.qp_type) { 494 case IB_QPT_RC: 495 return mana_table_store_rc_qp(mdev, qp); 496 case IB_QPT_UD: 497 case IB_QPT_GSI: 498 return mana_table_store_ud_qp(mdev, qp); 499 default: 500 ibdev_dbg(&mdev->ib_dev, "Unknown QP type for storing in mana table, %d\n", 501 qp->ibqp.qp_type); 502 } 503 504 return -EINVAL; 505 } 506 507 static void mana_table_remove_qp(struct mana_ib_dev *mdev, 508 struct mana_ib_qp *qp) 509 { 510 switch (qp->ibqp.qp_type) { 511 case IB_QPT_RC: 512 mana_table_remove_rc_qp(mdev, qp); 513 break; 514 case IB_QPT_UD: 515 case IB_QPT_GSI: 516 mana_table_remove_ud_qp(mdev, qp); 517 break; 518 default: 519 ibdev_dbg(&mdev->ib_dev, "Unknown QP type for removing from mana table, %d\n", 520 qp->ibqp.qp_type); 521 return; 522 } 523 mana_put_qp_ref(qp); 524 wait_for_completion(&qp->free); 525 } 526 527 static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, 528 struct ib_qp_init_attr *attr, struct ib_udata *udata) 529 { 530 struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev); 531 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 532 struct mana_ib_create_rc_qp_resp resp = {}; 533 struct mana_ib_ucontext *mana_ucontext; 534 struct mana_ib_create_rc_qp ucmd = {}; 535 int i, err, j; 536 u64 flags = 0; 537 u32 doorbell; 538 539 if (!udata || udata->inlen < sizeof(ucmd)) 540 return -EINVAL; 541 542 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext); 543 doorbell = mana_ucontext->doorbell; 544 flags = MANA_RC_FLAG_NO_FMR; 545 err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)); 546 if (err) { 547 ibdev_dbg(&mdev->ib_dev, "Failed to copy from udata, %d\n", err); 548 return err; 549 } 550 551 for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { 552 /* skip FMR for user-level RC QPs */ 553 if (i == MANA_RC_SEND_QUEUE_FMR) { 554 qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; 555 qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; 556 continue; 557 } 558 err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j], 559 &qp->rc_qp.queues[i]); 560 if (err) { 561 ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err); 562 goto destroy_queues; 563 } 564 j++; 565 } 566 567 err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags); 568 if (err) { 569 ibdev_err(&mdev->ib_dev, "Failed to create rc qp %d\n", err); 570 goto destroy_queues; 571 } 572 qp->ibqp.qp_num = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER].id; 573 qp->port = attr->port_num; 574 575 if (udata) { 576 for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { 577 if (i == MANA_RC_SEND_QUEUE_FMR) 578 continue; 579 resp.queue_id[j] = qp->rc_qp.queues[i].id; 580 j++; 581 } 582 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); 583 if (err) { 584 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); 585 goto destroy_qp; 586 } 587 } 588 589 err = mana_table_store_qp(mdev, qp); 590 if (err) 591 goto destroy_qp; 592 593 return 0; 594 595 destroy_qp: 596 mana_ib_gd_destroy_rc_qp(mdev, qp); 597 destroy_queues: 598 while (i-- > 0) 599 mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]); 600 return err; 601 } 602 603 static void mana_add_qp_to_cqs(struct mana_ib_qp *qp) 604 { 605 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 606 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); 607 unsigned long flags; 608 609 spin_lock_irqsave(&send_cq->cq_lock, flags); 610 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp); 611 spin_unlock_irqrestore(&send_cq->cq_lock, flags); 612 613 spin_lock_irqsave(&recv_cq->cq_lock, flags); 614 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp); 615 spin_unlock_irqrestore(&recv_cq->cq_lock, flags); 616 } 617 618 static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp) 619 { 620 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 621 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); 622 unsigned long flags; 623 624 spin_lock_irqsave(&send_cq->cq_lock, flags); 625 list_del(&qp->cq_send_list); 626 spin_unlock_irqrestore(&send_cq->cq_lock, flags); 627 628 spin_lock_irqsave(&recv_cq->cq_lock, flags); 629 list_del(&qp->cq_recv_list); 630 spin_unlock_irqrestore(&recv_cq->cq_lock, flags); 631 } 632 633 static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, 634 struct ib_qp_init_attr *attr, struct ib_udata *udata) 635 { 636 struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev); 637 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 638 struct gdma_context *gc = mdev_to_gc(mdev); 639 u32 doorbell, queue_size; 640 int i, err; 641 642 if (udata) { 643 ibdev_dbg(&mdev->ib_dev, "User-level UD QPs are not supported\n"); 644 return -EOPNOTSUPP; 645 } 646 647 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) { 648 queue_size = mana_ib_queue_size(attr, i); 649 err = mana_ib_create_kernel_queue(mdev, queue_size, mana_ib_queue_type(attr, i), 650 &qp->ud_qp.queues[i]); 651 if (err) { 652 ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", 653 i, err); 654 goto destroy_queues; 655 } 656 } 657 doorbell = gc->mana_ib.doorbell; 658 659 err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr, 660 sizeof(struct ud_rq_shadow_wqe)); 661 if (err) { 662 ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err); 663 goto destroy_queues; 664 } 665 err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr, 666 sizeof(struct ud_sq_shadow_wqe)); 667 if (err) { 668 ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err); 669 goto destroy_shadow_queues; 670 } 671 672 err = mana_ib_gd_create_ud_qp(mdev, qp, attr, doorbell, attr->qp_type); 673 if (err) { 674 ibdev_err(&mdev->ib_dev, "Failed to create ud qp %d\n", err); 675 goto destroy_shadow_queues; 676 } 677 qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 678 qp->port = attr->port_num; 679 680 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) 681 qp->ud_qp.queues[i].kmem->id = qp->ud_qp.queues[i].id; 682 683 err = mana_table_store_qp(mdev, qp); 684 if (err) 685 goto destroy_qp; 686 687 mana_add_qp_to_cqs(qp); 688 689 return 0; 690 691 destroy_qp: 692 mana_ib_gd_destroy_ud_qp(mdev, qp); 693 destroy_shadow_queues: 694 destroy_shadow_queue(&qp->shadow_rq); 695 destroy_shadow_queue(&qp->shadow_sq); 696 destroy_queues: 697 while (i-- > 0) 698 mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]); 699 return err; 700 } 701 702 int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 703 struct ib_udata *udata) 704 { 705 switch (attr->qp_type) { 706 case IB_QPT_RAW_PACKET: 707 /* When rwq_ind_tbl is used, it's for creating WQs for RSS */ 708 if (attr->rwq_ind_tbl) 709 return mana_ib_create_qp_rss(ibqp, ibqp->pd, attr, 710 udata); 711 712 return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata); 713 case IB_QPT_RC: 714 return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata); 715 case IB_QPT_UD: 716 case IB_QPT_GSI: 717 return mana_ib_create_ud_qp(ibqp, ibqp->pd, attr, udata); 718 default: 719 ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n", 720 attr->qp_type); 721 } 722 723 return -EINVAL; 724 } 725 726 static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 727 int attr_mask, struct ib_udata *udata) 728 { 729 struct mana_ib_dev *mdev = container_of(ibqp->device, struct mana_ib_dev, ib_dev); 730 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 731 struct mana_rnic_set_qp_state_resp resp = {}; 732 struct mana_rnic_set_qp_state_req req = {}; 733 struct gdma_context *gc = mdev_to_gc(mdev); 734 struct mana_port_context *mpc; 735 struct net_device *ndev; 736 int err; 737 738 mana_gd_init_req_hdr(&req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(resp)); 739 req.hdr.dev_id = gc->mana_ib.dev_id; 740 req.adapter = mdev->adapter_handle; 741 req.qp_handle = qp->qp_handle; 742 req.qp_state = attr->qp_state; 743 req.attr_mask = attr_mask; 744 req.path_mtu = attr->path_mtu; 745 req.rq_psn = attr->rq_psn; 746 req.sq_psn = attr->sq_psn; 747 req.dest_qpn = attr->dest_qp_num; 748 req.max_dest_rd_atomic = attr->max_dest_rd_atomic; 749 req.retry_cnt = attr->retry_cnt; 750 req.rnr_retry = attr->rnr_retry; 751 req.min_rnr_timer = attr->min_rnr_timer; 752 if (attr_mask & IB_QP_AV) { 753 ndev = mana_ib_get_netdev(&mdev->ib_dev, ibqp->port); 754 if (!ndev) { 755 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n", 756 ibqp->port, ibqp->qp_num); 757 return -EINVAL; 758 } 759 mpc = netdev_priv(ndev); 760 copy_in_reverse(req.ah_attr.src_mac, mpc->mac_addr, ETH_ALEN); 761 copy_in_reverse(req.ah_attr.dest_mac, attr->ah_attr.roce.dmac, ETH_ALEN); 762 copy_in_reverse(req.ah_attr.src_addr, attr->ah_attr.grh.sgid_attr->gid.raw, 763 sizeof(union ib_gid)); 764 copy_in_reverse(req.ah_attr.dest_addr, attr->ah_attr.grh.dgid.raw, 765 sizeof(union ib_gid)); 766 if (rdma_gid_attr_network_type(attr->ah_attr.grh.sgid_attr) == RDMA_NETWORK_IPV4) { 767 req.ah_attr.src_addr_type = SGID_TYPE_IPV4; 768 req.ah_attr.dest_addr_type = SGID_TYPE_IPV4; 769 } else { 770 req.ah_attr.src_addr_type = SGID_TYPE_IPV6; 771 req.ah_attr.dest_addr_type = SGID_TYPE_IPV6; 772 } 773 req.ah_attr.dest_port = ROCE_V2_UDP_DPORT; 774 req.ah_attr.src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label, 775 ibqp->qp_num, attr->dest_qp_num); 776 req.ah_attr.traffic_class = attr->ah_attr.grh.traffic_class; 777 req.ah_attr.hop_limit = attr->ah_attr.grh.hop_limit; 778 } 779 780 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 781 if (err) { 782 ibdev_err(&mdev->ib_dev, "Failed modify qp err %d", err); 783 return err; 784 } 785 786 return 0; 787 } 788 789 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 790 int attr_mask, struct ib_udata *udata) 791 { 792 switch (ibqp->qp_type) { 793 case IB_QPT_RC: 794 case IB_QPT_UD: 795 case IB_QPT_GSI: 796 return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata); 797 default: 798 ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type); 799 return -EOPNOTSUPP; 800 } 801 } 802 803 static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, 804 struct ib_rwq_ind_table *ind_tbl, 805 struct ib_udata *udata) 806 { 807 struct mana_ib_dev *mdev = 808 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 809 struct mana_port_context *mpc; 810 struct net_device *ndev; 811 struct mana_ib_wq *wq; 812 struct ib_wq *ibwq; 813 int i; 814 815 ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); 816 mpc = netdev_priv(ndev); 817 818 for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) { 819 ibwq = ind_tbl->ind_tbl[i]; 820 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 821 ibdev_dbg(&mdev->ib_dev, "destroying wq->rx_object %llu\n", 822 wq->rx_object); 823 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 824 } 825 826 return 0; 827 } 828 829 static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata) 830 { 831 struct mana_ib_dev *mdev = 832 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 833 struct ib_pd *ibpd = qp->ibqp.pd; 834 struct mana_port_context *mpc; 835 struct net_device *ndev; 836 struct mana_ib_pd *pd; 837 838 ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); 839 mpc = netdev_priv(ndev); 840 pd = container_of(ibpd, struct mana_ib_pd, ibpd); 841 842 mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); 843 844 mana_ib_destroy_queue(mdev, &qp->raw_sq); 845 846 mana_ib_uncfg_vport(mdev, pd, qp->port); 847 848 return 0; 849 } 850 851 static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata) 852 { 853 struct mana_ib_dev *mdev = 854 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 855 int i; 856 857 mana_table_remove_qp(mdev, qp); 858 859 /* Ignore return code as there is not much we can do about it. 860 * The error message is printed inside. 861 */ 862 mana_ib_gd_destroy_rc_qp(mdev, qp); 863 for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) 864 mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]); 865 866 return 0; 867 } 868 869 static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata) 870 { 871 struct mana_ib_dev *mdev = 872 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 873 int i; 874 875 mana_remove_qp_from_cqs(qp); 876 mana_table_remove_qp(mdev, qp); 877 878 destroy_shadow_queue(&qp->shadow_rq); 879 destroy_shadow_queue(&qp->shadow_sq); 880 881 /* Ignore return code as there is not much we can do about it. 882 * The error message is printed inside. 883 */ 884 mana_ib_gd_destroy_ud_qp(mdev, qp); 885 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) 886 mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]); 887 888 return 0; 889 } 890 891 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 892 { 893 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 894 895 switch (ibqp->qp_type) { 896 case IB_QPT_RAW_PACKET: 897 if (ibqp->rwq_ind_tbl) 898 return mana_ib_destroy_qp_rss(qp, ibqp->rwq_ind_tbl, 899 udata); 900 901 return mana_ib_destroy_qp_raw(qp, udata); 902 case IB_QPT_RC: 903 return mana_ib_destroy_rc_qp(qp, udata); 904 case IB_QPT_UD: 905 case IB_QPT_GSI: 906 return mana_ib_destroy_ud_qp(qp, udata); 907 default: 908 ibdev_dbg(ibqp->device, "Unexpected QP type %u\n", 909 ibqp->qp_type); 910 } 911 912 return -ENOENT; 913 } 914