1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev, 9 struct net_device *ndev, 10 mana_handle_t default_rxobj, 11 mana_handle_t ind_table[], 12 u32 log_ind_tbl_size, u32 rx_hash_key_len, 13 u8 *rx_hash_key) 14 { 15 struct mana_port_context *mpc = netdev_priv(ndev); 16 struct mana_cfg_rx_steer_req_v2 *req; 17 struct mana_cfg_rx_steer_resp resp = {}; 18 struct gdma_context *gc; 19 u32 req_buf_size; 20 int i, err; 21 22 gc = mdev_to_gc(dev); 23 24 req_buf_size = struct_size(req, indir_tab, MANA_INDIRECT_TABLE_DEF_SIZE); 25 req = kzalloc(req_buf_size, GFP_KERNEL); 26 if (!req) 27 return -ENOMEM; 28 29 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 30 sizeof(resp)); 31 32 req->hdr.req.msg_version = GDMA_MESSAGE_V2; 33 34 req->vport = mpc->port_handle; 35 req->rx_enable = 1; 36 req->update_default_rxobj = 1; 37 req->default_rxobj = default_rxobj; 38 req->hdr.dev_id = gc->mana.dev_id; 39 40 /* If there are more than 1 entries in indirection table, enable RSS */ 41 if (log_ind_tbl_size) 42 req->rss_enable = true; 43 44 req->num_indir_entries = MANA_INDIRECT_TABLE_DEF_SIZE; 45 req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, 46 indir_tab); 47 req->update_indir_tab = true; 48 req->cqe_coalescing_enable = 1; 49 50 /* The ind table passed to the hardware must have 51 * MANA_INDIRECT_TABLE_DEF_SIZE entries. Adjust the verb 52 * ind_table to MANA_INDIRECT_TABLE_SIZE if required 53 */ 54 ibdev_dbg(&dev->ib_dev, "ind table size %u\n", 1 << log_ind_tbl_size); 55 for (i = 0; i < MANA_INDIRECT_TABLE_DEF_SIZE; i++) { 56 req->indir_tab[i] = ind_table[i % (1 << log_ind_tbl_size)]; 57 ibdev_dbg(&dev->ib_dev, "index %u handle 0x%llx\n", i, 58 req->indir_tab[i]); 59 } 60 61 req->update_hashkey = true; 62 if (rx_hash_key_len) 63 memcpy(req->hashkey, rx_hash_key, rx_hash_key_len); 64 else 65 netdev_rss_key_fill(req->hashkey, MANA_HASH_KEY_SIZE); 66 67 ibdev_dbg(&dev->ib_dev, "vport handle %llu default_rxobj 0x%llx\n", 68 req->vport, default_rxobj); 69 70 err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &resp); 71 kfree(req); 72 return err; 73 } 74 75 static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, 76 struct ib_qp_init_attr *attr, 77 struct ib_udata *udata) 78 { 79 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 80 struct mana_ib_dev *mdev = 81 container_of(pd->device, struct mana_ib_dev, ib_dev); 82 struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl; 83 struct mana_ib_create_qp_rss_resp resp = {}; 84 struct mana_ib_create_qp_rss ucmd; 85 mana_handle_t *mana_ind_table; 86 struct mana_port_context *mpc; 87 unsigned int ind_tbl_size; 88 struct net_device *ndev; 89 struct mana_ib_cq *cq; 90 struct mana_ib_wq *wq; 91 struct mana_eq *eq; 92 struct ib_cq *ibcq; 93 struct ib_wq *ibwq; 94 int i = 0; 95 u32 port; 96 int ret; 97 98 if (!udata) 99 return -EINVAL; 100 101 ret = ib_copy_validate_udata_in(udata, ucmd, port); 102 if (ret) 103 return ret; 104 105 if (attr->cap.max_recv_wr > mdev->adapter_caps.max_qp_wr) { 106 ibdev_dbg(&mdev->ib_dev, 107 "Requested max_recv_wr %d exceeding limit\n", 108 attr->cap.max_recv_wr); 109 return -EINVAL; 110 } 111 112 if (attr->cap.max_recv_sge > MAX_RX_WQE_SGL_ENTRIES) { 113 ibdev_dbg(&mdev->ib_dev, 114 "Requested max_recv_sge %d exceeding limit\n", 115 attr->cap.max_recv_sge); 116 return -EINVAL; 117 } 118 119 ind_tbl_size = 1 << ind_tbl->log_ind_tbl_size; 120 if (ind_tbl_size > MANA_INDIRECT_TABLE_DEF_SIZE) { 121 ibdev_dbg(&mdev->ib_dev, 122 "Indirect table size %d exceeding limit\n", 123 ind_tbl_size); 124 return -EINVAL; 125 } 126 127 if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) { 128 ibdev_dbg(&mdev->ib_dev, 129 "RX Hash function is not supported, %d\n", 130 ucmd.rx_hash_function); 131 return -EINVAL; 132 } 133 134 /* IB ports start with 1, MANA start with 0 */ 135 port = ucmd.port; 136 ndev = mana_ib_get_netdev(pd->device, port); 137 if (!ndev) { 138 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n", 139 port); 140 return -EINVAL; 141 } 142 mpc = netdev_priv(ndev); 143 144 ibdev_dbg(&mdev->ib_dev, "rx_hash_function %d port %d\n", 145 ucmd.rx_hash_function, port); 146 147 mana_ind_table = kzalloc_objs(mana_handle_t, ind_tbl_size); 148 if (!mana_ind_table) { 149 ret = -ENOMEM; 150 goto fail; 151 } 152 153 qp->port = port; 154 155 for (i = 0; i < ind_tbl_size; i++) { 156 struct mana_obj_spec wq_spec = {}; 157 struct mana_obj_spec cq_spec = {}; 158 159 ibwq = ind_tbl->ind_tbl[i]; 160 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 161 162 ibcq = ibwq->cq; 163 cq = container_of(ibcq, struct mana_ib_cq, ibcq); 164 165 wq_spec.gdma_region = wq->queue.gdma_region; 166 wq_spec.queue_size = wq->wq_buf_size; 167 168 cq_spec.gdma_region = cq->queue.gdma_region; 169 cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE; 170 cq_spec.modr_ctx_id = 0; 171 eq = &mpc->ac->eqs[cq->comp_vector]; 172 cq_spec.attached_eq = eq->eq->id; 173 174 ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ, 175 &wq_spec, &cq_spec, &wq->rx_object); 176 if (ret) { 177 /* Do cleanup starting with index i-1 */ 178 i--; 179 goto fail; 180 } 181 182 /* The GDMA regions are now owned by the WQ object */ 183 wq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 184 cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 185 186 wq->queue.id = wq_spec.queue_index; 187 cq->queue.id = cq_spec.queue_index; 188 189 ibdev_dbg(&mdev->ib_dev, 190 "rx_object 0x%llx wq id %llu cq id %llu\n", 191 wq->rx_object, wq->queue.id, cq->queue.id); 192 193 resp.entries[i].cqid = cq->queue.id; 194 resp.entries[i].wqid = wq->queue.id; 195 196 mana_ind_table[i] = wq->rx_object; 197 198 /* Create CQ table entry */ 199 ret = mana_ib_install_cq_cb(mdev, cq); 200 if (ret) 201 goto fail; 202 } 203 resp.num_entries = i; 204 205 ret = mana_ib_cfg_vport_steering(mdev, ndev, wq->rx_object, 206 mana_ind_table, 207 ind_tbl->log_ind_tbl_size, 208 ucmd.rx_hash_key_len, 209 ucmd.rx_hash_key); 210 if (ret) 211 goto fail; 212 213 ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); 214 if (ret) { 215 ibdev_dbg(&mdev->ib_dev, 216 "Failed to copy to udata create rss-qp, %d\n", 217 ret); 218 goto fail; 219 } 220 221 kfree(mana_ind_table); 222 223 return 0; 224 225 fail: 226 while (i-- > 0) { 227 ibwq = ind_tbl->ind_tbl[i]; 228 ibcq = ibwq->cq; 229 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 230 cq = container_of(ibcq, struct mana_ib_cq, ibcq); 231 232 mana_ib_remove_cq_cb(mdev, cq); 233 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 234 } 235 236 kfree(mana_ind_table); 237 238 return ret; 239 } 240 241 static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, 242 struct ib_qp_init_attr *attr, 243 struct ib_udata *udata) 244 { 245 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 246 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 247 struct mana_ib_dev *mdev = 248 container_of(ibpd->device, struct mana_ib_dev, ib_dev); 249 struct mana_ib_cq *send_cq = 250 container_of(attr->send_cq, struct mana_ib_cq, ibcq); 251 struct mana_ib_ucontext *mana_ucontext = 252 rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 253 ibucontext); 254 struct mana_ib_create_qp_resp resp = {}; 255 struct mana_ib_create_qp ucmd = {}; 256 struct mana_obj_spec wq_spec = {}; 257 struct mana_obj_spec cq_spec = {}; 258 struct mana_port_context *mpc; 259 struct net_device *ndev; 260 struct mana_eq *eq; 261 int eq_vec; 262 u32 port; 263 int err; 264 265 if (!mana_ucontext) 266 return -EINVAL; 267 268 err = ib_copy_validate_udata_in(udata, ucmd, port); 269 if (err) 270 return err; 271 272 if (attr->cap.max_send_wr > mdev->adapter_caps.max_qp_wr) { 273 ibdev_dbg(&mdev->ib_dev, 274 "Requested max_send_wr %d exceeding limit\n", 275 attr->cap.max_send_wr); 276 return -EINVAL; 277 } 278 279 if (attr->cap.max_send_sge > MAX_TX_WQE_SGL_ENTRIES) { 280 ibdev_dbg(&mdev->ib_dev, 281 "Requested max_send_sge %d exceeding limit\n", 282 attr->cap.max_send_sge); 283 return -EINVAL; 284 } 285 286 port = ucmd.port; 287 ndev = mana_ib_get_netdev(ibpd->device, port); 288 if (!ndev) { 289 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n", 290 port); 291 return -EINVAL; 292 } 293 mpc = netdev_priv(ndev); 294 ibdev_dbg(&mdev->ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc); 295 296 err = mana_ib_cfg_vport(mdev, port, pd, mana_ucontext->doorbell); 297 if (err) 298 return -ENODEV; 299 300 qp->port = port; 301 302 ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", 303 ucmd.sq_buf_addr, ucmd.port); 304 305 err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq); 306 if (err) { 307 ibdev_dbg(&mdev->ib_dev, 308 "Failed to create queue for create qp-raw, err %d\n", err); 309 goto err_free_vport; 310 } 311 312 /* Create a WQ on the same port handle used by the Ethernet */ 313 wq_spec.gdma_region = qp->raw_sq.gdma_region; 314 wq_spec.queue_size = ucmd.sq_buf_size; 315 316 cq_spec.gdma_region = send_cq->queue.gdma_region; 317 cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE; 318 cq_spec.modr_ctx_id = 0; 319 eq_vec = send_cq->comp_vector; 320 eq = &mpc->ac->eqs[eq_vec]; 321 cq_spec.attached_eq = eq->eq->id; 322 323 err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec, 324 &cq_spec, &qp->qp_handle); 325 if (err) { 326 ibdev_dbg(&mdev->ib_dev, 327 "Failed to create wq for create raw-qp, err %d\n", 328 err); 329 goto err_destroy_queue; 330 } 331 332 /* The GDMA regions are now owned by the WQ object */ 333 qp->raw_sq.gdma_region = GDMA_INVALID_DMA_REGION; 334 send_cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 335 336 qp->raw_sq.id = wq_spec.queue_index; 337 send_cq->queue.id = cq_spec.queue_index; 338 339 /* Create CQ table entry */ 340 err = mana_ib_install_cq_cb(mdev, send_cq); 341 if (err) 342 goto err_destroy_wq_obj; 343 344 ibdev_dbg(&mdev->ib_dev, 345 "qp->qp_handle 0x%llx sq id %llu cq id %llu\n", 346 qp->qp_handle, qp->raw_sq.id, send_cq->queue.id); 347 348 resp.sqid = qp->raw_sq.id; 349 resp.cqid = send_cq->queue.id; 350 resp.tx_vp_offset = pd->tx_vp_offset; 351 352 err = ib_copy_to_udata(udata, &resp, sizeof(resp)); 353 if (err) { 354 ibdev_dbg(&mdev->ib_dev, 355 "Failed copy udata for create qp-raw, %d\n", 356 err); 357 goto err_remove_cq_cb; 358 } 359 360 return 0; 361 362 err_remove_cq_cb: 363 mana_ib_remove_cq_cb(mdev, send_cq); 364 365 err_destroy_wq_obj: 366 mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); 367 368 err_destroy_queue: 369 mana_ib_destroy_queue(mdev, &qp->raw_sq); 370 371 err_free_vport: 372 mana_ib_uncfg_vport(mdev, pd, port); 373 374 return err; 375 } 376 377 static u32 mana_ib_wqe_size(u32 sge, u32 oob_size) 378 { 379 u32 wqe_size = sge * sizeof(struct gdma_sge) + sizeof(struct gdma_wqe) + oob_size; 380 381 return ALIGN(wqe_size, GDMA_WQE_BU_SIZE); 382 } 383 384 static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type) 385 { 386 u32 queue_size; 387 388 switch (attr->qp_type) { 389 case IB_QPT_UD: 390 case IB_QPT_GSI: 391 if (queue_type == MANA_UD_SEND_QUEUE) 392 queue_size = attr->cap.max_send_wr * 393 mana_ib_wqe_size(attr->cap.max_send_sge, INLINE_OOB_LARGE_SIZE); 394 else 395 queue_size = attr->cap.max_recv_wr * 396 mana_ib_wqe_size(attr->cap.max_recv_sge, INLINE_OOB_SMALL_SIZE); 397 break; 398 default: 399 return 0; 400 } 401 402 return MANA_PAGE_ALIGN(roundup_pow_of_two(queue_size)); 403 } 404 405 static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32 queue_type) 406 { 407 enum gdma_queue_type type; 408 409 switch (attr->qp_type) { 410 case IB_QPT_UD: 411 case IB_QPT_GSI: 412 if (queue_type == MANA_UD_SEND_QUEUE) 413 type = GDMA_SQ; 414 else 415 type = GDMA_RQ; 416 break; 417 default: 418 type = GDMA_INVALID_QUEUE; 419 } 420 return type; 421 } 422 423 static int mana_table_store_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 424 { 425 return xa_insert_irq(&mdev->qp_table_wq, qp->ibqp.qp_num, qp, 426 GFP_KERNEL); 427 } 428 429 static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 430 { 431 xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num); 432 } 433 434 static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 435 { 436 u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; 437 u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 438 int err; 439 440 err = xa_insert_irq(&mdev->qp_table_wq, qids, qp, GFP_KERNEL); 441 if (err) 442 return err; 443 444 err = xa_insert_irq(&mdev->qp_table_wq, qidr, qp, GFP_KERNEL); 445 if (err) 446 goto remove_sq; 447 448 return 0; 449 450 remove_sq: 451 xa_erase_irq(&mdev->qp_table_wq, qids); 452 return err; 453 } 454 455 static void mana_table_remove_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 456 { 457 u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; 458 u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 459 460 xa_erase_irq(&mdev->qp_table_wq, qids); 461 xa_erase_irq(&mdev->qp_table_wq, qidr); 462 } 463 464 static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 465 { 466 refcount_set(&qp->refcount, 1); 467 init_completion(&qp->free); 468 469 switch (qp->ibqp.qp_type) { 470 case IB_QPT_RC: 471 return mana_table_store_rc_qp(mdev, qp); 472 case IB_QPT_UD: 473 case IB_QPT_GSI: 474 return mana_table_store_ud_qp(mdev, qp); 475 default: 476 ibdev_dbg(&mdev->ib_dev, "Unknown QP type for storing in mana table, %d\n", 477 qp->ibqp.qp_type); 478 } 479 480 return -EINVAL; 481 } 482 483 static void mana_table_remove_qp(struct mana_ib_dev *mdev, 484 struct mana_ib_qp *qp) 485 { 486 switch (qp->ibqp.qp_type) { 487 case IB_QPT_RC: 488 mana_table_remove_rc_qp(mdev, qp); 489 break; 490 case IB_QPT_UD: 491 case IB_QPT_GSI: 492 mana_table_remove_ud_qp(mdev, qp); 493 break; 494 default: 495 ibdev_dbg(&mdev->ib_dev, "Unknown QP type for removing from mana table, %d\n", 496 qp->ibqp.qp_type); 497 return; 498 } 499 mana_put_qp_ref(qp); 500 wait_for_completion(&qp->free); 501 } 502 503 static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, 504 struct ib_qp_init_attr *attr, struct ib_udata *udata) 505 { 506 struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev); 507 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 508 struct mana_ib_create_rc_qp_resp resp = {}; 509 struct mana_ib_ucontext *mana_ucontext; 510 struct mana_ib_create_rc_qp ucmd = {}; 511 int i, err, j; 512 u64 flags = 0; 513 u32 doorbell; 514 515 if (!udata) 516 return -EINVAL; 517 518 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext); 519 doorbell = mana_ucontext->doorbell; 520 flags = MANA_RC_FLAG_NO_FMR; 521 err = ib_copy_validate_udata_in(udata, ucmd, queue_size); 522 if (err) 523 return err; 524 525 for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { 526 /* skip FMR for user-level RC QPs */ 527 if (i == MANA_RC_SEND_QUEUE_FMR) { 528 qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; 529 qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; 530 continue; 531 } 532 err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j], 533 &qp->rc_qp.queues[i]); 534 if (err) { 535 ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err); 536 goto destroy_queues; 537 } 538 j++; 539 } 540 541 err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags); 542 if (err) { 543 ibdev_err(&mdev->ib_dev, "Failed to create rc qp %d\n", err); 544 goto destroy_queues; 545 } 546 qp->ibqp.qp_num = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER].id; 547 qp->port = attr->port_num; 548 549 if (udata) { 550 for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { 551 if (i == MANA_RC_SEND_QUEUE_FMR) 552 continue; 553 resp.queue_id[j] = qp->rc_qp.queues[i].id; 554 j++; 555 } 556 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); 557 if (err) { 558 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); 559 goto destroy_qp; 560 } 561 } 562 563 err = mana_table_store_qp(mdev, qp); 564 if (err) 565 goto destroy_qp; 566 567 return 0; 568 569 destroy_qp: 570 mana_ib_gd_destroy_rc_qp(mdev, qp); 571 destroy_queues: 572 while (i-- > 0) 573 mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]); 574 return err; 575 } 576 577 static void mana_add_qp_to_cqs(struct mana_ib_qp *qp) 578 { 579 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 580 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); 581 unsigned long flags; 582 583 spin_lock_irqsave(&send_cq->cq_lock, flags); 584 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp); 585 spin_unlock_irqrestore(&send_cq->cq_lock, flags); 586 587 spin_lock_irqsave(&recv_cq->cq_lock, flags); 588 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp); 589 spin_unlock_irqrestore(&recv_cq->cq_lock, flags); 590 } 591 592 static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp) 593 { 594 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 595 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); 596 unsigned long flags; 597 598 spin_lock_irqsave(&send_cq->cq_lock, flags); 599 list_del(&qp->cq_send_list); 600 spin_unlock_irqrestore(&send_cq->cq_lock, flags); 601 602 spin_lock_irqsave(&recv_cq->cq_lock, flags); 603 list_del(&qp->cq_recv_list); 604 spin_unlock_irqrestore(&recv_cq->cq_lock, flags); 605 } 606 607 static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, 608 struct ib_qp_init_attr *attr, struct ib_udata *udata) 609 { 610 struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev); 611 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 612 u32 doorbell, queue_size; 613 int i, err; 614 615 if (udata) { 616 ibdev_dbg(&mdev->ib_dev, "User-level UD QPs are not supported\n"); 617 return -EOPNOTSUPP; 618 } 619 620 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) { 621 queue_size = mana_ib_queue_size(attr, i); 622 err = mana_ib_create_kernel_queue(mdev, queue_size, mana_ib_queue_type(attr, i), 623 &qp->ud_qp.queues[i]); 624 if (err) { 625 ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", 626 i, err); 627 goto destroy_queues; 628 } 629 } 630 doorbell = mdev->gdma_dev->doorbell; 631 632 err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr, 633 sizeof(struct ud_rq_shadow_wqe)); 634 if (err) { 635 ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err); 636 goto destroy_queues; 637 } 638 err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr, 639 sizeof(struct ud_sq_shadow_wqe)); 640 if (err) { 641 ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err); 642 goto destroy_shadow_queues; 643 } 644 645 err = mana_ib_gd_create_ud_qp(mdev, qp, attr, doorbell, attr->qp_type); 646 if (err) { 647 ibdev_err(&mdev->ib_dev, "Failed to create ud qp %d\n", err); 648 goto destroy_shadow_queues; 649 } 650 qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 651 qp->port = attr->port_num; 652 653 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) 654 qp->ud_qp.queues[i].kmem->id = qp->ud_qp.queues[i].id; 655 656 err = mana_table_store_qp(mdev, qp); 657 if (err) 658 goto destroy_qp; 659 660 mana_add_qp_to_cqs(qp); 661 662 return 0; 663 664 destroy_qp: 665 mana_ib_gd_destroy_ud_qp(mdev, qp); 666 destroy_shadow_queues: 667 destroy_shadow_queue(&qp->shadow_rq); 668 destroy_shadow_queue(&qp->shadow_sq); 669 destroy_queues: 670 while (i-- > 0) 671 mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]); 672 return err; 673 } 674 675 int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 676 struct ib_udata *udata) 677 { 678 switch (attr->qp_type) { 679 case IB_QPT_RAW_PACKET: 680 /* When rwq_ind_tbl is used, it's for creating WQs for RSS */ 681 if (attr->rwq_ind_tbl) 682 return mana_ib_create_qp_rss(ibqp, ibqp->pd, attr, 683 udata); 684 685 return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata); 686 case IB_QPT_RC: 687 return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata); 688 case IB_QPT_UD: 689 case IB_QPT_GSI: 690 return mana_ib_create_ud_qp(ibqp, ibqp->pd, attr, udata); 691 default: 692 ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n", 693 attr->qp_type); 694 } 695 696 return -EINVAL; 697 } 698 699 static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 700 int attr_mask, struct ib_udata *udata) 701 { 702 struct mana_ib_dev *mdev = container_of(ibqp->device, struct mana_ib_dev, ib_dev); 703 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 704 struct mana_rnic_set_qp_state_resp resp = {}; 705 struct mana_rnic_set_qp_state_req req = {}; 706 struct gdma_context *gc = mdev_to_gc(mdev); 707 struct mana_port_context *mpc; 708 struct net_device *ndev; 709 710 mana_gd_init_req_hdr(&req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(resp)); 711 712 req.hdr.req.msg_version = GDMA_MESSAGE_V3; 713 req.hdr.dev_id = mdev->gdma_dev->dev_id; 714 req.adapter = mdev->adapter_handle; 715 req.qp_handle = qp->qp_handle; 716 req.qp_state = attr->qp_state; 717 req.attr_mask = attr_mask; 718 req.path_mtu = attr->path_mtu; 719 req.rq_psn = attr->rq_psn; 720 req.sq_psn = attr->sq_psn; 721 req.dest_qpn = attr->dest_qp_num; 722 req.max_dest_rd_atomic = attr->max_dest_rd_atomic; 723 req.retry_cnt = attr->retry_cnt; 724 req.rnr_retry = attr->rnr_retry; 725 req.min_rnr_timer = attr->min_rnr_timer; 726 req.rate_limit = attr->rate_limit; 727 req.qkey = attr->qkey; 728 req.local_ack_timeout = attr->timeout; 729 req.qp_access_flags = attr->qp_access_flags; 730 req.max_rd_atomic = attr->max_rd_atomic; 731 732 if (attr_mask & IB_QP_AV) { 733 ndev = mana_ib_get_netdev(&mdev->ib_dev, ibqp->port); 734 if (!ndev) { 735 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n", 736 ibqp->port, ibqp->qp_num); 737 return -EINVAL; 738 } 739 mpc = netdev_priv(ndev); 740 copy_in_reverse(req.ah_attr.src_mac, mpc->mac_addr, ETH_ALEN); 741 copy_in_reverse(req.ah_attr.dest_mac, attr->ah_attr.roce.dmac, ETH_ALEN); 742 copy_in_reverse(req.ah_attr.src_addr, attr->ah_attr.grh.sgid_attr->gid.raw, 743 sizeof(union ib_gid)); 744 copy_in_reverse(req.ah_attr.dest_addr, attr->ah_attr.grh.dgid.raw, 745 sizeof(union ib_gid)); 746 if (rdma_gid_attr_network_type(attr->ah_attr.grh.sgid_attr) == RDMA_NETWORK_IPV4) { 747 req.ah_attr.src_addr_type = SGID_TYPE_IPV4; 748 req.ah_attr.dest_addr_type = SGID_TYPE_IPV4; 749 } else { 750 req.ah_attr.src_addr_type = SGID_TYPE_IPV6; 751 req.ah_attr.dest_addr_type = SGID_TYPE_IPV6; 752 } 753 req.ah_attr.dest_port = ROCE_V2_UDP_DPORT; 754 req.ah_attr.src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label, 755 ibqp->qp_num, attr->dest_qp_num); 756 req.ah_attr.traffic_class = attr->ah_attr.grh.traffic_class >> 2; 757 req.ah_attr.hop_limit = attr->ah_attr.grh.hop_limit; 758 req.ah_attr.flow_label = attr->ah_attr.grh.flow_label; 759 } 760 761 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 762 } 763 764 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 765 int attr_mask, struct ib_udata *udata) 766 { 767 switch (ibqp->qp_type) { 768 case IB_QPT_RC: 769 case IB_QPT_UD: 770 case IB_QPT_GSI: 771 return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata); 772 default: 773 ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type); 774 return -EOPNOTSUPP; 775 } 776 } 777 778 static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, 779 struct ib_rwq_ind_table *ind_tbl, 780 struct ib_udata *udata) 781 { 782 struct mana_ib_dev *mdev = 783 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 784 struct mana_port_context *mpc; 785 struct net_device *ndev; 786 struct mana_ib_wq *wq; 787 struct ib_wq *ibwq; 788 int i; 789 790 ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); 791 mpc = netdev_priv(ndev); 792 793 /* Disable vPort RX steering before destroying RX WQ objects. 794 * Otherwise firmware still routes traffic to the destroyed queues, 795 * which can cause bogus completions on reused CQ IDs when the 796 * ethernet driver later creates new queues on mana_open(). 797 * 798 * Unlike the ethernet teardown path, mana_fence_rqs() cannot be 799 * used here because the fence completion CQE is delivered on the 800 * CQ which is polled by userspace (e.g. DPDK), so there is no way 801 * for the kernel to wait for fence completion. 802 * 803 * This is best effort — if it fails there is not much we can do, 804 * and mana_cfg_vport_steering() already logs the error. 805 */ 806 mana_disable_vport_rx(mpc); 807 808 for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) { 809 ibwq = ind_tbl->ind_tbl[i]; 810 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 811 ibdev_dbg(&mdev->ib_dev, "destroying wq->rx_object %llu\n", 812 wq->rx_object); 813 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 814 } 815 816 return 0; 817 } 818 819 static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata) 820 { 821 struct mana_ib_dev *mdev = 822 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 823 struct ib_pd *ibpd = qp->ibqp.pd; 824 struct mana_port_context *mpc; 825 struct net_device *ndev; 826 struct mana_ib_pd *pd; 827 828 ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); 829 mpc = netdev_priv(ndev); 830 pd = container_of(ibpd, struct mana_ib_pd, ibpd); 831 832 mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); 833 834 mana_ib_destroy_queue(mdev, &qp->raw_sq); 835 836 mana_ib_uncfg_vport(mdev, pd, qp->port); 837 838 return 0; 839 } 840 841 static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata) 842 { 843 struct mana_ib_dev *mdev = 844 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 845 int i; 846 847 mana_table_remove_qp(mdev, qp); 848 849 /* Ignore return code as there is not much we can do about it. 850 * The error message is printed inside. 851 */ 852 mana_ib_gd_destroy_rc_qp(mdev, qp); 853 for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) 854 mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]); 855 856 return 0; 857 } 858 859 static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata) 860 { 861 struct mana_ib_dev *mdev = 862 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 863 int i; 864 865 mana_remove_qp_from_cqs(qp); 866 mana_table_remove_qp(mdev, qp); 867 868 destroy_shadow_queue(&qp->shadow_rq); 869 destroy_shadow_queue(&qp->shadow_sq); 870 871 /* Ignore return code as there is not much we can do about it. 872 * The error message is printed inside. 873 */ 874 mana_ib_gd_destroy_ud_qp(mdev, qp); 875 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) 876 mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]); 877 878 return 0; 879 } 880 881 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 882 { 883 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 884 885 switch (ibqp->qp_type) { 886 case IB_QPT_RAW_PACKET: 887 if (ibqp->rwq_ind_tbl) 888 return mana_ib_destroy_qp_rss(qp, ibqp->rwq_ind_tbl, 889 udata); 890 891 return mana_ib_destroy_qp_raw(qp, udata); 892 case IB_QPT_RC: 893 return mana_ib_destroy_rc_qp(qp, udata); 894 case IB_QPT_UD: 895 case IB_QPT_GSI: 896 return mana_ib_destroy_ud_qp(qp, udata); 897 default: 898 ibdev_dbg(ibqp->device, "Unexpected QP type %u\n", 899 ibqp->qp_type); 900 } 901 902 return -ENOENT; 903 } 904