1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev, 9 struct net_device *ndev, 10 mana_handle_t default_rxobj, 11 mana_handle_t ind_table[], 12 u32 log_ind_tbl_size, u32 rx_hash_key_len, 13 u8 *rx_hash_key) 14 { 15 struct mana_port_context *mpc = netdev_priv(ndev); 16 struct mana_cfg_rx_steer_req_v2 *req; 17 struct mana_cfg_rx_steer_resp resp = {}; 18 struct gdma_context *gc; 19 u32 req_buf_size; 20 int i, err; 21 22 gc = mdev_to_gc(dev); 23 24 if (rx_hash_key_len > sizeof(req->hashkey)) 25 return -EINVAL; 26 27 req_buf_size = struct_size(req, indir_tab, MANA_INDIRECT_TABLE_DEF_SIZE); 28 req = kzalloc(req_buf_size, GFP_KERNEL); 29 if (!req) 30 return -ENOMEM; 31 32 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 33 sizeof(resp)); 34 35 req->hdr.req.msg_version = GDMA_MESSAGE_V2; 36 37 req->vport = mpc->port_handle; 38 req->rx_enable = 1; 39 req->update_default_rxobj = 1; 40 req->default_rxobj = default_rxobj; 41 req->hdr.dev_id = gc->mana.dev_id; 42 43 /* If there are more than 1 entries in indirection table, enable RSS */ 44 if (log_ind_tbl_size) 45 req->rss_enable = true; 46 47 req->num_indir_entries = MANA_INDIRECT_TABLE_DEF_SIZE; 48 req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, 49 indir_tab); 50 req->update_indir_tab = true; 51 req->cqe_coalescing_enable = 1; 52 53 /* The ind table passed to the hardware must have 54 * MANA_INDIRECT_TABLE_DEF_SIZE entries. Adjust the verb 55 * ind_table to MANA_INDIRECT_TABLE_SIZE if required 56 */ 57 ibdev_dbg(&dev->ib_dev, "ind table size %u\n", 1 << log_ind_tbl_size); 58 for (i = 0; i < MANA_INDIRECT_TABLE_DEF_SIZE; i++) { 59 req->indir_tab[i] = ind_table[i % (1 << log_ind_tbl_size)]; 60 ibdev_dbg(&dev->ib_dev, "index %u handle 0x%llx\n", i, 61 req->indir_tab[i]); 62 } 63 64 req->update_hashkey = true; 65 if (rx_hash_key_len) 66 memcpy(req->hashkey, rx_hash_key, rx_hash_key_len); 67 else 68 netdev_rss_key_fill(req->hashkey, MANA_HASH_KEY_SIZE); 69 70 ibdev_dbg(&dev->ib_dev, "vport handle %llu default_rxobj 0x%llx\n", 71 req->vport, default_rxobj); 72 73 err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &resp); 74 kfree(req); 75 return err; 76 } 77 78 static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, 79 struct ib_qp_init_attr *attr, 80 struct ib_udata *udata) 81 { 82 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 83 struct mana_ib_dev *mdev = 84 container_of(pd->device, struct mana_ib_dev, ib_dev); 85 struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl; 86 struct mana_ib_create_qp_rss_resp resp = {}; 87 struct mana_ib_create_qp_rss ucmd; 88 mana_handle_t *mana_ind_table; 89 struct mana_port_context *mpc; 90 unsigned int ind_tbl_size; 91 struct net_device *ndev; 92 struct mana_ib_cq *cq; 93 struct mana_ib_wq *wq; 94 struct mana_eq *eq; 95 struct ib_cq *ibcq; 96 struct ib_wq *ibwq; 97 int i = 0; 98 u32 port; 99 int ret; 100 101 if (!udata) 102 return -EINVAL; 103 104 ret = ib_copy_validate_udata_in(udata, ucmd, port); 105 if (ret) 106 return ret; 107 108 if (attr->cap.max_recv_wr > mdev->adapter_caps.max_qp_wr) { 109 ibdev_dbg(&mdev->ib_dev, 110 "Requested max_recv_wr %d exceeding limit\n", 111 attr->cap.max_recv_wr); 112 return -EINVAL; 113 } 114 115 if (attr->cap.max_recv_sge > MAX_RX_WQE_SGL_ENTRIES) { 116 ibdev_dbg(&mdev->ib_dev, 117 "Requested max_recv_sge %d exceeding limit\n", 118 attr->cap.max_recv_sge); 119 return -EINVAL; 120 } 121 122 ind_tbl_size = 1 << ind_tbl->log_ind_tbl_size; 123 if (ind_tbl_size > MANA_INDIRECT_TABLE_DEF_SIZE) { 124 ibdev_dbg(&mdev->ib_dev, 125 "Indirect table size %d exceeding limit\n", 126 ind_tbl_size); 127 return -EINVAL; 128 } 129 130 if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) { 131 ibdev_dbg(&mdev->ib_dev, 132 "RX Hash function is not supported, %d\n", 133 ucmd.rx_hash_function); 134 return -EINVAL; 135 } 136 137 /* IB ports start with 1, MANA start with 0 */ 138 port = ucmd.port; 139 ndev = mana_ib_get_netdev(pd->device, port); 140 if (!ndev) { 141 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n", 142 port); 143 return -EINVAL; 144 } 145 mpc = netdev_priv(ndev); 146 147 ibdev_dbg(&mdev->ib_dev, "rx_hash_function %d port %d\n", 148 ucmd.rx_hash_function, port); 149 150 mana_ind_table = kzalloc_objs(mana_handle_t, ind_tbl_size); 151 if (!mana_ind_table) { 152 ret = -ENOMEM; 153 goto fail; 154 } 155 156 qp->port = port; 157 158 for (i = 0; i < ind_tbl_size; i++) { 159 struct mana_obj_spec wq_spec = {}; 160 struct mana_obj_spec cq_spec = {}; 161 162 ibwq = ind_tbl->ind_tbl[i]; 163 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 164 165 ibcq = ibwq->cq; 166 cq = container_of(ibcq, struct mana_ib_cq, ibcq); 167 168 wq_spec.gdma_region = wq->queue.gdma_region; 169 wq_spec.queue_size = wq->wq_buf_size; 170 171 cq_spec.gdma_region = cq->queue.gdma_region; 172 cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE; 173 cq_spec.modr_ctx_id = 0; 174 eq = &mpc->ac->eqs[cq->comp_vector]; 175 cq_spec.attached_eq = eq->eq->id; 176 177 ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ, 178 &wq_spec, &cq_spec, &wq->rx_object); 179 if (ret) 180 goto fail; 181 182 /* The GDMA regions are now owned by the WQ object */ 183 wq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 184 cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 185 186 wq->queue.id = wq_spec.queue_index; 187 cq->queue.id = cq_spec.queue_index; 188 189 ibdev_dbg(&mdev->ib_dev, 190 "rx_object 0x%llx wq id %llu cq id %llu\n", 191 wq->rx_object, wq->queue.id, cq->queue.id); 192 193 resp.entries[i].cqid = cq->queue.id; 194 resp.entries[i].wqid = wq->queue.id; 195 196 mana_ind_table[i] = wq->rx_object; 197 198 /* Create CQ table entry */ 199 ret = mana_ib_install_cq_cb(mdev, cq); 200 if (ret) { 201 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 202 goto fail; 203 } 204 } 205 resp.num_entries = i; 206 207 ret = mana_ib_cfg_vport_steering(mdev, ndev, wq->rx_object, 208 mana_ind_table, 209 ind_tbl->log_ind_tbl_size, 210 ucmd.rx_hash_key_len, 211 ucmd.rx_hash_key); 212 if (ret) 213 goto fail; 214 215 ret = ib_copy_to_udata(udata, &resp, sizeof(resp)); 216 if (ret) { 217 ibdev_dbg(&mdev->ib_dev, 218 "Failed to copy to udata create rss-qp, %d\n", 219 ret); 220 goto err_disable_vport_rx; 221 } 222 223 kfree(mana_ind_table); 224 225 return 0; 226 227 err_disable_vport_rx: 228 mana_disable_vport_rx(mpc); 229 fail: 230 while (i-- > 0) { 231 ibwq = ind_tbl->ind_tbl[i]; 232 ibcq = ibwq->cq; 233 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 234 cq = container_of(ibcq, struct mana_ib_cq, ibcq); 235 236 mana_ib_remove_cq_cb(mdev, cq); 237 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 238 } 239 240 kfree(mana_ind_table); 241 242 return ret; 243 } 244 245 static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, 246 struct ib_qp_init_attr *attr, 247 struct ib_udata *udata) 248 { 249 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 250 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 251 struct mana_ib_dev *mdev = 252 container_of(ibpd->device, struct mana_ib_dev, ib_dev); 253 struct mana_ib_cq *send_cq = 254 container_of(attr->send_cq, struct mana_ib_cq, ibcq); 255 struct mana_ib_ucontext *mana_ucontext = 256 rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, 257 ibucontext); 258 struct mana_ib_create_qp_resp resp = {}; 259 struct mana_ib_create_qp ucmd = {}; 260 struct mana_obj_spec wq_spec = {}; 261 struct mana_obj_spec cq_spec = {}; 262 struct mana_port_context *mpc; 263 struct net_device *ndev; 264 struct mana_eq *eq; 265 int eq_vec; 266 u32 port; 267 int err; 268 269 if (!mana_ucontext) 270 return -EINVAL; 271 272 err = ib_copy_validate_udata_in(udata, ucmd, port); 273 if (err) 274 return err; 275 276 if (attr->cap.max_send_wr > mdev->adapter_caps.max_qp_wr) { 277 ibdev_dbg(&mdev->ib_dev, 278 "Requested max_send_wr %d exceeding limit\n", 279 attr->cap.max_send_wr); 280 return -EINVAL; 281 } 282 283 if (attr->cap.max_send_sge > MAX_TX_WQE_SGL_ENTRIES) { 284 ibdev_dbg(&mdev->ib_dev, 285 "Requested max_send_sge %d exceeding limit\n", 286 attr->cap.max_send_sge); 287 return -EINVAL; 288 } 289 290 port = ucmd.port; 291 ndev = mana_ib_get_netdev(ibpd->device, port); 292 if (!ndev) { 293 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n", 294 port); 295 return -EINVAL; 296 } 297 mpc = netdev_priv(ndev); 298 ibdev_dbg(&mdev->ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc); 299 300 err = mana_ib_cfg_vport(mdev, port, pd, mana_ucontext->doorbell); 301 if (err) 302 return -ENODEV; 303 304 qp->port = port; 305 306 ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", 307 ucmd.sq_buf_addr, ucmd.port); 308 309 err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq); 310 if (err) { 311 ibdev_dbg(&mdev->ib_dev, 312 "Failed to create queue for create qp-raw, err %d\n", err); 313 goto err_free_vport; 314 } 315 316 /* Create a WQ on the same port handle used by the Ethernet */ 317 wq_spec.gdma_region = qp->raw_sq.gdma_region; 318 wq_spec.queue_size = ucmd.sq_buf_size; 319 320 cq_spec.gdma_region = send_cq->queue.gdma_region; 321 cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE; 322 cq_spec.modr_ctx_id = 0; 323 eq_vec = send_cq->comp_vector; 324 eq = &mpc->ac->eqs[eq_vec]; 325 cq_spec.attached_eq = eq->eq->id; 326 327 err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec, 328 &cq_spec, &qp->qp_handle); 329 if (err) { 330 ibdev_dbg(&mdev->ib_dev, 331 "Failed to create wq for create raw-qp, err %d\n", 332 err); 333 goto err_destroy_queue; 334 } 335 336 /* The GDMA regions are now owned by the WQ object */ 337 qp->raw_sq.gdma_region = GDMA_INVALID_DMA_REGION; 338 send_cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 339 340 qp->raw_sq.id = wq_spec.queue_index; 341 send_cq->queue.id = cq_spec.queue_index; 342 343 /* Create CQ table entry */ 344 err = mana_ib_install_cq_cb(mdev, send_cq); 345 if (err) 346 goto err_destroy_wq_obj; 347 348 ibdev_dbg(&mdev->ib_dev, 349 "qp->qp_handle 0x%llx sq id %llu cq id %llu\n", 350 qp->qp_handle, qp->raw_sq.id, send_cq->queue.id); 351 352 resp.sqid = qp->raw_sq.id; 353 resp.cqid = send_cq->queue.id; 354 resp.tx_vp_offset = pd->tx_vp_offset; 355 356 err = ib_copy_to_udata(udata, &resp, sizeof(resp)); 357 if (err) { 358 ibdev_dbg(&mdev->ib_dev, 359 "Failed copy udata for create qp-raw, %d\n", 360 err); 361 goto err_remove_cq_cb; 362 } 363 364 return 0; 365 366 err_remove_cq_cb: 367 mana_ib_remove_cq_cb(mdev, send_cq); 368 369 err_destroy_wq_obj: 370 mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); 371 372 err_destroy_queue: 373 mana_ib_destroy_queue(mdev, &qp->raw_sq); 374 375 err_free_vport: 376 mana_ib_uncfg_vport(mdev, pd, port); 377 378 return err; 379 } 380 381 static u32 mana_ib_wqe_size(u32 sge, u32 oob_size) 382 { 383 u32 wqe_size = sge * sizeof(struct gdma_sge) + sizeof(struct gdma_wqe) + oob_size; 384 385 return ALIGN(wqe_size, GDMA_WQE_BU_SIZE); 386 } 387 388 static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type) 389 { 390 u32 queue_size; 391 392 switch (attr->qp_type) { 393 case IB_QPT_UD: 394 case IB_QPT_GSI: 395 if (queue_type == MANA_UD_SEND_QUEUE) 396 queue_size = attr->cap.max_send_wr * 397 mana_ib_wqe_size(attr->cap.max_send_sge, INLINE_OOB_LARGE_SIZE); 398 else 399 queue_size = attr->cap.max_recv_wr * 400 mana_ib_wqe_size(attr->cap.max_recv_sge, INLINE_OOB_SMALL_SIZE); 401 break; 402 default: 403 return 0; 404 } 405 406 return MANA_PAGE_ALIGN(roundup_pow_of_two(queue_size)); 407 } 408 409 static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32 queue_type) 410 { 411 enum gdma_queue_type type; 412 413 switch (attr->qp_type) { 414 case IB_QPT_UD: 415 case IB_QPT_GSI: 416 if (queue_type == MANA_UD_SEND_QUEUE) 417 type = GDMA_SQ; 418 else 419 type = GDMA_RQ; 420 break; 421 default: 422 type = GDMA_INVALID_QUEUE; 423 } 424 return type; 425 } 426 427 static int mana_table_store_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 428 { 429 return xa_insert_irq(&mdev->qp_table_wq, qp->ibqp.qp_num, qp, 430 GFP_KERNEL); 431 } 432 433 static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 434 { 435 xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num); 436 } 437 438 static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 439 { 440 u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; 441 u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 442 int err; 443 444 err = xa_insert_irq(&mdev->qp_table_wq, qids, qp, GFP_KERNEL); 445 if (err) 446 return err; 447 448 err = xa_insert_irq(&mdev->qp_table_wq, qidr, qp, GFP_KERNEL); 449 if (err) 450 goto remove_sq; 451 452 return 0; 453 454 remove_sq: 455 xa_erase_irq(&mdev->qp_table_wq, qids); 456 return err; 457 } 458 459 static void mana_table_remove_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 460 { 461 u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; 462 u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 463 464 xa_erase_irq(&mdev->qp_table_wq, qids); 465 xa_erase_irq(&mdev->qp_table_wq, qidr); 466 } 467 468 static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 469 { 470 refcount_set(&qp->refcount, 1); 471 init_completion(&qp->free); 472 473 switch (qp->ibqp.qp_type) { 474 case IB_QPT_RC: 475 return mana_table_store_rc_qp(mdev, qp); 476 case IB_QPT_UD: 477 case IB_QPT_GSI: 478 return mana_table_store_ud_qp(mdev, qp); 479 default: 480 ibdev_dbg(&mdev->ib_dev, "Unknown QP type for storing in mana table, %d\n", 481 qp->ibqp.qp_type); 482 } 483 484 return -EINVAL; 485 } 486 487 static void mana_table_remove_qp(struct mana_ib_dev *mdev, 488 struct mana_ib_qp *qp) 489 { 490 switch (qp->ibqp.qp_type) { 491 case IB_QPT_RC: 492 mana_table_remove_rc_qp(mdev, qp); 493 break; 494 case IB_QPT_UD: 495 case IB_QPT_GSI: 496 mana_table_remove_ud_qp(mdev, qp); 497 break; 498 default: 499 ibdev_dbg(&mdev->ib_dev, "Unknown QP type for removing from mana table, %d\n", 500 qp->ibqp.qp_type); 501 return; 502 } 503 mana_put_qp_ref(qp); 504 wait_for_completion(&qp->free); 505 } 506 507 static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, 508 struct ib_qp_init_attr *attr, struct ib_udata *udata) 509 { 510 struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev); 511 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 512 struct mana_ib_create_rc_qp_resp resp = {}; 513 struct mana_ib_ucontext *mana_ucontext; 514 struct mana_ib_create_rc_qp ucmd = {}; 515 int i, err, j; 516 u64 flags = 0; 517 u32 doorbell; 518 519 if (!udata) 520 return -EINVAL; 521 522 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext); 523 doorbell = mana_ucontext->doorbell; 524 flags = MANA_RC_FLAG_NO_FMR; 525 err = ib_copy_validate_udata_in(udata, ucmd, queue_size); 526 if (err) 527 return err; 528 529 for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { 530 /* skip FMR for user-level RC QPs */ 531 if (i == MANA_RC_SEND_QUEUE_FMR) { 532 qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; 533 qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; 534 continue; 535 } 536 err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j], 537 &qp->rc_qp.queues[i]); 538 if (err) { 539 ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err); 540 goto destroy_queues; 541 } 542 j++; 543 } 544 545 err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags); 546 if (err) { 547 ibdev_err(&mdev->ib_dev, "Failed to create rc qp %d\n", err); 548 goto destroy_queues; 549 } 550 qp->ibqp.qp_num = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER].id; 551 qp->port = attr->port_num; 552 553 if (udata) { 554 for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { 555 if (i == MANA_RC_SEND_QUEUE_FMR) 556 continue; 557 resp.queue_id[j] = qp->rc_qp.queues[i].id; 558 j++; 559 } 560 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen)); 561 if (err) { 562 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err); 563 goto destroy_qp; 564 } 565 } 566 567 err = mana_table_store_qp(mdev, qp); 568 if (err) 569 goto destroy_qp; 570 571 return 0; 572 573 destroy_qp: 574 mana_ib_gd_destroy_rc_qp(mdev, qp); 575 destroy_queues: 576 while (i-- > 0) 577 mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]); 578 return err; 579 } 580 581 static void mana_add_qp_to_cqs(struct mana_ib_qp *qp) 582 { 583 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 584 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); 585 unsigned long flags; 586 587 spin_lock_irqsave(&send_cq->cq_lock, flags); 588 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp); 589 spin_unlock_irqrestore(&send_cq->cq_lock, flags); 590 591 spin_lock_irqsave(&recv_cq->cq_lock, flags); 592 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp); 593 spin_unlock_irqrestore(&recv_cq->cq_lock, flags); 594 } 595 596 static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp) 597 { 598 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 599 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); 600 unsigned long flags; 601 602 spin_lock_irqsave(&send_cq->cq_lock, flags); 603 list_del(&qp->cq_send_list); 604 spin_unlock_irqrestore(&send_cq->cq_lock, flags); 605 606 spin_lock_irqsave(&recv_cq->cq_lock, flags); 607 list_del(&qp->cq_recv_list); 608 spin_unlock_irqrestore(&recv_cq->cq_lock, flags); 609 } 610 611 static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, 612 struct ib_qp_init_attr *attr, struct ib_udata *udata) 613 { 614 struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev); 615 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 616 u32 doorbell, queue_size; 617 int i, err; 618 619 if (udata) { 620 ibdev_dbg(&mdev->ib_dev, "User-level UD QPs are not supported\n"); 621 return -EOPNOTSUPP; 622 } 623 624 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) { 625 queue_size = mana_ib_queue_size(attr, i); 626 err = mana_ib_create_kernel_queue(mdev, queue_size, mana_ib_queue_type(attr, i), 627 &qp->ud_qp.queues[i]); 628 if (err) { 629 ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", 630 i, err); 631 goto destroy_queues; 632 } 633 } 634 doorbell = mdev->gdma_dev->doorbell; 635 636 err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr, 637 sizeof(struct ud_rq_shadow_wqe)); 638 if (err) { 639 ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err); 640 goto destroy_queues; 641 } 642 err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr, 643 sizeof(struct ud_sq_shadow_wqe)); 644 if (err) { 645 ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err); 646 goto destroy_shadow_queues; 647 } 648 649 err = mana_ib_gd_create_ud_qp(mdev, qp, attr, doorbell, attr->qp_type); 650 if (err) { 651 ibdev_err(&mdev->ib_dev, "Failed to create ud qp %d\n", err); 652 goto destroy_shadow_queues; 653 } 654 qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; 655 qp->port = attr->port_num; 656 657 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) 658 qp->ud_qp.queues[i].kmem->id = qp->ud_qp.queues[i].id; 659 660 err = mana_table_store_qp(mdev, qp); 661 if (err) 662 goto destroy_qp; 663 664 mana_add_qp_to_cqs(qp); 665 666 return 0; 667 668 destroy_qp: 669 mana_ib_gd_destroy_ud_qp(mdev, qp); 670 destroy_shadow_queues: 671 destroy_shadow_queue(&qp->shadow_rq); 672 destroy_shadow_queue(&qp->shadow_sq); 673 destroy_queues: 674 while (i-- > 0) 675 mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]); 676 return err; 677 } 678 679 int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 680 struct ib_udata *udata) 681 { 682 switch (attr->qp_type) { 683 case IB_QPT_RAW_PACKET: 684 /* When rwq_ind_tbl is used, it's for creating WQs for RSS */ 685 if (attr->rwq_ind_tbl) 686 return mana_ib_create_qp_rss(ibqp, ibqp->pd, attr, 687 udata); 688 689 return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata); 690 case IB_QPT_RC: 691 return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata); 692 case IB_QPT_UD: 693 case IB_QPT_GSI: 694 return mana_ib_create_ud_qp(ibqp, ibqp->pd, attr, udata); 695 default: 696 ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n", 697 attr->qp_type); 698 } 699 700 return -EINVAL; 701 } 702 703 static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 704 int attr_mask, struct ib_udata *udata) 705 { 706 struct mana_ib_dev *mdev = container_of(ibqp->device, struct mana_ib_dev, ib_dev); 707 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 708 struct mana_rnic_set_qp_state_resp resp = {}; 709 struct mana_rnic_set_qp_state_req req = {}; 710 struct gdma_context *gc = mdev_to_gc(mdev); 711 struct mana_port_context *mpc; 712 struct net_device *ndev; 713 714 mana_gd_init_req_hdr(&req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(resp)); 715 716 req.hdr.req.msg_version = GDMA_MESSAGE_V3; 717 req.hdr.dev_id = mdev->gdma_dev->dev_id; 718 req.adapter = mdev->adapter_handle; 719 req.qp_handle = qp->qp_handle; 720 req.qp_state = attr->qp_state; 721 req.attr_mask = attr_mask; 722 req.path_mtu = attr->path_mtu; 723 req.rq_psn = attr->rq_psn; 724 req.sq_psn = attr->sq_psn; 725 req.dest_qpn = attr->dest_qp_num; 726 req.max_dest_rd_atomic = attr->max_dest_rd_atomic; 727 req.retry_cnt = attr->retry_cnt; 728 req.rnr_retry = attr->rnr_retry; 729 req.min_rnr_timer = attr->min_rnr_timer; 730 req.rate_limit = attr->rate_limit; 731 req.qkey = attr->qkey; 732 req.local_ack_timeout = attr->timeout; 733 req.qp_access_flags = attr->qp_access_flags; 734 req.max_rd_atomic = attr->max_rd_atomic; 735 736 if (attr_mask & IB_QP_AV) { 737 ndev = mana_ib_get_netdev(&mdev->ib_dev, ibqp->port); 738 if (!ndev) { 739 ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n", 740 ibqp->port, ibqp->qp_num); 741 return -EINVAL; 742 } 743 mpc = netdev_priv(ndev); 744 copy_in_reverse(req.ah_attr.src_mac, mpc->mac_addr, ETH_ALEN); 745 copy_in_reverse(req.ah_attr.dest_mac, attr->ah_attr.roce.dmac, ETH_ALEN); 746 copy_in_reverse(req.ah_attr.src_addr, attr->ah_attr.grh.sgid_attr->gid.raw, 747 sizeof(union ib_gid)); 748 copy_in_reverse(req.ah_attr.dest_addr, attr->ah_attr.grh.dgid.raw, 749 sizeof(union ib_gid)); 750 if (rdma_gid_attr_network_type(attr->ah_attr.grh.sgid_attr) == RDMA_NETWORK_IPV4) { 751 req.ah_attr.src_addr_type = SGID_TYPE_IPV4; 752 req.ah_attr.dest_addr_type = SGID_TYPE_IPV4; 753 } else { 754 req.ah_attr.src_addr_type = SGID_TYPE_IPV6; 755 req.ah_attr.dest_addr_type = SGID_TYPE_IPV6; 756 } 757 req.ah_attr.dest_port = ROCE_V2_UDP_DPORT; 758 req.ah_attr.src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label, 759 ibqp->qp_num, attr->dest_qp_num); 760 req.ah_attr.traffic_class = attr->ah_attr.grh.traffic_class >> 2; 761 req.ah_attr.hop_limit = attr->ah_attr.grh.hop_limit; 762 req.ah_attr.flow_label = attr->ah_attr.grh.flow_label; 763 } 764 765 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 766 } 767 768 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 769 int attr_mask, struct ib_udata *udata) 770 { 771 switch (ibqp->qp_type) { 772 case IB_QPT_RC: 773 case IB_QPT_UD: 774 case IB_QPT_GSI: 775 return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata); 776 default: 777 ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type); 778 return -EOPNOTSUPP; 779 } 780 } 781 782 static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, 783 struct ib_rwq_ind_table *ind_tbl, 784 struct ib_udata *udata) 785 { 786 struct mana_ib_dev *mdev = 787 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 788 struct mana_port_context *mpc; 789 struct net_device *ndev; 790 struct mana_ib_wq *wq; 791 struct ib_wq *ibwq; 792 int i; 793 794 ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); 795 mpc = netdev_priv(ndev); 796 797 /* Disable vPort RX steering before destroying RX WQ objects. 798 * Otherwise firmware still routes traffic to the destroyed queues, 799 * which can cause bogus completions on reused CQ IDs when the 800 * ethernet driver later creates new queues on mana_open(). 801 * 802 * Unlike the ethernet teardown path, mana_fence_rqs() cannot be 803 * used here because the fence completion CQE is delivered on the 804 * CQ which is polled by userspace (e.g. DPDK), so there is no way 805 * for the kernel to wait for fence completion. 806 * 807 * This is best effort — if it fails there is not much we can do, 808 * and mana_cfg_vport_steering() already logs the error. 809 */ 810 mana_disable_vport_rx(mpc); 811 812 for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) { 813 ibwq = ind_tbl->ind_tbl[i]; 814 wq = container_of(ibwq, struct mana_ib_wq, ibwq); 815 ibdev_dbg(&mdev->ib_dev, "destroying wq->rx_object %llu\n", 816 wq->rx_object); 817 mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); 818 } 819 820 return 0; 821 } 822 823 static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata) 824 { 825 struct mana_ib_dev *mdev = 826 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 827 struct ib_pd *ibpd = qp->ibqp.pd; 828 struct mana_port_context *mpc; 829 struct net_device *ndev; 830 struct mana_ib_pd *pd; 831 832 ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); 833 mpc = netdev_priv(ndev); 834 pd = container_of(ibpd, struct mana_ib_pd, ibpd); 835 836 mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle); 837 838 mana_ib_destroy_queue(mdev, &qp->raw_sq); 839 840 mana_ib_uncfg_vport(mdev, pd, qp->port); 841 842 return 0; 843 } 844 845 static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata) 846 { 847 struct mana_ib_dev *mdev = 848 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 849 int i; 850 851 mana_table_remove_qp(mdev, qp); 852 853 /* Ignore return code as there is not much we can do about it. 854 * The error message is printed inside. 855 */ 856 mana_ib_gd_destroy_rc_qp(mdev, qp); 857 for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) 858 mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]); 859 860 return 0; 861 } 862 863 static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata) 864 { 865 struct mana_ib_dev *mdev = 866 container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); 867 int i; 868 869 mana_remove_qp_from_cqs(qp); 870 mana_table_remove_qp(mdev, qp); 871 872 destroy_shadow_queue(&qp->shadow_rq); 873 destroy_shadow_queue(&qp->shadow_sq); 874 875 /* Ignore return code as there is not much we can do about it. 876 * The error message is printed inside. 877 */ 878 mana_ib_gd_destroy_ud_qp(mdev, qp); 879 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) 880 mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]); 881 882 return 0; 883 } 884 885 int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 886 { 887 struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); 888 889 switch (ibqp->qp_type) { 890 case IB_QPT_RAW_PACKET: 891 if (ibqp->rwq_ind_tbl) 892 return mana_ib_destroy_qp_rss(qp, ibqp->rwq_ind_tbl, 893 udata); 894 895 return mana_ib_destroy_qp_raw(qp, udata); 896 case IB_QPT_RC: 897 return mana_ib_destroy_rc_qp(qp, udata); 898 case IB_QPT_UD: 899 case IB_QPT_GSI: 900 return mana_ib_destroy_ud_qp(qp, udata); 901 default: 902 ibdev_dbg(ibqp->device, "Unexpected QP type %u\n", 903 ibqp->qp_type); 904 } 905 906 return -ENOENT; 907 } 908