1 /* 2 * Copyright (c) 2012 Mellanox Technologies, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <config.h> 34 35 #include <stdlib.h> 36 #include <stdio.h> 37 #include <string.h> 38 #include <pthread.h> 39 #include <errno.h> 40 #include <limits.h> 41 #include <sys/types.h> 42 #include <sys/stat.h> 43 #include <fcntl.h> 44 #include <unistd.h> 45 #include <sys/mman.h> 46 47 #include "mlx5.h" 48 #include "mlx5-abi.h" 49 #include "wqe.h" 50 51 int mlx5_single_threaded = 0; 52 53 static inline int is_xrc_tgt(int type) 54 { 55 return type == IBV_QPT_XRC_RECV; 56 } 57 58 int mlx5_query_device(struct ibv_context *context, struct ibv_device_attr *attr) 59 { 60 struct ibv_query_device cmd; 61 uint64_t raw_fw_ver; 62 unsigned major, minor, sub_minor; 63 int ret; 64 65 ret = ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, sizeof cmd); 66 if (ret) 67 return ret; 68 69 major = (raw_fw_ver >> 32) & 0xffff; 70 minor = (raw_fw_ver >> 16) & 0xffff; 71 sub_minor = raw_fw_ver & 0xffff; 72 73 snprintf(attr->fw_ver, sizeof attr->fw_ver, 74 "%d.%d.%04d", major, minor, sub_minor); 75 76 return 0; 77 } 78 79 #define READL(ptr) (*((uint32_t *)(ptr))) 80 static int mlx5_read_clock(struct ibv_context *context, uint64_t *cycles) 81 { 82 unsigned int clockhi, clocklo, clockhi1; 83 int i; 84 struct mlx5_context *ctx = to_mctx(context); 85 86 if (!ctx->hca_core_clock) 87 return -EOPNOTSUPP; 88 89 /* Handle wraparound */ 90 for (i = 0; i < 2; i++) { 91 clockhi = be32toh(READL(ctx->hca_core_clock)); 92 clocklo = be32toh(READL(ctx->hca_core_clock + 4)); 93 clockhi1 = be32toh(READL(ctx->hca_core_clock)); 94 if (clockhi == clockhi1) 95 break; 96 } 97 98 *cycles = (uint64_t)clockhi << 32 | (uint64_t)clocklo; 99 100 return 0; 101 } 102 103 int mlx5_query_rt_values(struct ibv_context *context, 104 struct ibv_values_ex *values) 105 { 106 uint32_t comp_mask = 0; 107 int err = 0; 108 109 if (values->comp_mask & IBV_VALUES_MASK_RAW_CLOCK) { 110 uint64_t cycles; 111 112 err = mlx5_read_clock(context, &cycles); 113 if (!err) { 114 values->raw_clock.tv_sec = 0; 115 values->raw_clock.tv_nsec = cycles; 116 comp_mask |= IBV_VALUES_MASK_RAW_CLOCK; 117 } 118 } 119 120 values->comp_mask = comp_mask; 121 122 return err; 123 } 124 125 int mlx5_query_port(struct ibv_context *context, uint8_t port, 126 struct ibv_port_attr *attr) 127 { 128 struct ibv_query_port cmd; 129 130 return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd); 131 } 132 133 struct ibv_pd *mlx5_alloc_pd(struct ibv_context *context) 134 { 135 struct ibv_alloc_pd cmd; 136 struct mlx5_alloc_pd_resp resp; 137 struct mlx5_pd *pd; 138 139 pd = calloc(1, sizeof *pd); 140 if (!pd) 141 return NULL; 142 143 if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd, 144 &resp.ibv_resp, sizeof resp)) { 145 free(pd); 146 return NULL; 147 } 148 149 pd->pdn = resp.pdn; 150 151 return &pd->ibv_pd; 152 } 153 154 int mlx5_free_pd(struct ibv_pd *pd) 155 { 156 int ret; 157 158 ret = ibv_cmd_dealloc_pd(pd); 159 if (ret) 160 return ret; 161 162 free(to_mpd(pd)); 163 return 0; 164 } 165 166 struct ibv_mr *mlx5_reg_mr(struct ibv_pd *pd, void *addr, size_t length, 167 int acc) 168 { 169 struct mlx5_mr *mr; 170 struct ibv_reg_mr cmd; 171 int ret; 172 enum ibv_access_flags access = (enum ibv_access_flags)acc; 173 struct ibv_reg_mr_resp resp; 174 175 mr = calloc(1, sizeof(*mr)); 176 if (!mr) 177 return NULL; 178 179 ret = ibv_cmd_reg_mr(pd, addr, length, (uintptr_t)addr, access, 180 &(mr->ibv_mr), &cmd, sizeof(cmd), &resp, 181 sizeof resp); 182 if (ret) { 183 mlx5_free_buf(&(mr->buf)); 184 free(mr); 185 return NULL; 186 } 187 mr->alloc_flags = acc; 188 189 return &mr->ibv_mr; 190 } 191 192 int mlx5_rereg_mr(struct ibv_mr *ibmr, int flags, struct ibv_pd *pd, void *addr, 193 size_t length, int access) 194 { 195 struct ibv_rereg_mr cmd; 196 struct ibv_rereg_mr_resp resp; 197 198 if (flags & IBV_REREG_MR_KEEP_VALID) 199 return ENOTSUP; 200 201 return ibv_cmd_rereg_mr(ibmr, flags, addr, length, (uintptr_t)addr, 202 access, pd, &cmd, sizeof(cmd), &resp, 203 sizeof(resp)); 204 } 205 206 int mlx5_dereg_mr(struct ibv_mr *ibmr) 207 { 208 int ret; 209 struct mlx5_mr *mr = to_mmr(ibmr); 210 211 ret = ibv_cmd_dereg_mr(ibmr); 212 if (ret) 213 return ret; 214 215 free(mr); 216 return 0; 217 } 218 219 struct ibv_mw *mlx5_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type) 220 { 221 struct ibv_mw *mw; 222 struct ibv_alloc_mw cmd; 223 struct ibv_alloc_mw_resp resp; 224 int ret; 225 226 mw = malloc(sizeof(*mw)); 227 if (!mw) 228 return NULL; 229 230 memset(mw, 0, sizeof(*mw)); 231 232 ret = ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, 233 sizeof(resp)); 234 if (ret) { 235 free(mw); 236 return NULL; 237 } 238 239 return mw; 240 } 241 242 int mlx5_dealloc_mw(struct ibv_mw *mw) 243 { 244 int ret; 245 struct ibv_dealloc_mw cmd; 246 247 ret = ibv_cmd_dealloc_mw(mw, &cmd, sizeof(cmd)); 248 if (ret) 249 return ret; 250 251 free(mw); 252 return 0; 253 } 254 255 int mlx5_round_up_power_of_two(long long sz) 256 { 257 long long ret; 258 259 for (ret = 1; ret < sz; ret <<= 1) 260 ; /* nothing */ 261 262 if (ret > INT_MAX) { 263 fprintf(stderr, "%s: roundup overflow\n", __func__); 264 return -ENOMEM; 265 } 266 267 return (int)ret; 268 } 269 270 static int align_queue_size(long long req) 271 { 272 return mlx5_round_up_power_of_two(req); 273 } 274 275 static int get_cqe_size(void) 276 { 277 char *env; 278 int size = 64; 279 280 env = getenv("MLX5_CQE_SIZE"); 281 if (env) 282 size = atoi(env); 283 284 switch (size) { 285 case 64: 286 case 128: 287 return size; 288 289 default: 290 return -EINVAL; 291 } 292 } 293 294 static int use_scatter_to_cqe(void) 295 { 296 char *env; 297 298 env = getenv("MLX5_SCATTER_TO_CQE"); 299 if (env && !strcmp(env, "0")) 300 return 0; 301 302 return 1; 303 } 304 305 static int srq_sig_enabled(void) 306 { 307 char *env; 308 309 env = getenv("MLX5_SRQ_SIGNATURE"); 310 if (env) 311 return 1; 312 313 return 0; 314 } 315 316 static int qp_sig_enabled(void) 317 { 318 char *env; 319 320 env = getenv("MLX5_QP_SIGNATURE"); 321 if (env) 322 return 1; 323 324 return 0; 325 } 326 327 enum { 328 CREATE_CQ_SUPPORTED_WC_FLAGS = IBV_WC_STANDARD_FLAGS | 329 IBV_WC_EX_WITH_COMPLETION_TIMESTAMP | 330 IBV_WC_EX_WITH_CVLAN | 331 IBV_WC_EX_WITH_FLOW_TAG 332 }; 333 334 enum { 335 CREATE_CQ_SUPPORTED_COMP_MASK = IBV_CQ_INIT_ATTR_MASK_FLAGS 336 }; 337 338 enum { 339 CREATE_CQ_SUPPORTED_FLAGS = IBV_CREATE_CQ_ATTR_SINGLE_THREADED 340 }; 341 342 static struct ibv_cq_ex *create_cq(struct ibv_context *context, 343 const struct ibv_cq_init_attr_ex *cq_attr, 344 int cq_alloc_flags, 345 struct mlx5dv_cq_init_attr *mlx5cq_attr) 346 { 347 struct mlx5_create_cq cmd; 348 struct mlx5_create_cq_resp resp; 349 struct mlx5_cq *cq; 350 int cqe_sz; 351 int ret; 352 int ncqe; 353 struct mlx5_context *mctx = to_mctx(context); 354 FILE *fp = to_mctx(context)->dbg_fp; 355 356 if (!cq_attr->cqe) { 357 mlx5_dbg(fp, MLX5_DBG_CQ, "CQE invalid\n"); 358 errno = EINVAL; 359 return NULL; 360 } 361 362 if (cq_attr->comp_mask & ~CREATE_CQ_SUPPORTED_COMP_MASK) { 363 mlx5_dbg(fp, MLX5_DBG_CQ, 364 "Unsupported comp_mask for create_cq\n"); 365 errno = EINVAL; 366 return NULL; 367 } 368 369 if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && 370 cq_attr->flags & ~CREATE_CQ_SUPPORTED_FLAGS) { 371 mlx5_dbg(fp, MLX5_DBG_CQ, 372 "Unsupported creation flags requested for create_cq\n"); 373 errno = EINVAL; 374 return NULL; 375 } 376 377 if (cq_attr->wc_flags & ~CREATE_CQ_SUPPORTED_WC_FLAGS) { 378 mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 379 errno = ENOTSUP; 380 return NULL; 381 } 382 383 cq = calloc(1, sizeof *cq); 384 if (!cq) { 385 mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 386 return NULL; 387 } 388 389 memset(&cmd, 0, sizeof cmd); 390 cq->cons_index = 0; 391 392 if (mlx5_spinlock_init(&cq->lock)) 393 goto err; 394 395 ncqe = align_queue_size(cq_attr->cqe + 1); 396 if ((ncqe > (1 << 24)) || (ncqe < (cq_attr->cqe + 1))) { 397 mlx5_dbg(fp, MLX5_DBG_CQ, "ncqe %d\n", ncqe); 398 errno = EINVAL; 399 goto err_spl; 400 } 401 402 cqe_sz = get_cqe_size(); 403 if (cqe_sz < 0) { 404 mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 405 errno = -cqe_sz; 406 goto err_spl; 407 } 408 409 if (mlx5_alloc_cq_buf(to_mctx(context), cq, &cq->buf_a, ncqe, cqe_sz)) { 410 mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 411 goto err_spl; 412 } 413 414 cq->dbrec = mlx5_alloc_dbrec(to_mctx(context)); 415 if (!cq->dbrec) { 416 mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 417 goto err_buf; 418 } 419 420 cq->dbrec[MLX5_CQ_SET_CI] = 0; 421 cq->dbrec[MLX5_CQ_ARM_DB] = 0; 422 cq->arm_sn = 0; 423 cq->cqe_sz = cqe_sz; 424 cq->flags = cq_alloc_flags; 425 426 if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && 427 cq_attr->flags & IBV_CREATE_CQ_ATTR_SINGLE_THREADED) 428 cq->flags |= MLX5_CQ_FLAGS_SINGLE_THREADED; 429 cmd.buf_addr = (uintptr_t) cq->buf_a.buf; 430 cmd.db_addr = (uintptr_t) cq->dbrec; 431 cmd.cqe_size = cqe_sz; 432 433 if (mlx5cq_attr) { 434 if (!check_comp_mask(mlx5cq_attr->comp_mask, 435 MLX5DV_CQ_INIT_ATTR_MASK_RESERVED - 1)) { 436 mlx5_dbg(fp, MLX5_DBG_CQ, 437 "Unsupported vendor comp_mask for create_cq\n"); 438 errno = EINVAL; 439 goto err_db; 440 } 441 442 if (mlx5cq_attr->comp_mask & MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE) { 443 if (mctx->cqe_comp_caps.max_num && 444 (mlx5cq_attr->cqe_comp_res_format & 445 mctx->cqe_comp_caps.supported_format)) { 446 cmd.cqe_comp_en = 1; 447 cmd.cqe_comp_res_format = mlx5cq_attr->cqe_comp_res_format; 448 } else { 449 mlx5_dbg(fp, MLX5_DBG_CQ, "CQE Compression is not supported\n"); 450 errno = EINVAL; 451 goto err_db; 452 } 453 } 454 } 455 456 ret = ibv_cmd_create_cq(context, ncqe - 1, cq_attr->channel, 457 cq_attr->comp_vector, 458 ibv_cq_ex_to_cq(&cq->ibv_cq), &cmd.ibv_cmd, 459 sizeof(cmd), &resp.ibv_resp, sizeof(resp)); 460 if (ret) { 461 mlx5_dbg(fp, MLX5_DBG_CQ, "ret %d\n", ret); 462 goto err_db; 463 } 464 465 cq->active_buf = &cq->buf_a; 466 cq->resize_buf = NULL; 467 cq->cqn = resp.cqn; 468 cq->stall_enable = to_mctx(context)->stall_enable; 469 cq->stall_adaptive_enable = to_mctx(context)->stall_adaptive_enable; 470 cq->stall_cycles = to_mctx(context)->stall_cycles; 471 472 if (cq_alloc_flags & MLX5_CQ_FLAGS_EXTENDED) 473 mlx5_cq_fill_pfns(cq, cq_attr); 474 475 return &cq->ibv_cq; 476 477 err_db: 478 mlx5_free_db(to_mctx(context), cq->dbrec); 479 480 err_buf: 481 mlx5_free_cq_buf(to_mctx(context), &cq->buf_a); 482 483 err_spl: 484 mlx5_spinlock_destroy(&cq->lock); 485 486 err: 487 free(cq); 488 489 return NULL; 490 } 491 492 struct ibv_cq *mlx5_create_cq(struct ibv_context *context, int cqe, 493 struct ibv_comp_channel *channel, 494 int comp_vector) 495 { 496 struct ibv_cq_ex *cq; 497 struct ibv_cq_init_attr_ex cq_attr = {.cqe = cqe, .channel = channel, 498 .comp_vector = comp_vector, 499 .wc_flags = IBV_WC_STANDARD_FLAGS}; 500 501 if (cqe <= 0) { 502 errno = EINVAL; 503 return NULL; 504 } 505 506 cq = create_cq(context, &cq_attr, 0, NULL); 507 return cq ? ibv_cq_ex_to_cq(cq) : NULL; 508 } 509 510 struct ibv_cq_ex *mlx5_create_cq_ex(struct ibv_context *context, 511 struct ibv_cq_init_attr_ex *cq_attr) 512 { 513 return create_cq(context, cq_attr, MLX5_CQ_FLAGS_EXTENDED, NULL); 514 } 515 516 struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context, 517 struct ibv_cq_init_attr_ex *cq_attr, 518 struct mlx5dv_cq_init_attr *mlx5_cq_attr) 519 { 520 struct ibv_cq_ex *cq; 521 int err = 0; 522 523 cq = create_cq(context, cq_attr, MLX5_CQ_FLAGS_EXTENDED, mlx5_cq_attr); 524 if (!cq) 525 return NULL; 526 527 err = verbs_init_cq(ibv_cq_ex_to_cq(cq), context, 528 cq_attr->channel, cq_attr->cq_context); 529 if (err) 530 goto err; 531 532 return cq; 533 534 err: 535 context->ops.destroy_cq(ibv_cq_ex_to_cq(cq)); 536 537 return NULL; 538 } 539 540 int mlx5_resize_cq(struct ibv_cq *ibcq, int cqe) 541 { 542 struct mlx5_cq *cq = to_mcq(ibcq); 543 struct mlx5_resize_cq_resp resp; 544 struct mlx5_resize_cq cmd; 545 struct mlx5_context *mctx = to_mctx(ibcq->context); 546 int err; 547 548 if (cqe < 0) { 549 errno = EINVAL; 550 return errno; 551 } 552 553 memset(&cmd, 0, sizeof(cmd)); 554 memset(&resp, 0, sizeof(resp)); 555 556 if (((long long)cqe * 64) > INT_MAX) 557 return EINVAL; 558 559 mlx5_spin_lock(&cq->lock); 560 cq->active_cqes = cq->ibv_cq.cqe; 561 if (cq->active_buf == &cq->buf_a) 562 cq->resize_buf = &cq->buf_b; 563 else 564 cq->resize_buf = &cq->buf_a; 565 566 cqe = align_queue_size(cqe + 1); 567 if (cqe == ibcq->cqe + 1) { 568 cq->resize_buf = NULL; 569 err = 0; 570 goto out; 571 } 572 573 /* currently we don't change cqe size */ 574 cq->resize_cqe_sz = cq->cqe_sz; 575 cq->resize_cqes = cqe; 576 err = mlx5_alloc_cq_buf(mctx, cq, cq->resize_buf, cq->resize_cqes, cq->resize_cqe_sz); 577 if (err) { 578 cq->resize_buf = NULL; 579 errno = ENOMEM; 580 goto out; 581 } 582 583 cmd.buf_addr = (uintptr_t)cq->resize_buf->buf; 584 cmd.cqe_size = cq->resize_cqe_sz; 585 586 err = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof(cmd), 587 &resp.ibv_resp, sizeof(resp)); 588 if (err) 589 goto out_buf; 590 591 mlx5_cq_resize_copy_cqes(cq); 592 mlx5_free_cq_buf(mctx, cq->active_buf); 593 cq->active_buf = cq->resize_buf; 594 cq->ibv_cq.cqe = cqe - 1; 595 mlx5_spin_unlock(&cq->lock); 596 cq->resize_buf = NULL; 597 return 0; 598 599 out_buf: 600 mlx5_free_cq_buf(mctx, cq->resize_buf); 601 cq->resize_buf = NULL; 602 603 out: 604 mlx5_spin_unlock(&cq->lock); 605 return err; 606 } 607 608 int mlx5_destroy_cq(struct ibv_cq *cq) 609 { 610 int ret; 611 struct mlx5_cq *mcq = to_mcq(cq); 612 613 ret = ibv_cmd_destroy_cq(cq); 614 if (ret) 615 return ret; 616 617 verbs_cleanup_cq(cq); 618 mlx5_free_db(to_mctx(cq->context), to_mcq(cq)->dbrec); 619 mlx5_free_cq_buf(to_mctx(cq->context), to_mcq(cq)->active_buf); 620 mlx5_spinlock_destroy(&mcq->lock); 621 free(to_mcq(cq)); 622 623 return 0; 624 } 625 626 struct ibv_srq *mlx5_create_srq(struct ibv_pd *pd, 627 struct ibv_srq_init_attr *attr) 628 { 629 struct mlx5_create_srq cmd; 630 struct mlx5_create_srq_resp resp; 631 struct mlx5_srq *srq; 632 int ret; 633 struct mlx5_context *ctx; 634 int max_sge; 635 struct ibv_srq *ibsrq; 636 637 ctx = to_mctx(pd->context); 638 srq = calloc(1, sizeof *srq); 639 if (!srq) { 640 fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 641 return NULL; 642 } 643 ibsrq = &srq->vsrq.srq; 644 645 memset(&cmd, 0, sizeof cmd); 646 if (mlx5_spinlock_init(&srq->lock)) { 647 fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 648 goto err; 649 } 650 651 if (attr->attr.max_wr > ctx->max_srq_recv_wr) { 652 fprintf(stderr, "%s-%d:max_wr %d, max_srq_recv_wr %d\n", __func__, __LINE__, 653 attr->attr.max_wr, ctx->max_srq_recv_wr); 654 errno = EINVAL; 655 goto err_spl; 656 } 657 658 /* 659 * this calculation does not consider required control segments. The 660 * final calculation is done again later. This is done so to avoid 661 * overflows of variables 662 */ 663 max_sge = ctx->max_rq_desc_sz / sizeof(struct mlx5_wqe_data_seg); 664 if (attr->attr.max_sge > max_sge) { 665 fprintf(stderr, "%s-%d:max_wr %d, max_srq_recv_wr %d\n", __func__, __LINE__, 666 attr->attr.max_wr, ctx->max_srq_recv_wr); 667 errno = EINVAL; 668 goto err_spl; 669 } 670 671 srq->max = align_queue_size(attr->attr.max_wr + 1); 672 srq->max_gs = attr->attr.max_sge; 673 srq->counter = 0; 674 675 if (mlx5_alloc_srq_buf(pd->context, srq)) { 676 fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 677 goto err_spl; 678 } 679 680 srq->db = mlx5_alloc_dbrec(to_mctx(pd->context)); 681 if (!srq->db) { 682 fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 683 goto err_free; 684 } 685 686 *srq->db = 0; 687 688 cmd.buf_addr = (uintptr_t) srq->buf.buf; 689 cmd.db_addr = (uintptr_t) srq->db; 690 srq->wq_sig = srq_sig_enabled(); 691 if (srq->wq_sig) 692 cmd.flags = MLX5_SRQ_FLAG_SIGNATURE; 693 694 attr->attr.max_sge = srq->max_gs; 695 pthread_mutex_lock(&ctx->srq_table_mutex); 696 ret = ibv_cmd_create_srq(pd, ibsrq, attr, &cmd.ibv_cmd, sizeof(cmd), 697 &resp.ibv_resp, sizeof(resp)); 698 if (ret) 699 goto err_db; 700 701 ret = mlx5_store_srq(ctx, resp.srqn, srq); 702 if (ret) 703 goto err_destroy; 704 705 pthread_mutex_unlock(&ctx->srq_table_mutex); 706 707 srq->srqn = resp.srqn; 708 srq->rsc.rsn = resp.srqn; 709 srq->rsc.type = MLX5_RSC_TYPE_SRQ; 710 711 return ibsrq; 712 713 err_destroy: 714 ibv_cmd_destroy_srq(ibsrq); 715 716 err_db: 717 pthread_mutex_unlock(&ctx->srq_table_mutex); 718 mlx5_free_db(to_mctx(pd->context), srq->db); 719 720 err_free: 721 free(srq->wrid); 722 mlx5_free_buf(&srq->buf); 723 724 err_spl: 725 mlx5_spinlock_destroy(&srq->lock); 726 727 err: 728 free(srq); 729 730 return NULL; 731 } 732 733 int mlx5_modify_srq(struct ibv_srq *srq, 734 struct ibv_srq_attr *attr, 735 int attr_mask) 736 { 737 struct ibv_modify_srq cmd; 738 739 return ibv_cmd_modify_srq(srq, attr, attr_mask, &cmd, sizeof cmd); 740 } 741 742 int mlx5_query_srq(struct ibv_srq *srq, 743 struct ibv_srq_attr *attr) 744 { 745 struct ibv_query_srq cmd; 746 747 return ibv_cmd_query_srq(srq, attr, &cmd, sizeof cmd); 748 } 749 750 int mlx5_destroy_srq(struct ibv_srq *srq) 751 { 752 int ret; 753 struct mlx5_srq *msrq = to_msrq(srq); 754 struct mlx5_context *ctx = to_mctx(srq->context); 755 756 if (msrq->cmd_qp) { 757 ret = mlx5_destroy_qp(msrq->cmd_qp); 758 if (ret) 759 return ret; 760 msrq->cmd_qp = NULL; 761 } 762 763 ret = ibv_cmd_destroy_srq(srq); 764 if (ret) 765 return ret; 766 767 if (ctx->cqe_version && msrq->rsc.type == MLX5_RSC_TYPE_XSRQ) 768 mlx5_clear_uidx(ctx, msrq->rsc.rsn); 769 else 770 mlx5_clear_srq(ctx, msrq->srqn); 771 772 mlx5_free_db(ctx, msrq->db); 773 mlx5_free_buf(&msrq->buf); 774 free(msrq->wrid); 775 mlx5_spinlock_destroy(&msrq->lock); 776 free(msrq); 777 778 return 0; 779 } 780 781 static int sq_overhead(enum ibv_qp_type qp_type) 782 { 783 size_t size = 0; 784 size_t mw_bind_size = 785 sizeof(struct mlx5_wqe_umr_ctrl_seg) + 786 sizeof(struct mlx5_wqe_mkey_context_seg) + 787 max_t(size_t, sizeof(struct mlx5_wqe_umr_klm_seg), 64); 788 789 switch (qp_type) { 790 case IBV_QPT_RC: 791 size += sizeof(struct mlx5_wqe_ctrl_seg) + 792 max(sizeof(struct mlx5_wqe_atomic_seg) + 793 sizeof(struct mlx5_wqe_raddr_seg), 794 mw_bind_size); 795 break; 796 797 case IBV_QPT_UC: 798 size = sizeof(struct mlx5_wqe_ctrl_seg) + 799 max(sizeof(struct mlx5_wqe_raddr_seg), 800 mw_bind_size); 801 break; 802 803 case IBV_QPT_UD: 804 size = sizeof(struct mlx5_wqe_ctrl_seg) + 805 sizeof(struct mlx5_wqe_datagram_seg); 806 break; 807 808 case IBV_QPT_XRC_SEND: 809 size = sizeof(struct mlx5_wqe_ctrl_seg) + mw_bind_size; 810 SWITCH_FALLTHROUGH; 811 812 case IBV_QPT_XRC_RECV: 813 size = max(size, sizeof(struct mlx5_wqe_ctrl_seg) + 814 sizeof(struct mlx5_wqe_xrc_seg) + 815 sizeof(struct mlx5_wqe_raddr_seg)); 816 break; 817 818 case IBV_QPT_RAW_PACKET: 819 size = sizeof(struct mlx5_wqe_ctrl_seg) + 820 sizeof(struct mlx5_wqe_eth_seg); 821 break; 822 823 default: 824 return -EINVAL; 825 } 826 827 return size; 828 } 829 830 static int mlx5_calc_send_wqe(struct mlx5_context *ctx, 831 struct ibv_qp_init_attr_ex *attr, 832 struct mlx5_qp *qp) 833 { 834 int size; 835 int inl_size = 0; 836 int max_gather; 837 int tot_size; 838 839 size = sq_overhead(attr->qp_type); 840 if (size < 0) 841 return size; 842 843 if (attr->cap.max_inline_data) { 844 inl_size = size + align(sizeof(struct mlx5_wqe_inl_data_seg) + 845 attr->cap.max_inline_data, 16); 846 } 847 848 if (attr->comp_mask & IBV_QP_INIT_ATTR_MAX_TSO_HEADER) { 849 size += align(attr->max_tso_header, 16); 850 qp->max_tso_header = attr->max_tso_header; 851 } 852 853 max_gather = (ctx->max_sq_desc_sz - size) / 854 sizeof(struct mlx5_wqe_data_seg); 855 if (attr->cap.max_send_sge > max_gather) 856 return -EINVAL; 857 858 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg); 859 tot_size = max_int(size, inl_size); 860 861 if (tot_size > ctx->max_sq_desc_sz) 862 return -EINVAL; 863 864 return align(tot_size, MLX5_SEND_WQE_BB); 865 } 866 867 static int mlx5_calc_rcv_wqe(struct mlx5_context *ctx, 868 struct ibv_qp_init_attr_ex *attr, 869 struct mlx5_qp *qp) 870 { 871 uint32_t size; 872 int num_scatter; 873 874 if (attr->srq) 875 return 0; 876 877 num_scatter = max_t(uint32_t, attr->cap.max_recv_sge, 1); 878 size = sizeof(struct mlx5_wqe_data_seg) * num_scatter; 879 if (qp->wq_sig) 880 size += sizeof(struct mlx5_rwqe_sig); 881 882 if (size > ctx->max_rq_desc_sz) 883 return -EINVAL; 884 885 size = mlx5_round_up_power_of_two(size); 886 887 return size; 888 } 889 890 static int mlx5_calc_sq_size(struct mlx5_context *ctx, 891 struct ibv_qp_init_attr_ex *attr, 892 struct mlx5_qp *qp) 893 { 894 int wqe_size; 895 int wq_size; 896 FILE *fp = ctx->dbg_fp; 897 898 if (!attr->cap.max_send_wr) 899 return 0; 900 901 wqe_size = mlx5_calc_send_wqe(ctx, attr, qp); 902 if (wqe_size < 0) { 903 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 904 return wqe_size; 905 } 906 907 if (wqe_size > ctx->max_sq_desc_sz) { 908 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 909 return -EINVAL; 910 } 911 912 qp->max_inline_data = wqe_size - sq_overhead(attr->qp_type) - 913 sizeof(struct mlx5_wqe_inl_data_seg); 914 attr->cap.max_inline_data = qp->max_inline_data; 915 916 /* 917 * to avoid overflow, we limit max_send_wr so 918 * that the multiplication will fit in int 919 */ 920 if (attr->cap.max_send_wr > 0x7fffffff / ctx->max_sq_desc_sz) { 921 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 922 return -EINVAL; 923 } 924 925 wq_size = mlx5_round_up_power_of_two(attr->cap.max_send_wr * wqe_size); 926 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; 927 if (qp->sq.wqe_cnt > ctx->max_send_wqebb) { 928 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 929 return -EINVAL; 930 } 931 932 qp->sq.wqe_shift = mlx5_ilog2(MLX5_SEND_WQE_BB); 933 qp->sq.max_gs = attr->cap.max_send_sge; 934 qp->sq.max_post = wq_size / wqe_size; 935 936 return wq_size; 937 } 938 939 enum { 940 DV_CREATE_WQ_SUPPORTED_COMP_MASK = MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ 941 }; 942 943 static int mlx5_calc_rwq_size(struct mlx5_context *ctx, 944 struct mlx5_rwq *rwq, 945 struct ibv_wq_init_attr *attr, 946 struct mlx5dv_wq_init_attr *mlx5wq_attr) 947 { 948 size_t wqe_size; 949 int wq_size; 950 uint32_t num_scatter; 951 int is_mprq = 0; 952 int scat_spc; 953 954 if (!attr->max_wr) 955 return -EINVAL; 956 if (mlx5wq_attr) { 957 if (!check_comp_mask(mlx5wq_attr->comp_mask, 958 DV_CREATE_WQ_SUPPORTED_COMP_MASK)) 959 return -EINVAL; 960 961 is_mprq = !!(mlx5wq_attr->comp_mask & 962 MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ); 963 } 964 965 /* TBD: check caps for RQ */ 966 num_scatter = max_t(uint32_t, attr->max_sge, 1); 967 wqe_size = sizeof(struct mlx5_wqe_data_seg) * num_scatter + 968 sizeof(struct mlx5_wqe_srq_next_seg) * is_mprq; 969 970 if (rwq->wq_sig) 971 wqe_size += sizeof(struct mlx5_rwqe_sig); 972 973 if (wqe_size <= 0 || wqe_size > ctx->max_rq_desc_sz) 974 return -EINVAL; 975 976 wqe_size = mlx5_round_up_power_of_two(wqe_size); 977 wq_size = mlx5_round_up_power_of_two(attr->max_wr) * wqe_size; 978 wq_size = max(wq_size, MLX5_SEND_WQE_BB); 979 rwq->rq.wqe_cnt = wq_size / wqe_size; 980 rwq->rq.wqe_shift = mlx5_ilog2(wqe_size); 981 rwq->rq.max_post = 1 << mlx5_ilog2(wq_size / wqe_size); 982 scat_spc = wqe_size - 983 ((rwq->wq_sig) ? sizeof(struct mlx5_rwqe_sig) : 0) - 984 is_mprq * sizeof(struct mlx5_wqe_srq_next_seg); 985 rwq->rq.max_gs = scat_spc / sizeof(struct mlx5_wqe_data_seg); 986 return wq_size; 987 } 988 989 static int mlx5_calc_rq_size(struct mlx5_context *ctx, 990 struct ibv_qp_init_attr_ex *attr, 991 struct mlx5_qp *qp) 992 { 993 int wqe_size; 994 int wq_size; 995 int scat_spc; 996 FILE *fp = ctx->dbg_fp; 997 998 if (!attr->cap.max_recv_wr) 999 return 0; 1000 1001 if (attr->cap.max_recv_wr > ctx->max_recv_wr) { 1002 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 1003 return -EINVAL; 1004 } 1005 1006 wqe_size = mlx5_calc_rcv_wqe(ctx, attr, qp); 1007 if (wqe_size < 0 || wqe_size > ctx->max_rq_desc_sz) { 1008 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 1009 return -EINVAL; 1010 } 1011 1012 wq_size = mlx5_round_up_power_of_two(attr->cap.max_recv_wr) * wqe_size; 1013 if (wqe_size) { 1014 wq_size = max(wq_size, MLX5_SEND_WQE_BB); 1015 qp->rq.wqe_cnt = wq_size / wqe_size; 1016 qp->rq.wqe_shift = mlx5_ilog2(wqe_size); 1017 qp->rq.max_post = 1 << mlx5_ilog2(wq_size / wqe_size); 1018 scat_spc = wqe_size - 1019 (qp->wq_sig ? sizeof(struct mlx5_rwqe_sig) : 0); 1020 qp->rq.max_gs = scat_spc / sizeof(struct mlx5_wqe_data_seg); 1021 } else { 1022 qp->rq.wqe_cnt = 0; 1023 qp->rq.wqe_shift = 0; 1024 qp->rq.max_post = 0; 1025 qp->rq.max_gs = 0; 1026 } 1027 return wq_size; 1028 } 1029 1030 static int mlx5_calc_wq_size(struct mlx5_context *ctx, 1031 struct ibv_qp_init_attr_ex *attr, 1032 struct mlx5_qp *qp) 1033 { 1034 int ret; 1035 int result; 1036 1037 ret = mlx5_calc_sq_size(ctx, attr, qp); 1038 if (ret < 0) 1039 return ret; 1040 1041 result = ret; 1042 ret = mlx5_calc_rq_size(ctx, attr, qp); 1043 if (ret < 0) 1044 return ret; 1045 1046 result += ret; 1047 1048 qp->sq.offset = ret; 1049 qp->rq.offset = 0; 1050 1051 return result; 1052 } 1053 1054 static void map_uuar(struct ibv_context *context, struct mlx5_qp *qp, 1055 int uuar_index) 1056 { 1057 struct mlx5_context *ctx = to_mctx(context); 1058 1059 qp->bf = &ctx->bfs[uuar_index]; 1060 } 1061 1062 static const char *qptype2key(enum ibv_qp_type type) 1063 { 1064 switch (type) { 1065 case IBV_QPT_RC: return "HUGE_RC"; 1066 case IBV_QPT_UC: return "HUGE_UC"; 1067 case IBV_QPT_UD: return "HUGE_UD"; 1068 case IBV_QPT_RAW_PACKET: return "HUGE_RAW_ETH"; 1069 default: return "HUGE_NA"; 1070 } 1071 } 1072 1073 static int mlx5_alloc_qp_buf(struct ibv_context *context, 1074 struct ibv_qp_init_attr_ex *attr, 1075 struct mlx5_qp *qp, 1076 int size) 1077 { 1078 int err; 1079 enum mlx5_alloc_type alloc_type; 1080 enum mlx5_alloc_type default_alloc_type = MLX5_ALLOC_TYPE_ANON; 1081 const char *qp_huge_key; 1082 1083 if (qp->sq.wqe_cnt) { 1084 qp->sq.wrid = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wrid)); 1085 if (!qp->sq.wrid) { 1086 errno = ENOMEM; 1087 err = -1; 1088 return err; 1089 } 1090 1091 qp->sq.wr_data = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_data)); 1092 if (!qp->sq.wr_data) { 1093 errno = ENOMEM; 1094 err = -1; 1095 goto ex_wrid; 1096 } 1097 } 1098 1099 qp->sq.wqe_head = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wqe_head)); 1100 if (!qp->sq.wqe_head) { 1101 errno = ENOMEM; 1102 err = -1; 1103 goto ex_wrid; 1104 } 1105 1106 if (qp->rq.wqe_cnt) { 1107 qp->rq.wrid = malloc(qp->rq.wqe_cnt * sizeof(uint64_t)); 1108 if (!qp->rq.wrid) { 1109 errno = ENOMEM; 1110 err = -1; 1111 goto ex_wrid; 1112 } 1113 } 1114 1115 /* compatibility support */ 1116 qp_huge_key = qptype2key(qp->ibv_qp->qp_type); 1117 if (mlx5_use_huge(qp_huge_key)) 1118 default_alloc_type = MLX5_ALLOC_TYPE_HUGE; 1119 1120 mlx5_get_alloc_type(to_mctx(context), MLX5_QP_PREFIX, &alloc_type, 1121 default_alloc_type); 1122 1123 err = mlx5_alloc_prefered_buf(to_mctx(context), &qp->buf, 1124 align(qp->buf_size, to_mdev 1125 (context->device)->page_size), 1126 to_mdev(context->device)->page_size, 1127 alloc_type, 1128 MLX5_QP_PREFIX); 1129 1130 if (err) { 1131 err = -ENOMEM; 1132 goto ex_wrid; 1133 } 1134 1135 memset(qp->buf.buf, 0, qp->buf_size); 1136 1137 if (attr->qp_type == IBV_QPT_RAW_PACKET) { 1138 size_t aligned_sq_buf_size = align(qp->sq_buf_size, 1139 to_mdev(context->device)->page_size); 1140 /* For Raw Packet QP, allocate a separate buffer for the SQ */ 1141 err = mlx5_alloc_prefered_buf(to_mctx(context), &qp->sq_buf, 1142 aligned_sq_buf_size, 1143 to_mdev(context->device)->page_size, 1144 alloc_type, 1145 MLX5_QP_PREFIX); 1146 if (err) { 1147 err = -ENOMEM; 1148 goto rq_buf; 1149 } 1150 1151 memset(qp->sq_buf.buf, 0, aligned_sq_buf_size); 1152 } 1153 1154 return 0; 1155 rq_buf: 1156 mlx5_free_actual_buf(to_mctx(qp->verbs_qp.qp.context), &qp->buf); 1157 ex_wrid: 1158 if (qp->rq.wrid) 1159 free(qp->rq.wrid); 1160 1161 if (qp->sq.wqe_head) 1162 free(qp->sq.wqe_head); 1163 1164 if (qp->sq.wr_data) 1165 free(qp->sq.wr_data); 1166 if (qp->sq.wrid) 1167 free(qp->sq.wrid); 1168 1169 return err; 1170 } 1171 1172 static void mlx5_free_qp_buf(struct mlx5_qp *qp) 1173 { 1174 struct mlx5_context *ctx = to_mctx(qp->ibv_qp->context); 1175 1176 mlx5_free_actual_buf(ctx, &qp->buf); 1177 1178 if (qp->sq_buf.buf) 1179 mlx5_free_actual_buf(ctx, &qp->sq_buf); 1180 1181 if (qp->rq.wrid) 1182 free(qp->rq.wrid); 1183 1184 if (qp->sq.wqe_head) 1185 free(qp->sq.wqe_head); 1186 1187 if (qp->sq.wrid) 1188 free(qp->sq.wrid); 1189 1190 if (qp->sq.wr_data) 1191 free(qp->sq.wr_data); 1192 } 1193 1194 static int mlx5_cmd_create_rss_qp(struct ibv_context *context, 1195 struct ibv_qp_init_attr_ex *attr, 1196 struct mlx5_qp *qp, 1197 uint32_t mlx5_create_flags) 1198 { 1199 struct mlx5_create_qp_ex_rss cmd_ex_rss = {}; 1200 struct mlx5_create_qp_resp_ex resp = {}; 1201 int ret; 1202 1203 if (attr->rx_hash_conf.rx_hash_key_len > sizeof(cmd_ex_rss.rx_hash_key)) { 1204 errno = EINVAL; 1205 return errno; 1206 } 1207 1208 cmd_ex_rss.rx_hash_fields_mask = attr->rx_hash_conf.rx_hash_fields_mask; 1209 cmd_ex_rss.rx_hash_function = attr->rx_hash_conf.rx_hash_function; 1210 cmd_ex_rss.rx_key_len = attr->rx_hash_conf.rx_hash_key_len; 1211 cmd_ex_rss.create_flags = mlx5_create_flags; 1212 memcpy(cmd_ex_rss.rx_hash_key, attr->rx_hash_conf.rx_hash_key, 1213 attr->rx_hash_conf.rx_hash_key_len); 1214 1215 ret = ibv_cmd_create_qp_ex2(context, &qp->verbs_qp, 1216 sizeof(qp->verbs_qp), attr, 1217 &cmd_ex_rss.ibv_cmd, sizeof(cmd_ex_rss.ibv_cmd), 1218 sizeof(cmd_ex_rss), &resp.ibv_resp, 1219 sizeof(resp.ibv_resp), sizeof(resp)); 1220 if (ret) 1221 return ret; 1222 1223 qp->rss_qp = 1; 1224 return 0; 1225 } 1226 1227 static int mlx5_cmd_create_qp_ex(struct ibv_context *context, 1228 struct ibv_qp_init_attr_ex *attr, 1229 struct mlx5_create_qp *cmd, 1230 struct mlx5_qp *qp, 1231 struct mlx5_create_qp_resp_ex *resp) 1232 { 1233 struct mlx5_create_qp_ex cmd_ex; 1234 int ret; 1235 1236 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1237 memcpy(&cmd_ex.ibv_cmd.base, &cmd->ibv_cmd.user_handle, 1238 offsetof(typeof(cmd->ibv_cmd), is_srq) + 1239 sizeof(cmd->ibv_cmd.is_srq) - 1240 offsetof(typeof(cmd->ibv_cmd), user_handle)); 1241 1242 memcpy(&cmd_ex.drv_ex, &cmd->buf_addr, 1243 offsetof(typeof(*cmd), sq_buf_addr) + 1244 sizeof(cmd->sq_buf_addr) - sizeof(cmd->ibv_cmd)); 1245 1246 ret = ibv_cmd_create_qp_ex2(context, &qp->verbs_qp, 1247 sizeof(qp->verbs_qp), attr, 1248 &cmd_ex.ibv_cmd, sizeof(cmd_ex.ibv_cmd), 1249 sizeof(cmd_ex), &resp->ibv_resp, 1250 sizeof(resp->ibv_resp), sizeof(*resp)); 1251 1252 return ret; 1253 } 1254 1255 enum { 1256 MLX5_CREATE_QP_SUP_COMP_MASK = (IBV_QP_INIT_ATTR_PD | 1257 IBV_QP_INIT_ATTR_XRCD | 1258 IBV_QP_INIT_ATTR_CREATE_FLAGS | 1259 IBV_QP_INIT_ATTR_MAX_TSO_HEADER | 1260 IBV_QP_INIT_ATTR_IND_TABLE | 1261 IBV_QP_INIT_ATTR_RX_HASH), 1262 }; 1263 1264 enum { 1265 MLX5_DV_CREATE_QP_SUP_COMP_MASK = MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS 1266 }; 1267 1268 enum { 1269 MLX5DV_QP_CREATE_SUP_FLAGS = 1270 (MLX5DV_QP_CREATE_TUNNEL_OFFLOADS), 1271 }; 1272 1273 enum { 1274 MLX5_CREATE_QP_EX2_COMP_MASK = (IBV_QP_INIT_ATTR_CREATE_FLAGS | 1275 IBV_QP_INIT_ATTR_MAX_TSO_HEADER | 1276 IBV_QP_INIT_ATTR_IND_TABLE | 1277 IBV_QP_INIT_ATTR_RX_HASH), 1278 }; 1279 1280 static struct ibv_qp *create_qp(struct ibv_context *context, 1281 struct ibv_qp_init_attr_ex *attr, 1282 struct mlx5dv_qp_init_attr *mlx5_qp_attr) 1283 { 1284 struct mlx5_create_qp cmd; 1285 struct mlx5_create_qp_resp resp; 1286 struct mlx5_create_qp_resp_ex resp_ex; 1287 struct mlx5_qp *qp; 1288 int ret; 1289 struct mlx5_context *ctx = to_mctx(context); 1290 struct ibv_qp *ibqp; 1291 int32_t usr_idx = 0; 1292 uint32_t uuar_index; 1293 uint32_t mlx5_create_flags = 0; 1294 FILE *fp = ctx->dbg_fp; 1295 1296 if (attr->comp_mask & ~MLX5_CREATE_QP_SUP_COMP_MASK) 1297 return NULL; 1298 1299 if ((attr->comp_mask & IBV_QP_INIT_ATTR_MAX_TSO_HEADER) && 1300 (attr->qp_type != IBV_QPT_RAW_PACKET)) 1301 return NULL; 1302 1303 qp = calloc(1, sizeof(*qp)); 1304 if (!qp) { 1305 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 1306 return NULL; 1307 } 1308 ibqp = (struct ibv_qp *)&qp->verbs_qp; 1309 qp->ibv_qp = ibqp; 1310 1311 memset(&cmd, 0, sizeof(cmd)); 1312 memset(&resp, 0, sizeof(resp)); 1313 memset(&resp_ex, 0, sizeof(resp_ex)); 1314 1315 if (mlx5_qp_attr) { 1316 if (!check_comp_mask(mlx5_qp_attr->comp_mask, 1317 MLX5_DV_CREATE_QP_SUP_COMP_MASK)) { 1318 mlx5_dbg(fp, MLX5_DBG_QP, 1319 "Unsupported vendor comp_mask for create_qp\n"); 1320 errno = EINVAL; 1321 goto err; 1322 } 1323 1324 if (mlx5_qp_attr->comp_mask & 1325 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS) { 1326 if (!check_comp_mask(mlx5_qp_attr->create_flags, 1327 MLX5DV_QP_CREATE_SUP_FLAGS)) { 1328 mlx5_dbg(fp, MLX5_DBG_QP, 1329 "Unsupported creation flags requested for create_qp\n"); 1330 errno = EINVAL; 1331 goto err; 1332 } 1333 if (mlx5_qp_attr->create_flags & 1334 MLX5DV_QP_CREATE_TUNNEL_OFFLOADS) { 1335 mlx5_create_flags |= MLX5_QP_FLAG_TUNNEL_OFFLOADS; 1336 } 1337 } 1338 } 1339 1340 if (attr->comp_mask & IBV_QP_INIT_ATTR_RX_HASH) { 1341 ret = mlx5_cmd_create_rss_qp(context, attr, qp, 1342 mlx5_create_flags); 1343 if (ret) 1344 goto err; 1345 1346 return ibqp; 1347 } 1348 1349 cmd.flags = mlx5_create_flags; 1350 qp->wq_sig = qp_sig_enabled(); 1351 if (qp->wq_sig) 1352 cmd.flags |= MLX5_QP_FLAG_SIGNATURE; 1353 1354 if (use_scatter_to_cqe()) 1355 cmd.flags |= MLX5_QP_FLAG_SCATTER_CQE; 1356 1357 ret = mlx5_calc_wq_size(ctx, attr, qp); 1358 if (ret < 0) { 1359 errno = -ret; 1360 goto err; 1361 } 1362 1363 if (attr->qp_type == IBV_QPT_RAW_PACKET) { 1364 qp->buf_size = qp->sq.offset; 1365 qp->sq_buf_size = ret - qp->buf_size; 1366 qp->sq.offset = 0; 1367 } else { 1368 qp->buf_size = ret; 1369 qp->sq_buf_size = 0; 1370 } 1371 1372 if (mlx5_alloc_qp_buf(context, attr, qp, ret)) { 1373 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 1374 goto err; 1375 } 1376 1377 if (attr->qp_type == IBV_QPT_RAW_PACKET) { 1378 qp->sq_start = qp->sq_buf.buf; 1379 qp->sq.qend = qp->sq_buf.buf + 1380 (qp->sq.wqe_cnt << qp->sq.wqe_shift); 1381 } else { 1382 qp->sq_start = qp->buf.buf + qp->sq.offset; 1383 qp->sq.qend = qp->buf.buf + qp->sq.offset + 1384 (qp->sq.wqe_cnt << qp->sq.wqe_shift); 1385 } 1386 1387 mlx5_init_qp_indices(qp); 1388 1389 if (mlx5_spinlock_init(&qp->sq.lock)) 1390 goto err_free_qp_buf; 1391 1392 if (mlx5_spinlock_init(&qp->rq.lock)) 1393 goto err_sq_spl; 1394 1395 qp->db = mlx5_alloc_dbrec(ctx); 1396 if (!qp->db) { 1397 mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 1398 goto err_rq_spl; 1399 } 1400 1401 qp->db[MLX5_RCV_DBR] = 0; 1402 qp->db[MLX5_SND_DBR] = 0; 1403 1404 cmd.buf_addr = (uintptr_t) qp->buf.buf; 1405 cmd.sq_buf_addr = (attr->qp_type == IBV_QPT_RAW_PACKET) ? 1406 (uintptr_t) qp->sq_buf.buf : 0; 1407 cmd.db_addr = (uintptr_t) qp->db; 1408 cmd.sq_wqe_count = qp->sq.wqe_cnt; 1409 cmd.rq_wqe_count = qp->rq.wqe_cnt; 1410 cmd.rq_wqe_shift = qp->rq.wqe_shift; 1411 1412 if (ctx->atomic_cap == IBV_ATOMIC_HCA) 1413 qp->atomics_enabled = 1; 1414 1415 if (!ctx->cqe_version) { 1416 cmd.uidx = 0xffffff; 1417 pthread_mutex_lock(&ctx->qp_table_mutex); 1418 } else if (!is_xrc_tgt(attr->qp_type)) { 1419 usr_idx = mlx5_store_uidx(ctx, qp); 1420 if (usr_idx < 0) { 1421 mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n"); 1422 goto err_rq_db; 1423 } 1424 1425 cmd.uidx = usr_idx; 1426 } 1427 1428 if (attr->comp_mask & MLX5_CREATE_QP_EX2_COMP_MASK) 1429 ret = mlx5_cmd_create_qp_ex(context, attr, &cmd, qp, &resp_ex); 1430 else 1431 ret = ibv_cmd_create_qp_ex(context, &qp->verbs_qp, sizeof(qp->verbs_qp), 1432 attr, &cmd.ibv_cmd, sizeof(cmd), 1433 &resp.ibv_resp, sizeof(resp)); 1434 if (ret) { 1435 mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); 1436 goto err_free_uidx; 1437 } 1438 1439 uuar_index = (attr->comp_mask & MLX5_CREATE_QP_EX2_COMP_MASK) ? 1440 resp_ex.uuar_index : resp.uuar_index; 1441 if (!ctx->cqe_version) { 1442 if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) { 1443 ret = mlx5_store_qp(ctx, ibqp->qp_num, qp); 1444 if (ret) { 1445 mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); 1446 goto err_destroy; 1447 } 1448 } 1449 1450 pthread_mutex_unlock(&ctx->qp_table_mutex); 1451 } 1452 1453 map_uuar(context, qp, uuar_index); 1454 1455 qp->rq.max_post = qp->rq.wqe_cnt; 1456 if (attr->sq_sig_all) 1457 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE; 1458 else 1459 qp->sq_signal_bits = 0; 1460 1461 attr->cap.max_send_wr = qp->sq.max_post; 1462 attr->cap.max_recv_wr = qp->rq.max_post; 1463 attr->cap.max_recv_sge = qp->rq.max_gs; 1464 1465 qp->rsc.type = MLX5_RSC_TYPE_QP; 1466 qp->rsc.rsn = (ctx->cqe_version && !is_xrc_tgt(attr->qp_type)) ? 1467 usr_idx : ibqp->qp_num; 1468 1469 return ibqp; 1470 1471 err_destroy: 1472 ibv_cmd_destroy_qp(ibqp); 1473 1474 err_free_uidx: 1475 if (!ctx->cqe_version) 1476 pthread_mutex_unlock(&to_mctx(context)->qp_table_mutex); 1477 else if (!is_xrc_tgt(attr->qp_type)) 1478 mlx5_clear_uidx(ctx, usr_idx); 1479 1480 err_rq_db: 1481 mlx5_free_db(to_mctx(context), qp->db); 1482 1483 err_rq_spl: 1484 mlx5_spinlock_destroy(&qp->rq.lock); 1485 1486 err_sq_spl: 1487 mlx5_spinlock_destroy(&qp->sq.lock); 1488 1489 err_free_qp_buf: 1490 mlx5_free_qp_buf(qp); 1491 1492 err: 1493 free(qp); 1494 1495 return NULL; 1496 } 1497 1498 struct ibv_qp *mlx5_create_qp(struct ibv_pd *pd, 1499 struct ibv_qp_init_attr *attr) 1500 { 1501 struct ibv_qp *qp; 1502 struct ibv_qp_init_attr_ex attrx; 1503 1504 memset(&attrx, 0, sizeof(attrx)); 1505 memcpy(&attrx, attr, sizeof(*attr)); 1506 attrx.comp_mask = IBV_QP_INIT_ATTR_PD; 1507 attrx.pd = pd; 1508 qp = create_qp(pd->context, &attrx, NULL); 1509 if (qp) 1510 memcpy(attr, &attrx, sizeof(*attr)); 1511 1512 return qp; 1513 } 1514 1515 static void mlx5_lock_cqs(struct ibv_qp *qp) 1516 { 1517 struct mlx5_cq *send_cq = to_mcq(qp->send_cq); 1518 struct mlx5_cq *recv_cq = to_mcq(qp->recv_cq); 1519 1520 if (send_cq && recv_cq) { 1521 if (send_cq == recv_cq) { 1522 mlx5_spin_lock(&send_cq->lock); 1523 } else if (send_cq->cqn < recv_cq->cqn) { 1524 mlx5_spin_lock(&send_cq->lock); 1525 mlx5_spin_lock(&recv_cq->lock); 1526 } else { 1527 mlx5_spin_lock(&recv_cq->lock); 1528 mlx5_spin_lock(&send_cq->lock); 1529 } 1530 } else if (send_cq) { 1531 mlx5_spin_lock(&send_cq->lock); 1532 } else if (recv_cq) { 1533 mlx5_spin_lock(&recv_cq->lock); 1534 } 1535 } 1536 1537 static void mlx5_unlock_cqs(struct ibv_qp *qp) 1538 { 1539 struct mlx5_cq *send_cq = to_mcq(qp->send_cq); 1540 struct mlx5_cq *recv_cq = to_mcq(qp->recv_cq); 1541 1542 if (send_cq && recv_cq) { 1543 if (send_cq == recv_cq) { 1544 mlx5_spin_unlock(&send_cq->lock); 1545 } else if (send_cq->cqn < recv_cq->cqn) { 1546 mlx5_spin_unlock(&recv_cq->lock); 1547 mlx5_spin_unlock(&send_cq->lock); 1548 } else { 1549 mlx5_spin_unlock(&send_cq->lock); 1550 mlx5_spin_unlock(&recv_cq->lock); 1551 } 1552 } else if (send_cq) { 1553 mlx5_spin_unlock(&send_cq->lock); 1554 } else if (recv_cq) { 1555 mlx5_spin_unlock(&recv_cq->lock); 1556 } 1557 } 1558 1559 int mlx5_destroy_qp(struct ibv_qp *ibqp) 1560 { 1561 struct mlx5_qp *qp = to_mqp(ibqp); 1562 struct mlx5_context *ctx = to_mctx(ibqp->context); 1563 int ret; 1564 1565 if (qp->rss_qp) { 1566 ret = ibv_cmd_destroy_qp(ibqp); 1567 if (ret) 1568 return ret; 1569 goto free; 1570 } 1571 1572 if (!ctx->cqe_version) 1573 pthread_mutex_lock(&ctx->qp_table_mutex); 1574 1575 ret = ibv_cmd_destroy_qp(ibqp); 1576 if (ret) { 1577 if (!ctx->cqe_version) 1578 pthread_mutex_unlock(&ctx->qp_table_mutex); 1579 return ret; 1580 } 1581 1582 mlx5_lock_cqs(ibqp); 1583 1584 __mlx5_cq_clean(to_mcq(ibqp->recv_cq), qp->rsc.rsn, 1585 ibqp->srq ? to_msrq(ibqp->srq) : NULL); 1586 if (ibqp->send_cq != ibqp->recv_cq) 1587 __mlx5_cq_clean(to_mcq(ibqp->send_cq), qp->rsc.rsn, NULL); 1588 1589 if (!ctx->cqe_version) { 1590 if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) 1591 mlx5_clear_qp(ctx, ibqp->qp_num); 1592 } 1593 1594 mlx5_unlock_cqs(ibqp); 1595 if (!ctx->cqe_version) 1596 pthread_mutex_unlock(&ctx->qp_table_mutex); 1597 else if (!is_xrc_tgt(ibqp->qp_type)) 1598 mlx5_clear_uidx(ctx, qp->rsc.rsn); 1599 1600 mlx5_free_db(ctx, qp->db); 1601 mlx5_spinlock_destroy(&qp->rq.lock); 1602 mlx5_spinlock_destroy(&qp->sq.lock); 1603 mlx5_free_qp_buf(qp); 1604 free: 1605 free(qp); 1606 1607 return 0; 1608 } 1609 1610 int mlx5_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, 1611 int attr_mask, struct ibv_qp_init_attr *init_attr) 1612 { 1613 struct ibv_query_qp cmd; 1614 struct mlx5_qp *qp = to_mqp(ibqp); 1615 int ret; 1616 1617 if (qp->rss_qp) 1618 return ENOSYS; 1619 1620 ret = ibv_cmd_query_qp(ibqp, attr, attr_mask, init_attr, &cmd, sizeof(cmd)); 1621 if (ret) 1622 return ret; 1623 1624 init_attr->cap.max_send_wr = qp->sq.max_post; 1625 init_attr->cap.max_send_sge = qp->sq.max_gs; 1626 init_attr->cap.max_inline_data = qp->max_inline_data; 1627 1628 attr->cap = init_attr->cap; 1629 1630 return 0; 1631 } 1632 1633 enum { 1634 MLX5_MODIFY_QP_EX_ATTR_MASK = IBV_QP_RATE_LIMIT, 1635 }; 1636 1637 int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1638 int attr_mask) 1639 { 1640 struct ibv_modify_qp cmd = {}; 1641 struct ibv_modify_qp_ex cmd_ex = {}; 1642 struct ibv_modify_qp_resp_ex resp = {}; 1643 struct mlx5_qp *mqp = to_mqp(qp); 1644 struct mlx5_context *context = to_mctx(qp->context); 1645 int ret; 1646 __be32 *db; 1647 1648 if (mqp->rss_qp) 1649 return ENOSYS; 1650 1651 if (attr_mask & IBV_QP_PORT) { 1652 switch (qp->qp_type) { 1653 case IBV_QPT_RAW_PACKET: 1654 if (context->cached_link_layer[attr->port_num - 1] == 1655 IBV_LINK_LAYER_ETHERNET) { 1656 if (context->cached_device_cap_flags & 1657 IBV_DEVICE_RAW_IP_CSUM) 1658 mqp->qp_cap_cache |= 1659 MLX5_CSUM_SUPPORT_RAW_OVER_ETH | 1660 MLX5_RX_CSUM_VALID; 1661 1662 if (ibv_is_qpt_supported( 1663 context->cached_tso_caps.supported_qpts, 1664 IBV_QPT_RAW_PACKET)) 1665 mqp->max_tso = 1666 context->cached_tso_caps.max_tso; 1667 } 1668 break; 1669 default: 1670 break; 1671 } 1672 } 1673 1674 if (attr_mask & MLX5_MODIFY_QP_EX_ATTR_MASK) 1675 ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, 1676 &cmd_ex, 1677 sizeof(cmd_ex), sizeof(cmd_ex), 1678 &resp, 1679 sizeof(resp), sizeof(resp)); 1680 else 1681 ret = ibv_cmd_modify_qp(qp, attr, attr_mask, 1682 &cmd, sizeof(cmd)); 1683 1684 if (!ret && 1685 (attr_mask & IBV_QP_STATE) && 1686 attr->qp_state == IBV_QPS_RESET) { 1687 if (qp->recv_cq) { 1688 mlx5_cq_clean(to_mcq(qp->recv_cq), mqp->rsc.rsn, 1689 qp->srq ? to_msrq(qp->srq) : NULL); 1690 } 1691 if (qp->send_cq != qp->recv_cq && qp->send_cq) 1692 mlx5_cq_clean(to_mcq(qp->send_cq), 1693 to_mqp(qp)->rsc.rsn, NULL); 1694 1695 mlx5_init_qp_indices(mqp); 1696 db = mqp->db; 1697 db[MLX5_RCV_DBR] = 0; 1698 db[MLX5_SND_DBR] = 0; 1699 } 1700 1701 /* 1702 * When the Raw Packet QP is in INIT state, its RQ 1703 * underneath is already in RDY, which means it can 1704 * receive packets. According to the IB spec, a QP can't 1705 * receive packets until moved to RTR state. To achieve this, 1706 * for Raw Packet QPs, we update the doorbell record 1707 * once the QP is moved to RTR. 1708 */ 1709 if (!ret && 1710 (attr_mask & IBV_QP_STATE) && 1711 attr->qp_state == IBV_QPS_RTR && 1712 qp->qp_type == IBV_QPT_RAW_PACKET) { 1713 mlx5_spin_lock(&mqp->rq.lock); 1714 mqp->db[MLX5_RCV_DBR] = htobe32(mqp->rq.head & 0xffff); 1715 mlx5_spin_unlock(&mqp->rq.lock); 1716 } 1717 1718 return ret; 1719 } 1720 1721 #define RROCE_UDP_SPORT_MIN 0xC000 1722 #define RROCE_UDP_SPORT_MAX 0xFFFF 1723 struct ibv_ah *mlx5_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) 1724 { 1725 struct mlx5_context *ctx = to_mctx(pd->context); 1726 struct ibv_port_attr port_attr; 1727 struct mlx5_ah *ah; 1728 uint32_t gid_type; 1729 __be32 tmp; 1730 uint8_t grh; 1731 int is_eth; 1732 1733 if (attr->port_num < 1 || attr->port_num > ctx->num_ports) 1734 return NULL; 1735 1736 if (ctx->cached_link_layer[attr->port_num - 1]) { 1737 is_eth = ctx->cached_link_layer[attr->port_num - 1] == 1738 IBV_LINK_LAYER_ETHERNET; 1739 } else { 1740 if (ibv_query_port(pd->context, attr->port_num, &port_attr)) 1741 return NULL; 1742 1743 is_eth = (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET); 1744 } 1745 1746 if (unlikely((!attr->is_global) && is_eth)) { 1747 errno = EINVAL; 1748 return NULL; 1749 } 1750 1751 ah = calloc(1, sizeof *ah); 1752 if (!ah) 1753 return NULL; 1754 1755 if (is_eth) { 1756 if (ibv_query_gid_type(pd->context, attr->port_num, 1757 attr->grh.sgid_index, &gid_type)) 1758 goto err; 1759 1760 if (gid_type == IBV_GID_TYPE_ROCE_V2) 1761 ah->av.rlid = htobe16(rand() % (RROCE_UDP_SPORT_MAX + 1 1762 - RROCE_UDP_SPORT_MIN) 1763 + RROCE_UDP_SPORT_MIN); 1764 /* Since RoCE packets must contain GRH, this bit is reserved 1765 * for RoCE and shouldn't be set. 1766 */ 1767 grh = 0; 1768 } else { 1769 ah->av.fl_mlid = attr->src_path_bits & 0x7f; 1770 ah->av.rlid = htobe16(attr->dlid); 1771 grh = 1; 1772 } 1773 ah->av.stat_rate_sl = (attr->static_rate << 4) | attr->sl; 1774 if (attr->is_global) { 1775 ah->av.tclass = attr->grh.traffic_class; 1776 ah->av.hop_limit = attr->grh.hop_limit; 1777 tmp = htobe32((grh << 30) | 1778 ((attr->grh.sgid_index & 0xff) << 20) | 1779 (attr->grh.flow_label & 0xfffff)); 1780 ah->av.grh_gid_fl = tmp; 1781 memcpy(ah->av.rgid, attr->grh.dgid.raw, 16); 1782 } 1783 1784 if (is_eth) { 1785 if (ctx->cmds_supp_uhw & MLX5_USER_CMDS_SUPP_UHW_CREATE_AH) { 1786 struct mlx5_create_ah_resp resp = {}; 1787 1788 if (ibv_cmd_create_ah(pd, &ah->ibv_ah, attr, &resp.ibv_resp, sizeof(resp))) 1789 goto err; 1790 1791 ah->kern_ah = true; 1792 memcpy(ah->av.rmac, resp.dmac, ETHERNET_LL_SIZE); 1793 } else { 1794 uint16_t vid; 1795 1796 if (ibv_resolve_eth_l2_from_gid(pd->context, attr, 1797 ah->av.rmac, &vid)) 1798 goto err; 1799 } 1800 } 1801 1802 return &ah->ibv_ah; 1803 err: 1804 free(ah); 1805 return NULL; 1806 } 1807 1808 int mlx5_destroy_ah(struct ibv_ah *ah) 1809 { 1810 struct mlx5_ah *mah = to_mah(ah); 1811 int err; 1812 1813 if (mah->kern_ah) { 1814 err = ibv_cmd_destroy_ah(ah); 1815 if (err) 1816 return err; 1817 } 1818 1819 free(mah); 1820 return 0; 1821 } 1822 1823 int mlx5_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) 1824 { 1825 return ibv_cmd_attach_mcast(qp, gid, lid); 1826 } 1827 1828 int mlx5_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) 1829 { 1830 return ibv_cmd_detach_mcast(qp, gid, lid); 1831 } 1832 1833 struct ibv_qp *mlx5_create_qp_ex(struct ibv_context *context, 1834 struct ibv_qp_init_attr_ex *attr) 1835 { 1836 return create_qp(context, attr, NULL); 1837 } 1838 1839 struct ibv_qp *mlx5dv_create_qp(struct ibv_context *context, 1840 struct ibv_qp_init_attr_ex *qp_attr, 1841 struct mlx5dv_qp_init_attr *mlx5_qp_attr) 1842 { 1843 return create_qp(context, qp_attr, mlx5_qp_attr); 1844 } 1845 1846 int mlx5_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num) 1847 { 1848 struct mlx5_srq *msrq = to_msrq(srq); 1849 1850 *srq_num = msrq->srqn; 1851 1852 return 0; 1853 } 1854 1855 struct ibv_xrcd * 1856 mlx5_open_xrcd(struct ibv_context *context, 1857 struct ibv_xrcd_init_attr *xrcd_init_attr) 1858 { 1859 int err; 1860 struct verbs_xrcd *xrcd; 1861 struct ibv_open_xrcd cmd = {}; 1862 struct ibv_open_xrcd_resp resp = {}; 1863 1864 xrcd = calloc(1, sizeof(*xrcd)); 1865 if (!xrcd) 1866 return NULL; 1867 1868 err = ibv_cmd_open_xrcd(context, xrcd, sizeof(*xrcd), xrcd_init_attr, 1869 &cmd, sizeof(cmd), &resp, sizeof(resp)); 1870 if (err) { 1871 free(xrcd); 1872 return NULL; 1873 } 1874 1875 return &xrcd->xrcd; 1876 } 1877 1878 int mlx5_close_xrcd(struct ibv_xrcd *ib_xrcd) 1879 { 1880 struct verbs_xrcd *xrcd = container_of(ib_xrcd, struct verbs_xrcd, xrcd); 1881 int ret; 1882 1883 ret = ibv_cmd_close_xrcd(xrcd); 1884 if (!ret) 1885 free(xrcd); 1886 1887 return ret; 1888 } 1889 1890 static struct ibv_qp * 1891 create_cmd_qp(struct ibv_context *context, 1892 struct ibv_srq_init_attr_ex *srq_attr, 1893 struct ibv_srq *srq) 1894 { 1895 struct ibv_qp_init_attr_ex init_attr = {}; 1896 FILE *fp = to_mctx(context)->dbg_fp; 1897 struct ibv_port_attr port_attr; 1898 struct ibv_modify_qp qcmd = {}; 1899 struct ibv_qp_attr attr = {}; 1900 struct ibv_query_port pcmd; 1901 struct ibv_qp *qp; 1902 int attr_mask; 1903 int port = 1; 1904 int ret; 1905 1906 /* 1907 * The command QP send WQ is sized from tm_cap.max_ops; a value of 0 1908 * would create a zero-length WQ, so reject it explicitly. 1909 */ 1910 if (srq_attr->tm_cap.max_ops == 0) { 1911 errno = EINVAL; 1912 return NULL; 1913 } 1914 1915 ret = ibv_cmd_query_port(context, port, &port_attr, 1916 &pcmd, sizeof(pcmd)); 1917 if (ret) { 1918 mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); 1919 return NULL; 1920 } 1921 1922 init_attr.qp_type = IBV_QPT_RC; 1923 init_attr.srq = srq; 1924 /* Command QP will be used to pass MLX5_OPCODE_TAG_MATCHING messages 1925 * to add/remove tag matching list entries. 1926 * WQ size is based on max_ops parameter holding max number of 1927 * outstanding list operations. 1928 */ 1929 init_attr.cap.max_send_wr = srq_attr->tm_cap.max_ops; 1930 /* Tag matching list entry will point to a single sge buffer */ 1931 init_attr.cap.max_send_sge = 1; 1932 init_attr.comp_mask = IBV_QP_INIT_ATTR_PD; 1933 init_attr.pd = srq_attr->pd; 1934 init_attr.send_cq = srq_attr->cq; 1935 init_attr.recv_cq = srq_attr->cq; 1936 1937 qp = create_qp(context, &init_attr, NULL); 1938 if (!qp) 1939 return NULL; 1940 1941 attr.qp_state = IBV_QPS_INIT; 1942 attr.port_num = port; 1943 attr_mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX 1944 | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS; 1945 1946 ret = ibv_cmd_modify_qp(qp, &attr, attr_mask, &qcmd, sizeof(qcmd)); 1947 if (ret) { 1948 mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); 1949 goto err; 1950 } 1951 1952 attr.qp_state = IBV_QPS_RTR; 1953 attr.path_mtu = IBV_MTU_256; 1954 attr.dest_qp_num = qp->qp_num; /* Loopback */ 1955 attr.ah_attr.dlid = port_attr.lid; 1956 attr.ah_attr.port_num = port; 1957 attr_mask = IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU 1958 | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN 1959 | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER; 1960 1961 ret = ibv_cmd_modify_qp(qp, &attr, attr_mask, &qcmd, sizeof(qcmd)); 1962 if (ret) { 1963 mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); 1964 goto err; 1965 } 1966 1967 attr.qp_state = IBV_QPS_RTS; 1968 attr_mask = IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT 1969 | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN 1970 | IBV_QP_MAX_QP_RD_ATOMIC; 1971 1972 ret = ibv_cmd_modify_qp(qp, &attr, attr_mask, &qcmd, sizeof(qcmd)); 1973 if (ret) { 1974 mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); 1975 goto err; 1976 } 1977 1978 return qp; 1979 1980 err: 1981 mlx5_destroy_qp(qp); 1982 return NULL; 1983 } 1984 1985 struct ibv_srq *mlx5_create_srq_ex(struct ibv_context *context, 1986 struct ibv_srq_init_attr_ex *attr) 1987 { 1988 int err; 1989 struct mlx5_create_srq_ex cmd; 1990 struct mlx5_create_srq_resp resp; 1991 struct mlx5_srq *msrq; 1992 struct mlx5_context *ctx = to_mctx(context); 1993 int max_sge; 1994 struct ibv_srq *ibsrq; 1995 int uidx; 1996 FILE *fp = ctx->dbg_fp; 1997 1998 if (!(attr->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) || 1999 (attr->srq_type == IBV_SRQT_BASIC)) 2000 return mlx5_create_srq(attr->pd, 2001 (struct ibv_srq_init_attr *)attr); 2002 2003 if (attr->srq_type != IBV_SRQT_XRC && 2004 attr->srq_type != IBV_SRQT_TM) { 2005 errno = EINVAL; 2006 return NULL; 2007 } 2008 2009 /* An extended CQ is required to read TM information from */ 2010 if (attr->srq_type == IBV_SRQT_TM && 2011 !(attr->cq && (to_mcq(attr->cq)->flags & MLX5_CQ_FLAGS_EXTENDED))) { 2012 errno = EINVAL; 2013 return NULL; 2014 } 2015 2016 msrq = calloc(1, sizeof(*msrq)); 2017 if (!msrq) 2018 return NULL; 2019 2020 ibsrq = (struct ibv_srq *)&msrq->vsrq; 2021 2022 memset(&cmd, 0, sizeof(cmd)); 2023 memset(&resp, 0, sizeof(resp)); 2024 2025 if (mlx5_spinlock_init(&msrq->lock)) { 2026 fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 2027 goto err; 2028 } 2029 2030 if (attr->attr.max_wr > ctx->max_srq_recv_wr) { 2031 fprintf(stderr, "%s-%d:max_wr %d, max_srq_recv_wr %d\n", 2032 __func__, __LINE__, attr->attr.max_wr, 2033 ctx->max_srq_recv_wr); 2034 errno = EINVAL; 2035 goto err_spl; 2036 } 2037 2038 /* 2039 * this calculation does not consider required control segments. The 2040 * final calculation is done again later. This is done so to avoid 2041 * overflows of variables 2042 */ 2043 max_sge = ctx->max_recv_wr / sizeof(struct mlx5_wqe_data_seg); 2044 if (attr->attr.max_sge > max_sge) { 2045 fprintf(stderr, "%s-%d:max_wr %d, max_srq_recv_wr %d\n", 2046 __func__, __LINE__, attr->attr.max_wr, 2047 ctx->max_srq_recv_wr); 2048 errno = EINVAL; 2049 goto err_spl; 2050 } 2051 2052 msrq->max = align_queue_size(attr->attr.max_wr + 1); 2053 msrq->max_gs = attr->attr.max_sge; 2054 msrq->counter = 0; 2055 2056 if (mlx5_alloc_srq_buf(context, msrq)) { 2057 fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 2058 goto err_spl; 2059 } 2060 2061 msrq->db = mlx5_alloc_dbrec(ctx); 2062 if (!msrq->db) { 2063 fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 2064 goto err_free; 2065 } 2066 2067 *msrq->db = 0; 2068 2069 cmd.buf_addr = (uintptr_t)msrq->buf.buf; 2070 cmd.db_addr = (uintptr_t)msrq->db; 2071 msrq->wq_sig = srq_sig_enabled(); 2072 if (msrq->wq_sig) 2073 cmd.flags = MLX5_SRQ_FLAG_SIGNATURE; 2074 2075 attr->attr.max_sge = msrq->max_gs; 2076 if (ctx->cqe_version) { 2077 uidx = mlx5_store_uidx(ctx, msrq); 2078 if (uidx < 0) { 2079 mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n"); 2080 goto err_free_db; 2081 } 2082 cmd.uidx = uidx; 2083 } else { 2084 cmd.uidx = 0xffffff; 2085 pthread_mutex_lock(&ctx->srq_table_mutex); 2086 } 2087 2088 err = ibv_cmd_create_srq_ex(context, &msrq->vsrq, sizeof(msrq->vsrq), 2089 attr, &cmd.ibv_cmd, sizeof(cmd), 2090 &resp.ibv_resp, sizeof(resp)); 2091 if (err) 2092 goto err_free_uidx; 2093 2094 if (attr->srq_type == IBV_SRQT_TM) { 2095 msrq->cmd_qp = create_cmd_qp(context, attr, ibsrq); 2096 if (!msrq->cmd_qp) 2097 goto err_destroy; 2098 } 2099 2100 if (!ctx->cqe_version) { 2101 err = mlx5_store_srq(to_mctx(context), resp.srqn, msrq); 2102 if (err) 2103 goto err_free_cmd; 2104 2105 pthread_mutex_unlock(&ctx->srq_table_mutex); 2106 } 2107 2108 msrq->srqn = resp.srqn; 2109 msrq->rsc.type = MLX5_RSC_TYPE_XSRQ; 2110 msrq->rsc.rsn = ctx->cqe_version ? cmd.uidx : resp.srqn; 2111 2112 return ibsrq; 2113 2114 err_free_cmd: 2115 if (msrq->cmd_qp) 2116 mlx5_destroy_qp(msrq->cmd_qp); 2117 err_destroy: 2118 ibv_cmd_destroy_srq(ibsrq); 2119 2120 err_free_uidx: 2121 if (ctx->cqe_version) 2122 mlx5_clear_uidx(ctx, cmd.uidx); 2123 else 2124 pthread_mutex_unlock(&ctx->srq_table_mutex); 2125 2126 err_free_db: 2127 mlx5_free_db(ctx, msrq->db); 2128 2129 err_free: 2130 free(msrq->wrid); 2131 mlx5_free_buf(&msrq->buf); 2132 2133 err_spl: 2134 mlx5_spinlock_destroy(&msrq->lock); 2135 2136 err: 2137 free(msrq); 2138 2139 return NULL; 2140 } 2141 2142 int mlx5_query_device_ex(struct ibv_context *context, 2143 const struct ibv_query_device_ex_input *input, 2144 struct ibv_device_attr_ex *attr, 2145 size_t attr_size) 2146 { 2147 struct mlx5_context *mctx = to_mctx(context); 2148 struct mlx5_query_device_ex_resp resp; 2149 struct mlx5_query_device_ex cmd; 2150 struct ibv_device_attr *a; 2151 uint64_t raw_fw_ver; 2152 unsigned sub_minor; 2153 unsigned major; 2154 unsigned minor; 2155 int err; 2156 int cmd_supp_uhw = mctx->cmds_supp_uhw & 2157 MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE; 2158 2159 memset(&cmd, 0, sizeof(cmd)); 2160 memset(&resp, 0, sizeof(resp)); 2161 err = ibv_cmd_query_device_ex(context, input, attr, attr_size, 2162 &raw_fw_ver, 2163 &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), 2164 &resp.ibv_resp, sizeof(resp.ibv_resp), 2165 cmd_supp_uhw ? sizeof(resp) : sizeof(resp.ibv_resp)); 2166 if (err) 2167 return err; 2168 2169 attr->tso_caps = resp.tso_caps; 2170 attr->rss_caps.rx_hash_fields_mask = resp.rss_caps.rx_hash_fields_mask; 2171 attr->rss_caps.rx_hash_function = resp.rss_caps.rx_hash_function; 2172 attr->packet_pacing_caps = resp.packet_pacing_caps.caps; 2173 2174 /* 2175 * The kernel reports the raw two-bit 'multi_pkt_send_wqe' HW 2176 * capability (0 = unsupported, 1 = MPW allowed, 3 = enhanced MPW), 2177 * not the MLX5_ALLOW_MPW / MLX5_SUPPORT_EMPW bitmask that upstream 2178 * libmlx5 expects. Interpret the raw value so MPW is not silently 2179 * disabled on kernels that report the legacy encoding. 2180 */ 2181 if (resp.support_multi_pkt_send_wqe) 2182 mctx->vendor_cap_flags |= MLX5_VENDOR_CAP_FLAGS_MPW_ALLOWED; 2183 2184 if (resp.support_multi_pkt_send_wqe == 3) 2185 mctx->vendor_cap_flags |= MLX5_VENDOR_CAP_FLAGS_ENHANCED_MPW; 2186 2187 mctx->cqe_comp_caps = resp.cqe_comp_caps; 2188 mctx->sw_parsing_caps = resp.sw_parsing_caps; 2189 mctx->striding_rq_caps = resp.striding_rq_caps.caps; 2190 mctx->tunnel_offloads_caps = resp.tunnel_offloads_caps; 2191 2192 major = (raw_fw_ver >> 32) & 0xffff; 2193 minor = (raw_fw_ver >> 16) & 0xffff; 2194 sub_minor = raw_fw_ver & 0xffff; 2195 a = &attr->orig_attr; 2196 snprintf(a->fw_ver, sizeof(a->fw_ver), "%d.%d.%04d", 2197 major, minor, sub_minor); 2198 2199 return 0; 2200 } 2201 2202 static int rwq_sig_enabled(struct ibv_context *context) 2203 { 2204 char *env; 2205 2206 env = getenv("MLX5_RWQ_SIGNATURE"); 2207 if (env) 2208 return 1; 2209 2210 return 0; 2211 } 2212 2213 static void mlx5_free_rwq_buf(struct mlx5_rwq *rwq, struct ibv_context *context) 2214 { 2215 struct mlx5_context *ctx = to_mctx(context); 2216 2217 mlx5_free_actual_buf(ctx, &rwq->buf); 2218 free(rwq->rq.wrid); 2219 } 2220 2221 static int mlx5_alloc_rwq_buf(struct ibv_context *context, 2222 struct mlx5_rwq *rwq, 2223 int size) 2224 { 2225 int err; 2226 enum mlx5_alloc_type alloc_type; 2227 2228 mlx5_get_alloc_type(to_mctx(context), MLX5_RWQ_PREFIX, 2229 &alloc_type, MLX5_ALLOC_TYPE_ANON); 2230 2231 rwq->rq.wrid = malloc(rwq->rq.wqe_cnt * sizeof(uint64_t)); 2232 if (!rwq->rq.wrid) { 2233 errno = ENOMEM; 2234 return -1; 2235 } 2236 2237 err = mlx5_alloc_prefered_buf(to_mctx(context), &rwq->buf, 2238 align(rwq->buf_size, to_mdev 2239 (context->device)->page_size), 2240 to_mdev(context->device)->page_size, 2241 alloc_type, 2242 MLX5_RWQ_PREFIX); 2243 2244 if (err) { 2245 free(rwq->rq.wrid); 2246 errno = ENOMEM; 2247 return -1; 2248 } 2249 2250 return 0; 2251 } 2252 2253 static struct ibv_wq *create_wq(struct ibv_context *context, 2254 struct ibv_wq_init_attr *attr, 2255 struct mlx5dv_wq_init_attr *mlx5wq_attr) 2256 { 2257 struct mlx5_create_wq cmd; 2258 struct mlx5_create_wq_resp resp; 2259 int err; 2260 struct mlx5_rwq *rwq; 2261 struct mlx5_context *ctx = to_mctx(context); 2262 int ret; 2263 int32_t usr_idx = 0; 2264 FILE *fp = ctx->dbg_fp; 2265 2266 if (attr->wq_type != IBV_WQT_RQ) 2267 return NULL; 2268 2269 memset(&cmd, 0, sizeof(cmd)); 2270 memset(&resp, 0, sizeof(resp)); 2271 2272 rwq = calloc(1, sizeof(*rwq)); 2273 if (!rwq) 2274 return NULL; 2275 2276 ret = ibv_init_wq(&rwq->wq); 2277 if (ret < 0) 2278 goto err; 2279 2280 rwq->wq_sig = rwq_sig_enabled(context); 2281 if (rwq->wq_sig) 2282 cmd.drv.flags = MLX5_RWQ_FLAG_SIGNATURE; 2283 2284 ret = mlx5_calc_rwq_size(ctx, rwq, attr, mlx5wq_attr); 2285 if (ret < 0) { 2286 errno = -ret; 2287 goto err_cleanup_wq; 2288 } 2289 2290 rwq->buf_size = ret; 2291 if (mlx5_alloc_rwq_buf(context, rwq, ret)) 2292 goto err_cleanup_wq; 2293 2294 mlx5_init_rwq_indices(rwq); 2295 2296 if (mlx5_spinlock_init(&rwq->rq.lock)) 2297 goto err_free_rwq_buf; 2298 2299 rwq->db = mlx5_alloc_dbrec(ctx); 2300 if (!rwq->db) 2301 goto err_spl; 2302 2303 rwq->db[MLX5_RCV_DBR] = 0; 2304 rwq->db[MLX5_SND_DBR] = 0; 2305 rwq->pbuff = rwq->buf.buf + rwq->rq.offset; 2306 rwq->recv_db = &rwq->db[MLX5_RCV_DBR]; 2307 cmd.drv.buf_addr = (uintptr_t)rwq->buf.buf; 2308 cmd.drv.db_addr = (uintptr_t)rwq->db; 2309 cmd.drv.rq_wqe_count = rwq->rq.wqe_cnt; 2310 cmd.drv.rq_wqe_shift = rwq->rq.wqe_shift; 2311 usr_idx = mlx5_store_uidx(ctx, rwq); 2312 if (usr_idx < 0) { 2313 mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n"); 2314 goto err_free_db_rec; 2315 } 2316 2317 cmd.drv.user_index = usr_idx; 2318 if (mlx5wq_attr) { 2319 if (mlx5wq_attr->comp_mask & MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ) { 2320 if ((mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes < 2321 ctx->striding_rq_caps.min_single_stride_log_num_of_bytes) || 2322 (mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes > 2323 ctx->striding_rq_caps.max_single_stride_log_num_of_bytes)) { 2324 errno = EINVAL; 2325 goto err_create; 2326 } 2327 2328 if ((mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides < 2329 ctx->striding_rq_caps.min_single_wqe_log_num_of_strides) || 2330 (mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides > 2331 ctx->striding_rq_caps.max_single_wqe_log_num_of_strides)) { 2332 errno = EINVAL; 2333 goto err_create; 2334 } 2335 2336 cmd.drv.single_stride_log_num_of_bytes = 2337 mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes; 2338 cmd.drv.single_wqe_log_num_of_strides = 2339 mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides; 2340 cmd.drv.two_byte_shift_en = 2341 mlx5wq_attr->striding_rq_attrs.two_byte_shift_en; 2342 cmd.drv.comp_mask |= MLX5_IB_CREATE_WQ_STRIDING_RQ; 2343 } 2344 } 2345 err = ibv_cmd_create_wq(context, attr, &rwq->wq, &cmd.ibv_cmd, 2346 sizeof(cmd.ibv_cmd), 2347 sizeof(cmd), 2348 &resp.ibv_resp, sizeof(resp.ibv_resp), 2349 sizeof(resp)); 2350 if (err) 2351 goto err_create; 2352 2353 rwq->rsc.type = MLX5_RSC_TYPE_RWQ; 2354 rwq->rsc.rsn = cmd.drv.user_index; 2355 2356 rwq->wq.post_recv = mlx5_post_wq_recv; 2357 return &rwq->wq; 2358 2359 err_create: 2360 mlx5_clear_uidx(ctx, cmd.drv.user_index); 2361 err_free_db_rec: 2362 mlx5_free_db(to_mctx(context), rwq->db); 2363 err_spl: 2364 mlx5_spinlock_destroy(&rwq->rq.lock); 2365 err_free_rwq_buf: 2366 mlx5_free_rwq_buf(rwq, context); 2367 err_cleanup_wq: 2368 ibv_cleanup_wq(&rwq->wq); 2369 err: 2370 free(rwq); 2371 return NULL; 2372 } 2373 2374 struct ibv_wq *mlx5_create_wq(struct ibv_context *context, 2375 struct ibv_wq_init_attr *attr) 2376 { 2377 return create_wq(context, attr, NULL); 2378 } 2379 2380 struct ibv_wq *mlx5dv_create_wq(struct ibv_context *context, 2381 struct ibv_wq_init_attr *attr, 2382 struct mlx5dv_wq_init_attr *mlx5_wq_attr) 2383 { 2384 return create_wq(context, attr, mlx5_wq_attr); 2385 } 2386 2387 int mlx5_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr) 2388 { 2389 struct mlx5_modify_wq cmd = {}; 2390 struct mlx5_rwq *rwq = to_mrwq(wq); 2391 2392 if ((attr->attr_mask & IBV_WQ_ATTR_STATE) && 2393 attr->wq_state == IBV_WQS_RDY) { 2394 if ((attr->attr_mask & IBV_WQ_ATTR_CURR_STATE) && 2395 attr->curr_wq_state != wq->state) 2396 return -EINVAL; 2397 2398 if (wq->state == IBV_WQS_RESET) { 2399 mlx5_spin_lock(&to_mcq(wq->cq)->lock); 2400 __mlx5_cq_clean(to_mcq(wq->cq), 2401 rwq->rsc.rsn, NULL); 2402 mlx5_spin_unlock(&to_mcq(wq->cq)->lock); 2403 mlx5_init_rwq_indices(rwq); 2404 rwq->db[MLX5_RCV_DBR] = 0; 2405 rwq->db[MLX5_SND_DBR] = 0; 2406 } 2407 } 2408 2409 return ibv_cmd_modify_wq(wq, attr, &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd)); 2410 } 2411 2412 int mlx5_destroy_wq(struct ibv_wq *wq) 2413 { 2414 struct mlx5_rwq *rwq = to_mrwq(wq); 2415 int ret; 2416 2417 ret = ibv_cmd_destroy_wq(wq); 2418 if (ret) 2419 return ret; 2420 2421 mlx5_spin_lock(&to_mcq(wq->cq)->lock); 2422 __mlx5_cq_clean(to_mcq(wq->cq), rwq->rsc.rsn, NULL); 2423 mlx5_spin_unlock(&to_mcq(wq->cq)->lock); 2424 mlx5_clear_uidx(to_mctx(wq->context), rwq->rsc.rsn); 2425 mlx5_free_db(to_mctx(wq->context), rwq->db); 2426 mlx5_spinlock_destroy(&rwq->rq.lock); 2427 mlx5_free_rwq_buf(rwq, wq->context); 2428 ibv_cleanup_wq(&rwq->wq); 2429 free(rwq); 2430 2431 return 0; 2432 } 2433 2434 struct ibv_rwq_ind_table *mlx5_create_rwq_ind_table(struct ibv_context *context, 2435 struct ibv_rwq_ind_table_init_attr *init_attr) 2436 { 2437 struct ibv_create_rwq_ind_table *cmd; 2438 struct mlx5_create_rwq_ind_table_resp resp; 2439 struct ibv_rwq_ind_table *ind_table; 2440 uint32_t required_tbl_size; 2441 int num_tbl_entries; 2442 int cmd_size; 2443 int err; 2444 2445 num_tbl_entries = 1 << init_attr->log_ind_tbl_size; 2446 /* Data must be u64 aligned */ 2447 required_tbl_size = (num_tbl_entries * sizeof(uint32_t)) < sizeof(uint64_t) ? 2448 sizeof(uint64_t) : (num_tbl_entries * sizeof(uint32_t)); 2449 2450 cmd_size = required_tbl_size + sizeof(*cmd); 2451 cmd = calloc(1, cmd_size); 2452 if (!cmd) 2453 return NULL; 2454 2455 memset(&resp, 0, sizeof(resp)); 2456 ind_table = calloc(1, sizeof(*ind_table)); 2457 if (!ind_table) 2458 goto free_cmd; 2459 2460 err = ibv_cmd_create_rwq_ind_table(context, init_attr, ind_table, cmd, 2461 cmd_size, cmd_size, &resp.ibv_resp, sizeof(resp.ibv_resp), 2462 sizeof(resp)); 2463 if (err) 2464 goto err; 2465 2466 free(cmd); 2467 return ind_table; 2468 2469 err: 2470 free(ind_table); 2471 free_cmd: 2472 free(cmd); 2473 return NULL; 2474 } 2475 2476 int mlx5_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table) 2477 { 2478 int ret; 2479 2480 ret = ibv_cmd_destroy_rwq_ind_table(rwq_ind_table); 2481 2482 if (ret) 2483 return ret; 2484 2485 free(rwq_ind_table); 2486 return 0; 2487 } 2488