1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <linux/platform_device.h> 36 #include <rdma/ib_addr.h> 37 #include <rdma/ib_umem.h> 38 #include <rdma/uverbs_ioctl.h> 39 #include "hns_roce_common.h" 40 #include "hns_roce_device.h" 41 #include "hns_roce_hem.h" 42 #include <rdma/hns-abi.h> 43 44 #define SQP_NUM (2 * HNS_ROCE_MAX_PORTS) 45 46 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 47 { 48 struct device *dev = hr_dev->dev; 49 struct hns_roce_qp *qp; 50 51 xa_lock(&hr_dev->qp_table_xa); 52 qp = __hns_roce_qp_lookup(hr_dev, qpn); 53 if (qp) 54 atomic_inc(&qp->refcount); 55 xa_unlock(&hr_dev->qp_table_xa); 56 57 if (!qp) { 58 dev_warn(dev, "Async event for bogus QP %08x\n", qpn); 59 return; 60 } 61 62 qp->event(qp, (enum hns_roce_event)event_type); 63 64 if (atomic_dec_and_test(&qp->refcount)) 65 complete(&qp->free); 66 } 67 68 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 69 enum hns_roce_event type) 70 { 71 struct ib_event event; 72 struct ib_qp *ibqp = &hr_qp->ibqp; 73 74 if (ibqp->event_handler) { 75 event.device = ibqp->device; 76 event.element.qp = ibqp; 77 switch (type) { 78 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 79 event.event = IB_EVENT_PATH_MIG; 80 break; 81 case HNS_ROCE_EVENT_TYPE_COMM_EST: 82 event.event = IB_EVENT_COMM_EST; 83 break; 84 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 85 event.event = IB_EVENT_SQ_DRAINED; 86 break; 87 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 88 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 89 break; 90 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 91 event.event = IB_EVENT_QP_FATAL; 92 break; 93 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 94 event.event = IB_EVENT_PATH_MIG_ERR; 95 break; 96 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 97 event.event = IB_EVENT_QP_REQ_ERR; 98 break; 99 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 100 event.event = IB_EVENT_QP_ACCESS_ERR; 101 break; 102 default: 103 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 104 type, hr_qp->qpn); 105 return; 106 } 107 ibqp->event_handler(&event, ibqp->qp_context); 108 } 109 } 110 111 static int hns_roce_reserve_range_qp(struct hns_roce_dev *hr_dev, int cnt, 112 int align, unsigned long *base) 113 { 114 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 115 116 return hns_roce_bitmap_alloc_range(&qp_table->bitmap, cnt, align, 117 base) ? 118 -ENOMEM : 119 0; 120 } 121 122 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) 123 { 124 switch (state) { 125 case IB_QPS_RESET: 126 return HNS_ROCE_QP_STATE_RST; 127 case IB_QPS_INIT: 128 return HNS_ROCE_QP_STATE_INIT; 129 case IB_QPS_RTR: 130 return HNS_ROCE_QP_STATE_RTR; 131 case IB_QPS_RTS: 132 return HNS_ROCE_QP_STATE_RTS; 133 case IB_QPS_SQD: 134 return HNS_ROCE_QP_STATE_SQD; 135 case IB_QPS_ERR: 136 return HNS_ROCE_QP_STATE_ERR; 137 default: 138 return HNS_ROCE_QP_NUM_STATE; 139 } 140 } 141 142 static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, 143 struct hns_roce_qp *hr_qp) 144 { 145 struct xarray *xa = &hr_dev->qp_table_xa; 146 int ret; 147 148 if (!qpn) 149 return -EINVAL; 150 151 hr_qp->qpn = qpn; 152 atomic_set(&hr_qp->refcount, 1); 153 init_completion(&hr_qp->free); 154 155 ret = xa_err(xa_store_irq(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1), 156 hr_qp, GFP_KERNEL)); 157 if (ret) 158 dev_err(hr_dev->dev, "QPC xa_store failed\n"); 159 160 return ret; 161 } 162 163 static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, 164 struct hns_roce_qp *hr_qp) 165 { 166 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 167 struct device *dev = hr_dev->dev; 168 int ret; 169 170 if (!qpn) 171 return -EINVAL; 172 173 hr_qp->qpn = qpn; 174 175 /* Alloc memory for QPC */ 176 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 177 if (ret) { 178 dev_err(dev, "QPC table get failed\n"); 179 goto err_out; 180 } 181 182 /* Alloc memory for IRRL */ 183 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 184 if (ret) { 185 dev_err(dev, "IRRL table get failed\n"); 186 goto err_put_qp; 187 } 188 189 if (hr_dev->caps.trrl_entry_sz) { 190 /* Alloc memory for TRRL */ 191 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 192 hr_qp->qpn); 193 if (ret) { 194 dev_err(dev, "TRRL table get failed\n"); 195 goto err_put_irrl; 196 } 197 } 198 199 if (hr_dev->caps.sccc_entry_sz) { 200 /* Alloc memory for SCC CTX */ 201 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 202 hr_qp->qpn); 203 if (ret) { 204 dev_err(dev, "SCC CTX table get failed\n"); 205 goto err_put_trrl; 206 } 207 } 208 209 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); 210 if (ret) 211 goto err_put_sccc; 212 213 return 0; 214 215 err_put_sccc: 216 if (hr_dev->caps.sccc_entry_sz) 217 hns_roce_table_put(hr_dev, &qp_table->sccc_table, 218 hr_qp->qpn); 219 220 err_put_trrl: 221 if (hr_dev->caps.trrl_entry_sz) 222 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 223 224 err_put_irrl: 225 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 226 227 err_put_qp: 228 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 229 230 err_out: 231 return ret; 232 } 233 234 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 235 { 236 struct xarray *xa = &hr_dev->qp_table_xa; 237 unsigned long flags; 238 239 xa_lock_irqsave(xa, flags); 240 __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); 241 xa_unlock_irqrestore(xa, flags); 242 } 243 244 void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 245 { 246 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 247 248 if (atomic_dec_and_test(&hr_qp->refcount)) 249 complete(&hr_qp->free); 250 wait_for_completion(&hr_qp->free); 251 252 if ((hr_qp->ibqp.qp_type) != IB_QPT_GSI) { 253 if (hr_dev->caps.trrl_entry_sz) 254 hns_roce_table_put(hr_dev, &qp_table->trrl_table, 255 hr_qp->qpn); 256 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 257 } 258 } 259 260 void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, 261 int cnt) 262 { 263 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 264 265 if (base_qpn < hr_dev->caps.reserved_qps) 266 return; 267 268 hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR); 269 } 270 271 static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, 272 struct ib_qp_cap *cap, bool is_user, int has_rq, 273 struct hns_roce_qp *hr_qp) 274 { 275 struct device *dev = hr_dev->dev; 276 u32 max_cnt; 277 278 /* Check the validity of QP support capacity */ 279 if (cap->max_recv_wr > hr_dev->caps.max_wqes || 280 cap->max_recv_sge > hr_dev->caps.max_rq_sg) { 281 dev_err(dev, "RQ WR or sge error!max_recv_wr=%d max_recv_sge=%d\n", 282 cap->max_recv_wr, cap->max_recv_sge); 283 return -EINVAL; 284 } 285 286 /* If srq exist, set zero for relative number of rq */ 287 if (!has_rq) { 288 hr_qp->rq.wqe_cnt = 0; 289 hr_qp->rq.max_gs = 0; 290 cap->max_recv_wr = 0; 291 cap->max_recv_sge = 0; 292 } else { 293 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) { 294 dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n"); 295 return -EINVAL; 296 } 297 298 if (hr_dev->caps.min_wqes) 299 max_cnt = max(cap->max_recv_wr, hr_dev->caps.min_wqes); 300 else 301 max_cnt = cap->max_recv_wr; 302 303 hr_qp->rq.wqe_cnt = roundup_pow_of_two(max_cnt); 304 305 if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) { 306 dev_err(dev, "while setting rq size, rq.wqe_cnt too large\n"); 307 return -EINVAL; 308 } 309 310 max_cnt = max(1U, cap->max_recv_sge); 311 hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt); 312 if (hr_dev->caps.max_rq_sg <= 2) 313 hr_qp->rq.wqe_shift = 314 ilog2(hr_dev->caps.max_rq_desc_sz); 315 else 316 hr_qp->rq.wqe_shift = 317 ilog2(hr_dev->caps.max_rq_desc_sz 318 * hr_qp->rq.max_gs); 319 } 320 321 cap->max_recv_wr = hr_qp->rq.max_post = hr_qp->rq.wqe_cnt; 322 cap->max_recv_sge = hr_qp->rq.max_gs; 323 324 return 0; 325 } 326 327 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 328 struct ib_qp_cap *cap, 329 struct hns_roce_ib_create_qp *ucmd) 330 { 331 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 332 u8 max_sq_stride = ilog2(roundup_sq_stride); 333 334 /* Sanity check SQ size before proceeding */ 335 if ((u32)(1 << ucmd->log_sq_bb_count) > hr_dev->caps.max_wqes || 336 ucmd->log_sq_stride > max_sq_stride || 337 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 338 ibdev_err(&hr_dev->ib_dev, "check SQ size error!\n"); 339 return -EINVAL; 340 } 341 342 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 343 ibdev_err(&hr_dev->ib_dev, "SQ sge error! max_send_sge=%d\n", 344 cap->max_send_sge); 345 return -EINVAL; 346 } 347 348 return 0; 349 } 350 351 static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, 352 struct ib_qp_cap *cap, 353 struct hns_roce_qp *hr_qp, 354 struct hns_roce_ib_create_qp *ucmd) 355 { 356 u32 ex_sge_num; 357 u32 page_size; 358 u32 max_cnt; 359 int ret; 360 361 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 362 if (ret) { 363 ibdev_err(&hr_dev->ib_dev, "Sanity check sq size failed\n"); 364 return ret; 365 } 366 367 hr_qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count; 368 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 369 370 max_cnt = max(1U, cap->max_send_sge); 371 if (hr_dev->caps.max_sq_sg <= 2) 372 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); 373 else 374 hr_qp->sq.max_gs = max_cnt; 375 376 if (hr_qp->sq.max_gs > 2) 377 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 378 (hr_qp->sq.max_gs - 2)); 379 380 if ((hr_qp->sq.max_gs > 2) && (hr_dev->pci_dev->revision == 0x20)) { 381 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) { 382 dev_err(hr_dev->dev, 383 "The extended sge cnt error! sge_cnt=%d\n", 384 hr_qp->sge.sge_cnt); 385 return -EINVAL; 386 } 387 } 388 389 hr_qp->sge.sge_shift = 4; 390 ex_sge_num = hr_qp->sge.sge_cnt; 391 392 /* Get buf size, SQ and RQ are aligned to page_szie */ 393 if (hr_dev->caps.max_sq_sg <= 2) { 394 hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << 395 hr_qp->rq.wqe_shift), PAGE_SIZE) + 396 HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << 397 hr_qp->sq.wqe_shift), PAGE_SIZE); 398 399 hr_qp->sq.offset = 0; 400 hr_qp->rq.offset = HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << 401 hr_qp->sq.wqe_shift), PAGE_SIZE); 402 } else { 403 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 404 hr_qp->sge.sge_cnt = ex_sge_num ? 405 max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0; 406 hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << 407 hr_qp->rq.wqe_shift), page_size) + 408 HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt << 409 hr_qp->sge.sge_shift), page_size) + 410 HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt << 411 hr_qp->sq.wqe_shift), page_size); 412 413 hr_qp->sq.offset = 0; 414 if (ex_sge_num) { 415 hr_qp->sge.offset = HNS_ROCE_ALOGN_UP( 416 (hr_qp->sq.wqe_cnt << 417 hr_qp->sq.wqe_shift), 418 page_size); 419 hr_qp->rq.offset = hr_qp->sge.offset + 420 HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt << 421 hr_qp->sge.sge_shift), 422 page_size); 423 } else { 424 hr_qp->rq.offset = HNS_ROCE_ALOGN_UP( 425 (hr_qp->sq.wqe_cnt << 426 hr_qp->sq.wqe_shift), 427 page_size); 428 } 429 } 430 431 return 0; 432 } 433 434 static int split_wqe_buf_region(struct hns_roce_dev *hr_dev, 435 struct hns_roce_qp *hr_qp, 436 struct hns_roce_buf_region *regions, 437 int region_max, int page_shift) 438 { 439 int page_size = 1 << page_shift; 440 bool is_extend_sge; 441 int region_cnt = 0; 442 int buf_size; 443 int buf_cnt; 444 445 if (hr_qp->buff_size < 1 || region_max < 1) 446 return region_cnt; 447 448 if (hr_qp->sge.sge_cnt > 0) 449 is_extend_sge = true; 450 else 451 is_extend_sge = false; 452 453 /* sq region */ 454 if (is_extend_sge) 455 buf_size = hr_qp->sge.offset - hr_qp->sq.offset; 456 else 457 buf_size = hr_qp->rq.offset - hr_qp->sq.offset; 458 459 if (buf_size > 0 && region_cnt < region_max) { 460 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 461 hns_roce_init_buf_region(®ions[region_cnt], 462 hr_dev->caps.wqe_sq_hop_num, 463 hr_qp->sq.offset / page_size, 464 buf_cnt); 465 region_cnt++; 466 } 467 468 /* sge region */ 469 if (is_extend_sge) { 470 buf_size = hr_qp->rq.offset - hr_qp->sge.offset; 471 if (buf_size > 0 && region_cnt < region_max) { 472 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 473 hns_roce_init_buf_region(®ions[region_cnt], 474 hr_dev->caps.wqe_sge_hop_num, 475 hr_qp->sge.offset / page_size, 476 buf_cnt); 477 region_cnt++; 478 } 479 } 480 481 /* rq region */ 482 buf_size = hr_qp->buff_size - hr_qp->rq.offset; 483 if (buf_size > 0) { 484 buf_cnt = DIV_ROUND_UP(buf_size, page_size); 485 hns_roce_init_buf_region(®ions[region_cnt], 486 hr_dev->caps.wqe_rq_hop_num, 487 hr_qp->rq.offset / page_size, 488 buf_cnt); 489 region_cnt++; 490 } 491 492 return region_cnt; 493 } 494 495 static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev, 496 struct hns_roce_buf_region *regions, 497 int region_cnt) 498 { 499 int bt_pg_shift; 500 int ba_num; 501 int ret; 502 503 bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz; 504 505 /* all root ba entries must in one bt page */ 506 do { 507 ba_num = (1 << bt_pg_shift) / BA_BYTE_LEN; 508 ret = hns_roce_hem_list_calc_root_ba(regions, region_cnt, 509 ba_num); 510 if (ret <= ba_num) 511 break; 512 513 bt_pg_shift++; 514 } while (ret > ba_num); 515 516 return bt_pg_shift - PAGE_SHIFT; 517 } 518 519 static int set_extend_sge_param(struct hns_roce_dev *hr_dev, 520 struct hns_roce_qp *hr_qp) 521 { 522 struct device *dev = hr_dev->dev; 523 524 if (hr_qp->sq.max_gs > 2) { 525 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 526 (hr_qp->sq.max_gs - 2)); 527 hr_qp->sge.sge_shift = 4; 528 } 529 530 /* ud sqwqe's sge use extend sge */ 531 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->ibqp.qp_type == IB_QPT_GSI) { 532 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt * 533 hr_qp->sq.max_gs); 534 hr_qp->sge.sge_shift = 4; 535 } 536 537 if ((hr_qp->sq.max_gs > 2) && hr_dev->pci_dev->revision == 0x20) { 538 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) { 539 dev_err(dev, "The extended sge cnt error! sge_cnt=%d\n", 540 hr_qp->sge.sge_cnt); 541 return -EINVAL; 542 } 543 } 544 545 return 0; 546 } 547 548 static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, 549 struct ib_qp_cap *cap, 550 struct hns_roce_qp *hr_qp) 551 { 552 struct device *dev = hr_dev->dev; 553 u32 page_size; 554 u32 max_cnt; 555 int size; 556 int ret; 557 558 if (cap->max_send_wr > hr_dev->caps.max_wqes || 559 cap->max_send_sge > hr_dev->caps.max_sq_sg || 560 cap->max_inline_data > hr_dev->caps.max_sq_inline) { 561 dev_err(dev, "SQ WR or sge or inline data error!\n"); 562 return -EINVAL; 563 } 564 565 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 566 567 if (hr_dev->caps.min_wqes) 568 max_cnt = max(cap->max_send_wr, hr_dev->caps.min_wqes); 569 else 570 max_cnt = cap->max_send_wr; 571 572 hr_qp->sq.wqe_cnt = roundup_pow_of_two(max_cnt); 573 if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) { 574 dev_err(dev, "while setting kernel sq size, sq.wqe_cnt too large\n"); 575 return -EINVAL; 576 } 577 578 /* Get data_seg numbers */ 579 max_cnt = max(1U, cap->max_send_sge); 580 if (hr_dev->caps.max_sq_sg <= 2) 581 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt); 582 else 583 hr_qp->sq.max_gs = max_cnt; 584 585 ret = set_extend_sge_param(hr_dev, hr_qp); 586 if (ret) { 587 dev_err(dev, "set extend sge parameters fail\n"); 588 return ret; 589 } 590 591 /* Get buf size, SQ and RQ are aligned to PAGE_SIZE */ 592 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); 593 hr_qp->sq.offset = 0; 594 size = HNS_ROCE_ALOGN_UP(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift, 595 page_size); 596 597 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->sge.sge_cnt) { 598 hr_qp->sge.sge_cnt = max(page_size/(1 << hr_qp->sge.sge_shift), 599 (u32)hr_qp->sge.sge_cnt); 600 hr_qp->sge.offset = size; 601 size += HNS_ROCE_ALOGN_UP(hr_qp->sge.sge_cnt << 602 hr_qp->sge.sge_shift, page_size); 603 } 604 605 hr_qp->rq.offset = size; 606 size += HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), 607 page_size); 608 hr_qp->buff_size = size; 609 610 /* Get wr and sge number which send */ 611 cap->max_send_wr = hr_qp->sq.max_post = hr_qp->sq.wqe_cnt; 612 cap->max_send_sge = hr_qp->sq.max_gs; 613 614 /* We don't support inline sends for kernel QPs (yet) */ 615 cap->max_inline_data = 0; 616 617 return 0; 618 } 619 620 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 621 { 622 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 623 return 0; 624 625 return 1; 626 } 627 628 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 629 { 630 if (attr->qp_type == IB_QPT_XRC_INI || 631 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 632 !attr->cap.max_recv_wr) 633 return 0; 634 635 return 1; 636 } 637 638 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 639 struct ib_qp_init_attr *init_attr) 640 { 641 u32 max_recv_sge = init_attr->cap.max_recv_sge; 642 struct hns_roce_rinl_wqe *wqe_list; 643 u32 wqe_cnt = hr_qp->rq.wqe_cnt; 644 int i; 645 646 /* allocate recv inline buf */ 647 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 648 GFP_KERNEL); 649 650 if (!wqe_list) 651 goto err; 652 653 /* Allocate a continuous buffer for all inline sge we need */ 654 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 655 sizeof(struct hns_roce_rinl_sge)), 656 GFP_KERNEL); 657 if (!wqe_list[0].sg_list) 658 goto err_wqe_list; 659 660 /* Assign buffers of sg_list to each inline wqe */ 661 for (i = 1; i < wqe_cnt; i++) 662 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 663 664 hr_qp->rq_inl_buf.wqe_list = wqe_list; 665 hr_qp->rq_inl_buf.wqe_cnt = wqe_cnt; 666 667 return 0; 668 669 err_wqe_list: 670 kfree(wqe_list); 671 672 err: 673 return -ENOMEM; 674 } 675 676 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 677 { 678 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 679 kfree(hr_qp->rq_inl_buf.wqe_list); 680 } 681 682 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 683 struct ib_pd *ib_pd, 684 struct ib_qp_init_attr *init_attr, 685 struct ib_udata *udata, unsigned long sqpn, 686 struct hns_roce_qp *hr_qp) 687 { 688 dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { NULL }; 689 struct device *dev = hr_dev->dev; 690 struct hns_roce_ib_create_qp ucmd; 691 struct hns_roce_ib_create_qp_resp resp = {}; 692 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 693 udata, struct hns_roce_ucontext, ibucontext); 694 struct hns_roce_buf_region *r; 695 unsigned long qpn = 0; 696 u32 page_shift; 697 int buf_count; 698 int ret; 699 int i; 700 701 mutex_init(&hr_qp->mutex); 702 spin_lock_init(&hr_qp->sq.lock); 703 spin_lock_init(&hr_qp->rq.lock); 704 705 hr_qp->state = IB_QPS_RESET; 706 707 hr_qp->ibqp.qp_type = init_attr->qp_type; 708 709 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 710 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 711 else 712 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 713 714 ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, udata, 715 hns_roce_qp_has_rq(init_attr), hr_qp); 716 if (ret) { 717 dev_err(dev, "hns_roce_set_rq_size failed\n"); 718 goto err_out; 719 } 720 721 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && 722 hns_roce_qp_has_rq(init_attr)) { 723 ret = alloc_rq_inline_buf(hr_qp, init_attr); 724 if (ret) { 725 dev_err(dev, "allocate receive inline buffer failed\n"); 726 goto err_out; 727 } 728 } 729 730 page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 731 if (udata) { 732 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { 733 dev_err(dev, "ib_copy_from_udata error for create qp\n"); 734 ret = -EFAULT; 735 goto err_alloc_rq_inline_buf; 736 } 737 738 ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, 739 &ucmd); 740 if (ret) { 741 dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n"); 742 goto err_alloc_rq_inline_buf; 743 } 744 745 hr_qp->umem = ib_umem_get(udata, ucmd.buf_addr, 746 hr_qp->buff_size, 0, 0); 747 if (IS_ERR(hr_qp->umem)) { 748 dev_err(dev, "ib_umem_get error for create qp\n"); 749 ret = PTR_ERR(hr_qp->umem); 750 goto err_alloc_rq_inline_buf; 751 } 752 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, 753 hr_qp->regions, ARRAY_SIZE(hr_qp->regions), 754 page_shift); 755 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, 756 hr_qp->region_cnt); 757 if (ret) { 758 dev_err(dev, "alloc buf_list error for create qp\n"); 759 goto err_alloc_list; 760 } 761 762 for (i = 0; i < hr_qp->region_cnt; i++) { 763 r = &hr_qp->regions[i]; 764 buf_count = hns_roce_get_umem_bufs(hr_dev, 765 buf_list[i], r->count, r->offset, 766 hr_qp->umem, page_shift); 767 if (buf_count != r->count) { 768 dev_err(dev, 769 "get umem buf err, expect %d,ret %d.\n", 770 r->count, buf_count); 771 ret = -ENOBUFS; 772 goto err_get_bufs; 773 } 774 } 775 776 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && 777 (udata->inlen >= sizeof(ucmd)) && 778 (udata->outlen >= sizeof(resp)) && 779 hns_roce_qp_has_sq(init_attr)) { 780 ret = hns_roce_db_map_user(uctx, udata, ucmd.sdb_addr, 781 &hr_qp->sdb); 782 if (ret) { 783 dev_err(dev, "sq record doorbell map failed!\n"); 784 goto err_get_bufs; 785 } 786 787 /* indicate kernel supports sq record db */ 788 resp.cap_flags |= HNS_ROCE_SUPPORT_SQ_RECORD_DB; 789 hr_qp->sdb_en = 1; 790 } 791 792 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 793 (udata->outlen >= sizeof(resp)) && 794 hns_roce_qp_has_rq(init_attr)) { 795 ret = hns_roce_db_map_user(uctx, udata, ucmd.db_addr, 796 &hr_qp->rdb); 797 if (ret) { 798 dev_err(dev, "rq record doorbell map failed!\n"); 799 goto err_sq_dbmap; 800 } 801 802 /* indicate kernel supports rq record db */ 803 resp.cap_flags |= HNS_ROCE_SUPPORT_RQ_RECORD_DB; 804 hr_qp->rdb_en = 1; 805 } 806 } else { 807 if (init_attr->create_flags & 808 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) { 809 dev_err(dev, "init_attr->create_flags error!\n"); 810 ret = -EINVAL; 811 goto err_alloc_rq_inline_buf; 812 } 813 814 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) { 815 dev_err(dev, "init_attr->create_flags error!\n"); 816 ret = -EINVAL; 817 goto err_alloc_rq_inline_buf; 818 } 819 820 /* Set SQ size */ 821 ret = hns_roce_set_kernel_sq_size(hr_dev, &init_attr->cap, 822 hr_qp); 823 if (ret) { 824 dev_err(dev, "hns_roce_set_kernel_sq_size error!\n"); 825 goto err_alloc_rq_inline_buf; 826 } 827 828 /* QP doorbell register address */ 829 hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset + 830 DB_REG_OFFSET * hr_dev->priv_uar.index; 831 hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset + 832 DB_REG_OFFSET * hr_dev->priv_uar.index; 833 834 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 835 hns_roce_qp_has_rq(init_attr)) { 836 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 837 if (ret) { 838 dev_err(dev, "rq record doorbell alloc failed!\n"); 839 goto err_alloc_rq_inline_buf; 840 } 841 *hr_qp->rdb.db_record = 0; 842 hr_qp->rdb_en = 1; 843 } 844 845 /* Allocate QP buf */ 846 if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, 847 (1 << page_shift) * 2, 848 &hr_qp->hr_buf, page_shift)) { 849 dev_err(dev, "hns_roce_buf_alloc error!\n"); 850 ret = -ENOMEM; 851 goto err_db; 852 } 853 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, 854 hr_qp->regions, ARRAY_SIZE(hr_qp->regions), 855 page_shift); 856 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, 857 hr_qp->region_cnt); 858 if (ret) { 859 dev_err(dev, "alloc buf_list error for create qp!\n"); 860 goto err_alloc_list; 861 } 862 863 for (i = 0; i < hr_qp->region_cnt; i++) { 864 r = &hr_qp->regions[i]; 865 buf_count = hns_roce_get_kmem_bufs(hr_dev, 866 buf_list[i], r->count, r->offset, 867 &hr_qp->hr_buf); 868 if (buf_count != r->count) { 869 dev_err(dev, 870 "get kmem buf err, expect %d,ret %d.\n", 871 r->count, buf_count); 872 ret = -ENOBUFS; 873 goto err_get_bufs; 874 } 875 } 876 877 hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), 878 GFP_KERNEL); 879 if (ZERO_OR_NULL_PTR(hr_qp->sq.wrid)) { 880 ret = -ENOMEM; 881 goto err_get_bufs; 882 } 883 884 if (hr_qp->rq.wqe_cnt) { 885 hr_qp->rq.wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), 886 GFP_KERNEL); 887 if (ZERO_OR_NULL_PTR(hr_qp->rq.wrid)) { 888 ret = -ENOMEM; 889 goto err_sq_wrid; 890 } 891 } 892 } 893 894 if (sqpn) { 895 qpn = sqpn; 896 } else { 897 /* Get QPN */ 898 ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &qpn); 899 if (ret) { 900 dev_err(dev, "hns_roce_reserve_range_qp alloc qpn error\n"); 901 goto err_wrid; 902 } 903 } 904 905 hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions, 906 hr_qp->region_cnt); 907 hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift, 908 page_shift); 909 ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, 910 hr_qp->regions, hr_qp->region_cnt); 911 if (ret) { 912 dev_err(dev, "mtr attach error for create qp\n"); 913 goto err_mtr; 914 } 915 916 if (init_attr->qp_type == IB_QPT_GSI && 917 hr_dev->hw_rev == HNS_ROCE_HW_VER1) { 918 /* In v1 engine, GSI QP context in RoCE engine's register */ 919 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp); 920 if (ret) { 921 dev_err(dev, "hns_roce_qp_alloc failed!\n"); 922 goto err_qpn; 923 } 924 } else { 925 ret = hns_roce_qp_alloc(hr_dev, qpn, hr_qp); 926 if (ret) { 927 dev_err(dev, "hns_roce_qp_alloc failed!\n"); 928 goto err_qpn; 929 } 930 } 931 932 if (sqpn) 933 hr_qp->doorbell_qpn = 1; 934 else 935 hr_qp->doorbell_qpn = (u32)hr_qp->qpn; 936 937 if (udata) { 938 ret = ib_copy_to_udata(udata, &resp, 939 min(udata->outlen, sizeof(resp))); 940 if (ret) 941 goto err_qp; 942 } 943 944 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 945 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 946 if (ret) 947 goto err_qp; 948 } 949 950 hr_qp->event = hns_roce_ib_qp_event; 951 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); 952 953 return 0; 954 955 err_qp: 956 if (init_attr->qp_type == IB_QPT_GSI && 957 hr_dev->hw_rev == HNS_ROCE_HW_VER1) 958 hns_roce_qp_remove(hr_dev, hr_qp); 959 else 960 hns_roce_qp_free(hr_dev, hr_qp); 961 962 err_qpn: 963 if (!sqpn) 964 hns_roce_release_range_qp(hr_dev, qpn, 1); 965 966 err_mtr: 967 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); 968 969 err_wrid: 970 if (udata) { 971 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && 972 (udata->outlen >= sizeof(resp)) && 973 hns_roce_qp_has_rq(init_attr)) 974 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 975 } else { 976 if (hr_qp->rq.wqe_cnt) 977 kfree(hr_qp->rq.wrid); 978 } 979 980 err_sq_dbmap: 981 if (udata) 982 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && 983 (udata->inlen >= sizeof(ucmd)) && 984 (udata->outlen >= sizeof(resp)) && 985 hns_roce_qp_has_sq(init_attr)) 986 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 987 988 err_sq_wrid: 989 if (!udata) 990 kfree(hr_qp->sq.wrid); 991 992 err_get_bufs: 993 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); 994 995 err_alloc_list: 996 if (!hr_qp->umem) 997 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); 998 ib_umem_release(hr_qp->umem); 999 1000 err_db: 1001 if (!udata && hns_roce_qp_has_rq(init_attr) && 1002 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)) 1003 hns_roce_free_db(hr_dev, &hr_qp->rdb); 1004 1005 err_alloc_rq_inline_buf: 1006 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && 1007 hns_roce_qp_has_rq(init_attr)) 1008 free_rq_inline_buf(hr_qp); 1009 1010 err_out: 1011 return ret; 1012 } 1013 1014 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, 1015 struct ib_qp_init_attr *init_attr, 1016 struct ib_udata *udata) 1017 { 1018 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); 1019 struct ib_device *ibdev = &hr_dev->ib_dev; 1020 struct hns_roce_sqp *hr_sqp; 1021 struct hns_roce_qp *hr_qp; 1022 int ret; 1023 1024 switch (init_attr->qp_type) { 1025 case IB_QPT_RC: { 1026 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 1027 if (!hr_qp) 1028 return ERR_PTR(-ENOMEM); 1029 1030 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0, 1031 hr_qp); 1032 if (ret) { 1033 ibdev_err(ibdev, "Create RC QP 0x%06lx failed(%d)\n", 1034 hr_qp->qpn, ret); 1035 kfree(hr_qp); 1036 return ERR_PTR(ret); 1037 } 1038 1039 hr_qp->ibqp.qp_num = hr_qp->qpn; 1040 1041 break; 1042 } 1043 case IB_QPT_GSI: { 1044 /* Userspace is not allowed to create special QPs: */ 1045 if (udata) { 1046 ibdev_err(ibdev, "not support usr space GSI\n"); 1047 return ERR_PTR(-EINVAL); 1048 } 1049 1050 hr_sqp = kzalloc(sizeof(*hr_sqp), GFP_KERNEL); 1051 if (!hr_sqp) 1052 return ERR_PTR(-ENOMEM); 1053 1054 hr_qp = &hr_sqp->hr_qp; 1055 hr_qp->port = init_attr->port_num - 1; 1056 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1057 1058 /* when hw version is v1, the sqpn is allocated */ 1059 if (hr_dev->caps.max_sq_sg <= 2) 1060 hr_qp->ibqp.qp_num = HNS_ROCE_MAX_PORTS + 1061 hr_dev->iboe.phy_port[hr_qp->port]; 1062 else 1063 hr_qp->ibqp.qp_num = 1; 1064 1065 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 1066 hr_qp->ibqp.qp_num, hr_qp); 1067 if (ret) { 1068 ibdev_err(ibdev, "Create GSI QP failed!\n"); 1069 kfree(hr_sqp); 1070 return ERR_PTR(ret); 1071 } 1072 1073 break; 1074 } 1075 default:{ 1076 ibdev_err(ibdev, "not support QP type %d\n", 1077 init_attr->qp_type); 1078 return ERR_PTR(-EINVAL); 1079 } 1080 } 1081 1082 return &hr_qp->ibqp; 1083 } 1084 1085 int to_hr_qp_type(int qp_type) 1086 { 1087 int transport_type; 1088 1089 if (qp_type == IB_QPT_RC) 1090 transport_type = SERV_TYPE_RC; 1091 else if (qp_type == IB_QPT_UC) 1092 transport_type = SERV_TYPE_UC; 1093 else if (qp_type == IB_QPT_UD) 1094 transport_type = SERV_TYPE_UD; 1095 else if (qp_type == IB_QPT_GSI) 1096 transport_type = SERV_TYPE_UD; 1097 else 1098 transport_type = -1; 1099 1100 return transport_type; 1101 } 1102 1103 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1104 struct hns_roce_qp *hr_qp, 1105 struct ib_qp_attr *attr, int attr_mask) 1106 { 1107 enum ib_mtu active_mtu; 1108 int p; 1109 1110 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1111 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1112 1113 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1114 attr->path_mtu > hr_dev->caps.max_mtu) || 1115 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1116 ibdev_err(&hr_dev->ib_dev, 1117 "attr path_mtu(%d)invalid while modify qp", 1118 attr->path_mtu); 1119 return -EINVAL; 1120 } 1121 1122 return 0; 1123 } 1124 1125 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1126 int attr_mask) 1127 { 1128 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1129 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1130 int p; 1131 1132 if ((attr_mask & IB_QP_PORT) && 1133 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1134 ibdev_err(&hr_dev->ib_dev, 1135 "attr port_num invalid.attr->port_num=%d\n", 1136 attr->port_num); 1137 return -EINVAL; 1138 } 1139 1140 if (attr_mask & IB_QP_PKEY_INDEX) { 1141 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1142 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1143 ibdev_err(&hr_dev->ib_dev, 1144 "attr pkey_index invalid.attr->pkey_index=%d\n", 1145 attr->pkey_index); 1146 return -EINVAL; 1147 } 1148 } 1149 1150 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1151 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1152 ibdev_err(&hr_dev->ib_dev, 1153 "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n", 1154 attr->max_rd_atomic); 1155 return -EINVAL; 1156 } 1157 1158 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1159 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1160 ibdev_err(&hr_dev->ib_dev, 1161 "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n", 1162 attr->max_dest_rd_atomic); 1163 return -EINVAL; 1164 } 1165 1166 if (attr_mask & IB_QP_PATH_MTU) 1167 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1168 1169 return 0; 1170 } 1171 1172 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1173 int attr_mask, struct ib_udata *udata) 1174 { 1175 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1176 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1177 enum ib_qp_state cur_state, new_state; 1178 int ret = -EINVAL; 1179 1180 mutex_lock(&hr_qp->mutex); 1181 1182 cur_state = attr_mask & IB_QP_CUR_STATE ? 1183 attr->cur_qp_state : (enum ib_qp_state)hr_qp->state; 1184 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1185 1186 if (ibqp->uobject && 1187 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1188 if (hr_qp->sdb_en == 1) { 1189 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1190 1191 if (hr_qp->rdb_en == 1) 1192 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1193 } else { 1194 ibdev_warn(&hr_dev->ib_dev, 1195 "flush cqe is not supported in userspace!\n"); 1196 goto out; 1197 } 1198 } 1199 1200 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1201 attr_mask)) { 1202 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1203 goto out; 1204 } 1205 1206 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1207 if (ret) 1208 goto out; 1209 1210 if (cur_state == new_state && cur_state == IB_QPS_RESET) { 1211 if (hr_dev->caps.min_wqes) { 1212 ret = -EPERM; 1213 ibdev_err(&hr_dev->ib_dev, 1214 "cur_state=%d new_state=%d\n", cur_state, 1215 new_state); 1216 } else { 1217 ret = 0; 1218 } 1219 1220 goto out; 1221 } 1222 1223 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1224 new_state); 1225 1226 out: 1227 mutex_unlock(&hr_qp->mutex); 1228 1229 return ret; 1230 } 1231 1232 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1233 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1234 { 1235 if (send_cq == recv_cq) { 1236 spin_lock_irq(&send_cq->lock); 1237 __acquire(&recv_cq->lock); 1238 } else if (send_cq->cqn < recv_cq->cqn) { 1239 spin_lock_irq(&send_cq->lock); 1240 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1241 } else { 1242 spin_lock_irq(&recv_cq->lock); 1243 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1244 } 1245 } 1246 1247 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1248 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1249 __releases(&recv_cq->lock) 1250 { 1251 if (send_cq == recv_cq) { 1252 __release(&recv_cq->lock); 1253 spin_unlock_irq(&send_cq->lock); 1254 } else if (send_cq->cqn < recv_cq->cqn) { 1255 spin_unlock(&recv_cq->lock); 1256 spin_unlock_irq(&send_cq->lock); 1257 } else { 1258 spin_unlock(&send_cq->lock); 1259 spin_unlock_irq(&recv_cq->lock); 1260 } 1261 } 1262 1263 static void *get_wqe(struct hns_roce_qp *hr_qp, int offset) 1264 { 1265 1266 return hns_roce_buf_offset(&hr_qp->hr_buf, offset); 1267 } 1268 1269 void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n) 1270 { 1271 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1272 } 1273 1274 void *get_send_wqe(struct hns_roce_qp *hr_qp, int n) 1275 { 1276 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1277 } 1278 1279 void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n) 1280 { 1281 return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset + 1282 (n << hr_qp->sge.sge_shift)); 1283 } 1284 1285 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, 1286 struct ib_cq *ib_cq) 1287 { 1288 struct hns_roce_cq *hr_cq; 1289 u32 cur; 1290 1291 cur = hr_wq->head - hr_wq->tail; 1292 if (likely(cur + nreq < hr_wq->max_post)) 1293 return false; 1294 1295 hr_cq = to_hr_cq(ib_cq); 1296 spin_lock(&hr_cq->lock); 1297 cur = hr_wq->head - hr_wq->tail; 1298 spin_unlock(&hr_cq->lock); 1299 1300 return cur + nreq >= hr_wq->max_post; 1301 } 1302 1303 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1304 { 1305 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1306 int reserved_from_top = 0; 1307 int reserved_from_bot; 1308 int ret; 1309 1310 mutex_init(&qp_table->scc_mutex); 1311 xa_init(&hr_dev->qp_table_xa); 1312 1313 reserved_from_bot = hr_dev->caps.reserved_qps; 1314 1315 ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps, 1316 hr_dev->caps.num_qps - 1, reserved_from_bot, 1317 reserved_from_top); 1318 if (ret) { 1319 dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n", 1320 ret); 1321 return ret; 1322 } 1323 1324 return 0; 1325 } 1326 1327 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1328 { 1329 hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap); 1330 } 1331