1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <rdma/ib_addr.h> 36 #include <rdma/ib_umem.h> 37 #include <rdma/uverbs_ioctl.h> 38 #include "hns_roce_common.h" 39 #include "hns_roce_device.h" 40 #include "hns_roce_hem.h" 41 42 static void flush_work_handle(struct work_struct *work) 43 { 44 struct hns_roce_work *flush_work = container_of(work, 45 struct hns_roce_work, work); 46 struct hns_roce_qp *hr_qp = container_of(flush_work, 47 struct hns_roce_qp, flush_work); 48 struct device *dev = flush_work->hr_dev->dev; 49 struct ib_qp_attr attr; 50 int attr_mask; 51 int ret; 52 53 attr_mask = IB_QP_STATE; 54 attr.qp_state = IB_QPS_ERR; 55 56 if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) { 57 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL); 58 if (ret) 59 dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n", 60 ret); 61 } 62 63 /* 64 * make sure we signal QP destroy leg that flush QP was completed 65 * so that it can safely proceed ahead now and destroy QP 66 */ 67 if (refcount_dec_and_test(&hr_qp->refcount)) 68 complete(&hr_qp->free); 69 } 70 71 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 72 { 73 struct hns_roce_work *flush_work = &hr_qp->flush_work; 74 75 flush_work->hr_dev = hr_dev; 76 INIT_WORK(&flush_work->work, flush_work_handle); 77 refcount_inc(&hr_qp->refcount); 78 queue_work(hr_dev->irq_workq, &flush_work->work); 79 } 80 81 void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp) 82 { 83 /* 84 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state 85 * gets into errored mode. Hence, as a workaround to this 86 * hardware limitation, driver needs to assist in flushing. But 87 * the flushing operation uses mailbox to convey the QP state to 88 * the hardware and which can sleep due to the mutex protection 89 * around the mailbox calls. Hence, use the deferred flush for 90 * now. 91 */ 92 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) 93 init_flush_work(dev, qp); 94 } 95 96 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 97 { 98 struct device *dev = hr_dev->dev; 99 struct hns_roce_qp *qp; 100 101 xa_lock(&hr_dev->qp_table_xa); 102 qp = __hns_roce_qp_lookup(hr_dev, qpn); 103 if (qp) 104 refcount_inc(&qp->refcount); 105 xa_unlock(&hr_dev->qp_table_xa); 106 107 if (!qp) { 108 dev_warn(dev, "Async event for bogus QP %08x\n", qpn); 109 return; 110 } 111 112 if (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || 113 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || 114 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR || 115 event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION || 116 event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH) { 117 qp->state = IB_QPS_ERR; 118 119 flush_cqe(hr_dev, qp); 120 } 121 122 qp->event(qp, (enum hns_roce_event)event_type); 123 124 if (refcount_dec_and_test(&qp->refcount)) 125 complete(&qp->free); 126 } 127 128 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 129 enum hns_roce_event type) 130 { 131 struct ib_qp *ibqp = &hr_qp->ibqp; 132 struct ib_event event; 133 134 if (ibqp->event_handler) { 135 event.device = ibqp->device; 136 event.element.qp = ibqp; 137 switch (type) { 138 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 139 event.event = IB_EVENT_PATH_MIG; 140 break; 141 case HNS_ROCE_EVENT_TYPE_COMM_EST: 142 event.event = IB_EVENT_COMM_EST; 143 break; 144 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 145 event.event = IB_EVENT_SQ_DRAINED; 146 break; 147 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 148 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 149 break; 150 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 151 event.event = IB_EVENT_QP_FATAL; 152 break; 153 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 154 event.event = IB_EVENT_PATH_MIG_ERR; 155 break; 156 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 157 event.event = IB_EVENT_QP_REQ_ERR; 158 break; 159 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 160 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 161 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 162 event.event = IB_EVENT_QP_ACCESS_ERR; 163 break; 164 default: 165 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 166 type, hr_qp->qpn); 167 return; 168 } 169 ibqp->event_handler(&event, ibqp->qp_context); 170 } 171 } 172 173 static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank) 174 { 175 u32 least_load = bank[0].inuse; 176 u8 bankid = 0; 177 u32 bankcnt; 178 u8 i; 179 180 for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) { 181 bankcnt = bank[i].inuse; 182 if (bankcnt < least_load) { 183 least_load = bankcnt; 184 bankid = i; 185 } 186 } 187 188 return bankid; 189 } 190 191 static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid, 192 unsigned long *qpn) 193 { 194 int id; 195 196 id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL); 197 if (id < 0) { 198 id = ida_alloc_range(&bank->ida, bank->min, bank->max, 199 GFP_KERNEL); 200 if (id < 0) 201 return id; 202 } 203 204 /* the QPN should keep increasing until the max value is reached. */ 205 bank->next = (id + 1) > bank->max ? bank->min : id + 1; 206 207 /* the lower 3 bits is bankid */ 208 *qpn = (id << 3) | bankid; 209 210 return 0; 211 } 212 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 213 { 214 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 215 unsigned long num = 0; 216 u8 bankid; 217 int ret; 218 219 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { 220 num = 1; 221 hr_qp->doorbell_qpn = 1; 222 } else { 223 mutex_lock(&qp_table->bank_mutex); 224 bankid = get_least_load_bankid_for_qp(qp_table->bank); 225 226 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid, 227 &num); 228 if (ret) { 229 ibdev_err(&hr_dev->ib_dev, 230 "failed to alloc QPN, ret = %d\n", ret); 231 mutex_unlock(&qp_table->bank_mutex); 232 return ret; 233 } 234 235 qp_table->bank[bankid].inuse++; 236 mutex_unlock(&qp_table->bank_mutex); 237 238 hr_qp->doorbell_qpn = (u32)num; 239 } 240 241 hr_qp->qpn = num; 242 243 return 0; 244 } 245 246 static void add_qp_to_list(struct hns_roce_dev *hr_dev, 247 struct hns_roce_qp *hr_qp, 248 struct ib_cq *send_cq, struct ib_cq *recv_cq) 249 { 250 struct hns_roce_cq *hr_send_cq, *hr_recv_cq; 251 unsigned long flags; 252 253 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; 254 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; 255 256 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 257 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); 258 259 list_add_tail(&hr_qp->node, &hr_dev->qp_list); 260 if (hr_send_cq) 261 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); 262 if (hr_recv_cq) 263 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); 264 265 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); 266 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 267 } 268 269 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev, 270 struct hns_roce_qp *hr_qp, 271 struct ib_qp_init_attr *init_attr) 272 { 273 struct xarray *xa = &hr_dev->qp_table_xa; 274 int ret; 275 276 if (!hr_qp->qpn) 277 return -EINVAL; 278 279 ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL)); 280 if (ret) 281 dev_err(hr_dev->dev, "Failed to xa store for QPC\n"); 282 else 283 /* add QP to device's QP list for softwc */ 284 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, 285 init_attr->recv_cq); 286 287 return ret; 288 } 289 290 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 291 { 292 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 293 struct device *dev = hr_dev->dev; 294 int ret; 295 296 if (!hr_qp->qpn) 297 return -EINVAL; 298 299 /* Alloc memory for QPC */ 300 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 301 if (ret) { 302 dev_err(dev, "Failed to get QPC table\n"); 303 goto err_out; 304 } 305 306 /* Alloc memory for IRRL */ 307 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 308 if (ret) { 309 dev_err(dev, "Failed to get IRRL table\n"); 310 goto err_put_qp; 311 } 312 313 if (hr_dev->caps.trrl_entry_sz) { 314 /* Alloc memory for TRRL */ 315 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 316 hr_qp->qpn); 317 if (ret) { 318 dev_err(dev, "Failed to get TRRL table\n"); 319 goto err_put_irrl; 320 } 321 } 322 323 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 324 /* Alloc memory for SCC CTX */ 325 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 326 hr_qp->qpn); 327 if (ret) { 328 dev_err(dev, "Failed to get SCC CTX table\n"); 329 goto err_put_trrl; 330 } 331 } 332 333 return 0; 334 335 err_put_trrl: 336 if (hr_dev->caps.trrl_entry_sz) 337 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 338 339 err_put_irrl: 340 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 341 342 err_put_qp: 343 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 344 345 err_out: 346 return ret; 347 } 348 349 static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp) 350 { 351 rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry); 352 } 353 354 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 355 { 356 struct xarray *xa = &hr_dev->qp_table_xa; 357 unsigned long flags; 358 359 list_del(&hr_qp->node); 360 361 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 362 list_del(&hr_qp->sq_node); 363 364 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI && 365 hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 366 list_del(&hr_qp->rq_node); 367 368 xa_lock_irqsave(xa, flags); 369 __xa_erase(xa, hr_qp->qpn); 370 xa_unlock_irqrestore(xa, flags); 371 } 372 373 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 374 { 375 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 376 377 if (hr_dev->caps.trrl_entry_sz) 378 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 379 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 380 } 381 382 static inline u8 get_qp_bankid(unsigned long qpn) 383 { 384 /* The lower 3 bits of QPN are used to hash to different banks */ 385 return (u8)(qpn & GENMASK(2, 0)); 386 } 387 388 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 389 { 390 u8 bankid; 391 392 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) 393 return; 394 395 if (hr_qp->qpn < hr_dev->caps.reserved_qps) 396 return; 397 398 bankid = get_qp_bankid(hr_qp->qpn); 399 400 ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); 401 402 mutex_lock(&hr_dev->qp_table.bank_mutex); 403 hr_dev->qp_table.bank[bankid].inuse--; 404 mutex_unlock(&hr_dev->qp_table.bank_mutex); 405 } 406 407 static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp, 408 bool user) 409 { 410 u32 max_sge = dev->caps.max_rq_sg; 411 412 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 413 return max_sge; 414 415 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will 416 * calculate number of max_sge with reserved SGEs when allocating wqe 417 * buf, so there is no need to do this again in kernel. But the number 418 * may exceed the capacity of SGEs recorded in the firmware, so the 419 * kernel driver should just adapt the value accordingly. 420 */ 421 if (user) 422 max_sge = roundup_pow_of_two(max_sge + 1); 423 else 424 hr_qp->rq.rsv_sge = 1; 425 426 return max_sge; 427 } 428 429 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, 430 struct hns_roce_qp *hr_qp, int has_rq, bool user) 431 { 432 u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user); 433 u32 cnt; 434 435 /* If srq exist, set zero for relative number of rq */ 436 if (!has_rq) { 437 hr_qp->rq.wqe_cnt = 0; 438 hr_qp->rq.max_gs = 0; 439 hr_qp->rq_inl_buf.wqe_cnt = 0; 440 cap->max_recv_wr = 0; 441 cap->max_recv_sge = 0; 442 443 return 0; 444 } 445 446 /* Check the validity of QP support capacity */ 447 if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes || 448 cap->max_recv_sge > max_sge) { 449 ibdev_err(&hr_dev->ib_dev, 450 "RQ config error, depth = %u, sge = %u\n", 451 cap->max_recv_wr, cap->max_recv_sge); 452 return -EINVAL; 453 } 454 455 cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes)); 456 if (cnt > hr_dev->caps.max_wqes) { 457 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n", 458 cap->max_recv_wr); 459 return -EINVAL; 460 } 461 462 hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) + 463 hr_qp->rq.rsv_sge); 464 465 if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE) 466 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz); 467 else 468 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * 469 hr_qp->rq.max_gs); 470 471 hr_qp->rq.wqe_cnt = cnt; 472 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE && 473 hr_qp->ibqp.qp_type != IB_QPT_UD && 474 hr_qp->ibqp.qp_type != IB_QPT_GSI) 475 hr_qp->rq_inl_buf.wqe_cnt = cnt; 476 else 477 hr_qp->rq_inl_buf.wqe_cnt = 0; 478 479 cap->max_recv_wr = cnt; 480 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 481 482 return 0; 483 } 484 485 static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp) 486 { 487 /* GSI/UD QP only has extended sge */ 488 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) 489 return qp->sq.max_gs; 490 491 if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) 492 return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE; 493 494 return 0; 495 } 496 497 static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, 498 struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap) 499 { 500 u32 total_sge_cnt; 501 u32 wqe_sge_cnt; 502 503 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT; 504 505 hr_qp->sq.max_gs = max(1U, cap->max_send_sge); 506 507 wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp); 508 509 /* If the number of extended sge is not zero, they MUST use the 510 * space of HNS_HW_PAGE_SIZE at least. 511 */ 512 if (wqe_sge_cnt) { 513 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); 514 hr_qp->sge.sge_cnt = max(total_sge_cnt, 515 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE); 516 } 517 } 518 519 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 520 struct ib_qp_cap *cap, 521 struct hns_roce_ib_create_qp *ucmd) 522 { 523 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 524 u8 max_sq_stride = ilog2(roundup_sq_stride); 525 526 /* Sanity check SQ size before proceeding */ 527 if (ucmd->log_sq_stride > max_sq_stride || 528 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 529 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n"); 530 return -EINVAL; 531 } 532 533 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 534 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n", 535 cap->max_send_sge); 536 return -EINVAL; 537 } 538 539 return 0; 540 } 541 542 static int set_user_sq_size(struct hns_roce_dev *hr_dev, 543 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp, 544 struct hns_roce_ib_create_qp *ucmd) 545 { 546 struct ib_device *ibdev = &hr_dev->ib_dev; 547 u32 cnt = 0; 548 int ret; 549 550 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) || 551 cnt > hr_dev->caps.max_wqes) 552 return -EINVAL; 553 554 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 555 if (ret) { 556 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n", 557 ret); 558 return ret; 559 } 560 561 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 562 563 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 564 hr_qp->sq.wqe_cnt = cnt; 565 566 return 0; 567 } 568 569 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev, 570 struct hns_roce_qp *hr_qp, 571 struct hns_roce_buf_attr *buf_attr) 572 { 573 int buf_size; 574 int idx = 0; 575 576 hr_qp->buff_size = 0; 577 578 /* SQ WQE */ 579 hr_qp->sq.offset = 0; 580 buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt, 581 hr_qp->sq.wqe_shift); 582 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 583 buf_attr->region[idx].size = buf_size; 584 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num; 585 idx++; 586 hr_qp->buff_size += buf_size; 587 } 588 589 /* extend SGE WQE in SQ */ 590 hr_qp->sge.offset = hr_qp->buff_size; 591 buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt, 592 hr_qp->sge.sge_shift); 593 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 594 buf_attr->region[idx].size = buf_size; 595 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num; 596 idx++; 597 hr_qp->buff_size += buf_size; 598 } 599 600 /* RQ WQE */ 601 hr_qp->rq.offset = hr_qp->buff_size; 602 buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt, 603 hr_qp->rq.wqe_shift); 604 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 605 buf_attr->region[idx].size = buf_size; 606 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num; 607 idx++; 608 hr_qp->buff_size += buf_size; 609 } 610 611 if (hr_qp->buff_size < 1) 612 return -EINVAL; 613 614 buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 615 buf_attr->region_count = idx; 616 617 return 0; 618 } 619 620 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, 621 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) 622 { 623 struct ib_device *ibdev = &hr_dev->ib_dev; 624 u32 cnt; 625 626 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes || 627 cap->max_send_sge > hr_dev->caps.max_sq_sg) { 628 ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n"); 629 return -EINVAL; 630 } 631 632 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes)); 633 if (cnt > hr_dev->caps.max_wqes) { 634 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n", 635 cnt); 636 return -EINVAL; 637 } 638 639 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 640 hr_qp->sq.wqe_cnt = cnt; 641 642 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 643 644 /* sync the parameters of kernel QP to user's configuration */ 645 cap->max_send_wr = cnt; 646 cap->max_send_sge = hr_qp->sq.max_gs; 647 648 return 0; 649 } 650 651 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 652 { 653 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 654 return 0; 655 656 return 1; 657 } 658 659 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 660 { 661 if (attr->qp_type == IB_QPT_XRC_INI || 662 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 663 !attr->cap.max_recv_wr) 664 return 0; 665 666 return 1; 667 } 668 669 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 670 struct ib_qp_init_attr *init_attr) 671 { 672 u32 max_recv_sge = init_attr->cap.max_recv_sge; 673 u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt; 674 struct hns_roce_rinl_wqe *wqe_list; 675 int i; 676 677 /* allocate recv inline buf */ 678 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 679 GFP_KERNEL); 680 if (!wqe_list) 681 goto err; 682 683 /* Allocate a continuous buffer for all inline sge we need */ 684 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 685 sizeof(struct hns_roce_rinl_sge)), 686 GFP_KERNEL); 687 if (!wqe_list[0].sg_list) 688 goto err_wqe_list; 689 690 /* Assign buffers of sg_list to each inline wqe */ 691 for (i = 1; i < wqe_cnt; i++) 692 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 693 694 hr_qp->rq_inl_buf.wqe_list = wqe_list; 695 696 return 0; 697 698 err_wqe_list: 699 kfree(wqe_list); 700 701 err: 702 return -ENOMEM; 703 } 704 705 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 706 { 707 if (hr_qp->rq_inl_buf.wqe_list) 708 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 709 kfree(hr_qp->rq_inl_buf.wqe_list); 710 } 711 712 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 713 struct ib_qp_init_attr *init_attr, 714 struct ib_udata *udata, unsigned long addr) 715 { 716 struct ib_device *ibdev = &hr_dev->ib_dev; 717 struct hns_roce_buf_attr buf_attr = {}; 718 int ret; 719 720 if (!udata && hr_qp->rq_inl_buf.wqe_cnt) { 721 ret = alloc_rq_inline_buf(hr_qp, init_attr); 722 if (ret) { 723 ibdev_err(ibdev, 724 "failed to alloc inline buf, ret = %d.\n", 725 ret); 726 return ret; 727 } 728 } else { 729 hr_qp->rq_inl_buf.wqe_list = NULL; 730 } 731 732 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr); 733 if (ret) { 734 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); 735 goto err_inline; 736 } 737 ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, 738 PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, 739 udata, addr); 740 if (ret) { 741 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); 742 goto err_inline; 743 } 744 745 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE) 746 hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE; 747 748 return 0; 749 750 err_inline: 751 free_rq_inline_buf(hr_qp); 752 753 return ret; 754 } 755 756 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 757 { 758 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); 759 free_rq_inline_buf(hr_qp); 760 } 761 762 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, 763 struct ib_qp_init_attr *init_attr, 764 struct ib_udata *udata, 765 struct hns_roce_ib_create_qp_resp *resp, 766 struct hns_roce_ib_create_qp *ucmd) 767 { 768 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 769 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 770 hns_roce_qp_has_sq(init_attr) && 771 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr)); 772 } 773 774 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, 775 struct ib_qp_init_attr *init_attr, 776 struct ib_udata *udata, 777 struct hns_roce_ib_create_qp_resp *resp) 778 { 779 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 780 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 781 hns_roce_qp_has_rq(init_attr)); 782 } 783 784 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, 785 struct ib_qp_init_attr *init_attr) 786 { 787 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 788 hns_roce_qp_has_rq(init_attr)); 789 } 790 791 static int qp_mmap_entry(struct hns_roce_qp *hr_qp, 792 struct hns_roce_dev *hr_dev, 793 struct ib_udata *udata, 794 struct hns_roce_ib_create_qp_resp *resp) 795 { 796 struct hns_roce_ucontext *uctx = 797 rdma_udata_to_drv_context(udata, 798 struct hns_roce_ucontext, ibucontext); 799 struct rdma_user_mmap_entry *rdma_entry; 800 u64 address; 801 802 address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE; 803 804 hr_qp->dwqe_mmap_entry = 805 hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address, 806 HNS_ROCE_DWQE_SIZE, 807 HNS_ROCE_MMAP_TYPE_DWQE); 808 809 if (!hr_qp->dwqe_mmap_entry) { 810 ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n"); 811 return -ENOMEM; 812 } 813 814 rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry; 815 resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry); 816 817 return 0; 818 } 819 820 static int alloc_user_qp_db(struct hns_roce_dev *hr_dev, 821 struct hns_roce_qp *hr_qp, 822 struct ib_qp_init_attr *init_attr, 823 struct ib_udata *udata, 824 struct hns_roce_ib_create_qp *ucmd, 825 struct hns_roce_ib_create_qp_resp *resp) 826 { 827 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, 828 struct hns_roce_ucontext, ibucontext); 829 struct ib_device *ibdev = &hr_dev->ib_dev; 830 int ret; 831 832 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) { 833 ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, &hr_qp->sdb); 834 if (ret) { 835 ibdev_err(ibdev, 836 "failed to map user SQ doorbell, ret = %d.\n", 837 ret); 838 goto err_out; 839 } 840 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; 841 } 842 843 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) { 844 ret = hns_roce_db_map_user(uctx, ucmd->db_addr, &hr_qp->rdb); 845 if (ret) { 846 ibdev_err(ibdev, 847 "failed to map user RQ doorbell, ret = %d.\n", 848 ret); 849 goto err_sdb; 850 } 851 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 852 } 853 854 return 0; 855 856 err_sdb: 857 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 858 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 859 err_out: 860 return ret; 861 } 862 863 static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev, 864 struct hns_roce_qp *hr_qp, 865 struct ib_qp_init_attr *init_attr) 866 { 867 struct ib_device *ibdev = &hr_dev->ib_dev; 868 int ret; 869 870 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 871 hr_qp->sq.db_reg = hr_dev->mem_base + 872 HNS_ROCE_DWQE_SIZE * hr_qp->qpn; 873 else 874 hr_qp->sq.db_reg = hr_dev->reg_base + hr_dev->sdb_offset + 875 DB_REG_OFFSET * hr_dev->priv_uar.index; 876 877 hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset + 878 DB_REG_OFFSET * hr_dev->priv_uar.index; 879 880 if (kernel_qp_has_rdb(hr_dev, init_attr)) { 881 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 882 if (ret) { 883 ibdev_err(ibdev, 884 "failed to alloc kernel RQ doorbell, ret = %d.\n", 885 ret); 886 return ret; 887 } 888 *hr_qp->rdb.db_record = 0; 889 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 890 } 891 892 return 0; 893 } 894 895 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 896 struct ib_qp_init_attr *init_attr, 897 struct ib_udata *udata, 898 struct hns_roce_ib_create_qp *ucmd, 899 struct hns_roce_ib_create_qp_resp *resp) 900 { 901 int ret; 902 903 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE) 904 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB; 905 906 if (udata) { 907 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) { 908 ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp); 909 if (ret) 910 return ret; 911 } 912 913 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd, 914 resp); 915 if (ret) 916 goto err_remove_qp; 917 } else { 918 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr); 919 if (ret) 920 return ret; 921 } 922 923 return 0; 924 925 err_remove_qp: 926 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) 927 qp_user_mmap_entry_remove(hr_qp); 928 929 return ret; 930 } 931 932 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 933 struct ib_udata *udata) 934 { 935 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 936 udata, struct hns_roce_ucontext, ibucontext); 937 938 if (udata) { 939 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 940 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 941 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 942 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 943 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) 944 qp_user_mmap_entry_remove(hr_qp); 945 } else { 946 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 947 hns_roce_free_db(hr_dev, &hr_qp->rdb); 948 } 949 } 950 951 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev, 952 struct hns_roce_qp *hr_qp) 953 { 954 struct ib_device *ibdev = &hr_dev->ib_dev; 955 u64 *sq_wrid = NULL; 956 u64 *rq_wrid = NULL; 957 int ret; 958 959 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL); 960 if (ZERO_OR_NULL_PTR(sq_wrid)) { 961 ibdev_err(ibdev, "failed to alloc SQ wrid.\n"); 962 return -ENOMEM; 963 } 964 965 if (hr_qp->rq.wqe_cnt) { 966 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL); 967 if (ZERO_OR_NULL_PTR(rq_wrid)) { 968 ibdev_err(ibdev, "failed to alloc RQ wrid.\n"); 969 ret = -ENOMEM; 970 goto err_sq; 971 } 972 } 973 974 hr_qp->sq.wrid = sq_wrid; 975 hr_qp->rq.wrid = rq_wrid; 976 return 0; 977 err_sq: 978 kfree(sq_wrid); 979 980 return ret; 981 } 982 983 static void free_kernel_wrid(struct hns_roce_qp *hr_qp) 984 { 985 kfree(hr_qp->rq.wrid); 986 kfree(hr_qp->sq.wrid); 987 } 988 989 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 990 struct ib_qp_init_attr *init_attr, 991 struct ib_udata *udata, 992 struct hns_roce_ib_create_qp *ucmd) 993 { 994 struct ib_device *ibdev = &hr_dev->ib_dev; 995 int ret; 996 997 if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) 998 init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; 999 1000 hr_qp->max_inline_data = init_attr->cap.max_inline_data; 1001 1002 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 1003 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 1004 else 1005 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 1006 1007 ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp, 1008 hns_roce_qp_has_rq(init_attr), !!udata); 1009 if (ret) { 1010 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n", 1011 ret); 1012 return ret; 1013 } 1014 1015 if (udata) { 1016 ret = ib_copy_from_udata(ucmd, udata, 1017 min(udata->inlen, sizeof(*ucmd))); 1018 if (ret) { 1019 ibdev_err(ibdev, 1020 "failed to copy QP ucmd, ret = %d\n", ret); 1021 return ret; 1022 } 1023 1024 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd); 1025 if (ret) 1026 ibdev_err(ibdev, 1027 "failed to set user SQ size, ret = %d.\n", 1028 ret); 1029 } else { 1030 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp); 1031 if (ret) 1032 ibdev_err(ibdev, 1033 "failed to set kernel SQ size, ret = %d.\n", 1034 ret); 1035 } 1036 1037 return ret; 1038 } 1039 1040 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 1041 struct ib_pd *ib_pd, 1042 struct ib_qp_init_attr *init_attr, 1043 struct ib_udata *udata, 1044 struct hns_roce_qp *hr_qp) 1045 { 1046 struct hns_roce_ib_create_qp_resp resp = {}; 1047 struct ib_device *ibdev = &hr_dev->ib_dev; 1048 struct hns_roce_ib_create_qp ucmd; 1049 int ret; 1050 1051 mutex_init(&hr_qp->mutex); 1052 spin_lock_init(&hr_qp->sq.lock); 1053 spin_lock_init(&hr_qp->rq.lock); 1054 1055 hr_qp->state = IB_QPS_RESET; 1056 hr_qp->flush_flag = 0; 1057 1058 if (init_attr->create_flags) 1059 return -EOPNOTSUPP; 1060 1061 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); 1062 if (ret) { 1063 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); 1064 return ret; 1065 } 1066 1067 if (!udata) { 1068 ret = alloc_kernel_wrid(hr_dev, hr_qp); 1069 if (ret) { 1070 ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", 1071 ret); 1072 return ret; 1073 } 1074 } 1075 1076 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); 1077 if (ret) { 1078 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret); 1079 goto err_buf; 1080 } 1081 1082 ret = alloc_qpn(hr_dev, hr_qp); 1083 if (ret) { 1084 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret); 1085 goto err_qpn; 1086 } 1087 1088 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); 1089 if (ret) { 1090 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", 1091 ret); 1092 goto err_db; 1093 } 1094 1095 ret = alloc_qpc(hr_dev, hr_qp); 1096 if (ret) { 1097 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n", 1098 ret); 1099 goto err_qpc; 1100 } 1101 1102 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr); 1103 if (ret) { 1104 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret); 1105 goto err_store; 1106 } 1107 1108 if (udata) { 1109 resp.cap_flags = hr_qp->en_flags; 1110 ret = ib_copy_to_udata(udata, &resp, 1111 min(udata->outlen, sizeof(resp))); 1112 if (ret) { 1113 ibdev_err(ibdev, "copy qp resp failed!\n"); 1114 goto err_store; 1115 } 1116 } 1117 1118 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 1119 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 1120 if (ret) 1121 goto err_flow_ctrl; 1122 } 1123 1124 hr_qp->ibqp.qp_num = hr_qp->qpn; 1125 hr_qp->event = hns_roce_ib_qp_event; 1126 refcount_set(&hr_qp->refcount, 1); 1127 init_completion(&hr_qp->free); 1128 1129 return 0; 1130 1131 err_flow_ctrl: 1132 hns_roce_qp_remove(hr_dev, hr_qp); 1133 err_store: 1134 free_qpc(hr_dev, hr_qp); 1135 err_qpc: 1136 free_qp_db(hr_dev, hr_qp, udata); 1137 err_db: 1138 free_qpn(hr_dev, hr_qp); 1139 err_qpn: 1140 free_qp_buf(hr_dev, hr_qp); 1141 err_buf: 1142 free_kernel_wrid(hr_qp); 1143 return ret; 1144 } 1145 1146 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 1147 struct ib_udata *udata) 1148 { 1149 if (refcount_dec_and_test(&hr_qp->refcount)) 1150 complete(&hr_qp->free); 1151 wait_for_completion(&hr_qp->free); 1152 1153 free_qpc(hr_dev, hr_qp); 1154 free_qpn(hr_dev, hr_qp); 1155 free_qp_buf(hr_dev, hr_qp); 1156 free_kernel_wrid(hr_qp); 1157 free_qp_db(hr_dev, hr_qp, udata); 1158 } 1159 1160 static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, 1161 bool is_user) 1162 { 1163 switch (type) { 1164 case IB_QPT_XRC_INI: 1165 case IB_QPT_XRC_TGT: 1166 if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)) 1167 goto out; 1168 break; 1169 case IB_QPT_UD: 1170 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && 1171 is_user) 1172 goto out; 1173 break; 1174 case IB_QPT_RC: 1175 case IB_QPT_GSI: 1176 break; 1177 default: 1178 goto out; 1179 } 1180 1181 return 0; 1182 1183 out: 1184 ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type); 1185 1186 return -EOPNOTSUPP; 1187 } 1188 1189 int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, 1190 struct ib_udata *udata) 1191 { 1192 struct ib_device *ibdev = qp->device; 1193 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 1194 struct hns_roce_qp *hr_qp = to_hr_qp(qp); 1195 struct ib_pd *pd = qp->pd; 1196 int ret; 1197 1198 ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); 1199 if (ret) 1200 return ret; 1201 1202 if (init_attr->qp_type == IB_QPT_XRC_TGT) 1203 hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn; 1204 1205 if (init_attr->qp_type == IB_QPT_GSI) { 1206 hr_qp->port = init_attr->port_num - 1; 1207 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1208 } 1209 1210 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); 1211 if (ret) 1212 ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", 1213 init_attr->qp_type, ret); 1214 1215 return ret; 1216 } 1217 1218 int to_hr_qp_type(int qp_type) 1219 { 1220 switch (qp_type) { 1221 case IB_QPT_RC: 1222 return SERV_TYPE_RC; 1223 case IB_QPT_UD: 1224 case IB_QPT_GSI: 1225 return SERV_TYPE_UD; 1226 case IB_QPT_XRC_INI: 1227 case IB_QPT_XRC_TGT: 1228 return SERV_TYPE_XRC; 1229 default: 1230 return -1; 1231 } 1232 } 1233 1234 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1235 struct hns_roce_qp *hr_qp, 1236 struct ib_qp_attr *attr, int attr_mask) 1237 { 1238 enum ib_mtu active_mtu; 1239 int p; 1240 1241 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1242 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1243 1244 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1245 attr->path_mtu > hr_dev->caps.max_mtu) || 1246 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1247 ibdev_err(&hr_dev->ib_dev, 1248 "attr path_mtu(%d)invalid while modify qp", 1249 attr->path_mtu); 1250 return -EINVAL; 1251 } 1252 1253 return 0; 1254 } 1255 1256 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1257 int attr_mask) 1258 { 1259 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1260 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1261 int p; 1262 1263 if ((attr_mask & IB_QP_PORT) && 1264 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1265 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n", 1266 attr->port_num); 1267 return -EINVAL; 1268 } 1269 1270 if (attr_mask & IB_QP_PKEY_INDEX) { 1271 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1272 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1273 ibdev_err(&hr_dev->ib_dev, 1274 "invalid attr, pkey_index = %u.\n", 1275 attr->pkey_index); 1276 return -EINVAL; 1277 } 1278 } 1279 1280 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1281 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1282 ibdev_err(&hr_dev->ib_dev, 1283 "invalid attr, max_rd_atomic = %u.\n", 1284 attr->max_rd_atomic); 1285 return -EINVAL; 1286 } 1287 1288 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1289 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1290 ibdev_err(&hr_dev->ib_dev, 1291 "invalid attr, max_dest_rd_atomic = %u.\n", 1292 attr->max_dest_rd_atomic); 1293 return -EINVAL; 1294 } 1295 1296 if (attr_mask & IB_QP_PATH_MTU) 1297 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1298 1299 return 0; 1300 } 1301 1302 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1303 int attr_mask, struct ib_udata *udata) 1304 { 1305 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1306 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1307 enum ib_qp_state cur_state, new_state; 1308 int ret = -EINVAL; 1309 1310 mutex_lock(&hr_qp->mutex); 1311 1312 if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) 1313 goto out; 1314 1315 cur_state = hr_qp->state; 1316 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1317 1318 if (ibqp->uobject && 1319 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1320 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) { 1321 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1322 1323 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 1324 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1325 } else { 1326 ibdev_warn(&hr_dev->ib_dev, 1327 "flush cqe is not supported in userspace!\n"); 1328 goto out; 1329 } 1330 } 1331 1332 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1333 attr_mask)) { 1334 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1335 goto out; 1336 } 1337 1338 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1339 if (ret) 1340 goto out; 1341 1342 if (cur_state == new_state && cur_state == IB_QPS_RESET) 1343 goto out; 1344 1345 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1346 new_state); 1347 1348 out: 1349 mutex_unlock(&hr_qp->mutex); 1350 1351 return ret; 1352 } 1353 1354 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1355 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1356 { 1357 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1358 __acquire(&send_cq->lock); 1359 __acquire(&recv_cq->lock); 1360 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1361 spin_lock_irq(&send_cq->lock); 1362 __acquire(&recv_cq->lock); 1363 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1364 spin_lock_irq(&recv_cq->lock); 1365 __acquire(&send_cq->lock); 1366 } else if (send_cq == recv_cq) { 1367 spin_lock_irq(&send_cq->lock); 1368 __acquire(&recv_cq->lock); 1369 } else if (send_cq->cqn < recv_cq->cqn) { 1370 spin_lock_irq(&send_cq->lock); 1371 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1372 } else { 1373 spin_lock_irq(&recv_cq->lock); 1374 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1375 } 1376 } 1377 1378 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1379 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1380 __releases(&recv_cq->lock) 1381 { 1382 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1383 __release(&recv_cq->lock); 1384 __release(&send_cq->lock); 1385 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1386 __release(&recv_cq->lock); 1387 spin_unlock(&send_cq->lock); 1388 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1389 __release(&send_cq->lock); 1390 spin_unlock(&recv_cq->lock); 1391 } else if (send_cq == recv_cq) { 1392 __release(&recv_cq->lock); 1393 spin_unlock_irq(&send_cq->lock); 1394 } else if (send_cq->cqn < recv_cq->cqn) { 1395 spin_unlock(&recv_cq->lock); 1396 spin_unlock_irq(&send_cq->lock); 1397 } else { 1398 spin_unlock(&send_cq->lock); 1399 spin_unlock_irq(&recv_cq->lock); 1400 } 1401 } 1402 1403 static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset) 1404 { 1405 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); 1406 } 1407 1408 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1409 { 1410 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1411 } 1412 1413 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1414 { 1415 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1416 } 1417 1418 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n) 1419 { 1420 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); 1421 } 1422 1423 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq, 1424 struct ib_cq *ib_cq) 1425 { 1426 struct hns_roce_cq *hr_cq; 1427 u32 cur; 1428 1429 cur = hr_wq->head - hr_wq->tail; 1430 if (likely(cur + nreq < hr_wq->wqe_cnt)) 1431 return false; 1432 1433 hr_cq = to_hr_cq(ib_cq); 1434 spin_lock(&hr_cq->lock); 1435 cur = hr_wq->head - hr_wq->tail; 1436 spin_unlock(&hr_cq->lock); 1437 1438 return cur + nreq >= hr_wq->wqe_cnt; 1439 } 1440 1441 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1442 { 1443 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1444 unsigned int reserved_from_bot; 1445 unsigned int i; 1446 1447 qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps, 1448 sizeof(u32), GFP_KERNEL); 1449 if (!qp_table->idx_table.spare_idx) 1450 return -ENOMEM; 1451 1452 mutex_init(&qp_table->scc_mutex); 1453 mutex_init(&qp_table->bank_mutex); 1454 xa_init(&hr_dev->qp_table_xa); 1455 1456 reserved_from_bot = hr_dev->caps.reserved_qps; 1457 1458 for (i = 0; i < reserved_from_bot; i++) { 1459 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++; 1460 hr_dev->qp_table.bank[get_qp_bankid(i)].min++; 1461 } 1462 1463 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) { 1464 ida_init(&hr_dev->qp_table.bank[i].ida); 1465 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps / 1466 HNS_ROCE_QP_BANK_NUM - 1; 1467 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min; 1468 } 1469 1470 return 0; 1471 } 1472 1473 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1474 { 1475 int i; 1476 1477 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) 1478 ida_destroy(&hr_dev->qp_table.bank[i].ida); 1479 kfree(hr_dev->qp_table.idx_table.spare_idx); 1480 } 1481