1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018 Hisilicon Limited. 4 */ 5 6 #include <linux/pci.h> 7 #include <rdma/ib_umem.h> 8 #include <rdma/uverbs_ioctl.h> 9 #include "hns_roce_device.h" 10 #include "hns_roce_cmd.h" 11 #include "hns_roce_hem.h" 12 13 void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type) 14 { 15 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 16 struct hns_roce_srq *srq; 17 18 xa_lock(&srq_table->xa); 19 srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1)); 20 if (srq) 21 refcount_inc(&srq->refcount); 22 xa_unlock(&srq_table->xa); 23 24 if (!srq) { 25 dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn); 26 return; 27 } 28 29 srq->event(srq, event_type); 30 31 if (refcount_dec_and_test(&srq->refcount)) 32 complete(&srq->free); 33 } 34 35 static void hns_roce_ib_srq_event(struct hns_roce_srq *srq, 36 enum hns_roce_event event_type) 37 { 38 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); 39 struct ib_srq *ibsrq = &srq->ibsrq; 40 struct ib_event event; 41 42 if (ibsrq->event_handler) { 43 event.device = ibsrq->device; 44 event.element.srq = ibsrq; 45 switch (event_type) { 46 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 47 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 48 break; 49 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 50 event.event = IB_EVENT_SRQ_ERR; 51 break; 52 default: 53 dev_err(hr_dev->dev, 54 "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n", 55 event_type, srq->srqn); 56 return; 57 } 58 59 ibsrq->event_handler(&event, ibsrq->srq_context); 60 } 61 } 62 63 static int alloc_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 64 { 65 struct hns_roce_ida *srq_ida = &hr_dev->srq_table.srq_ida; 66 int id; 67 68 id = ida_alloc_range(&srq_ida->ida, srq_ida->min, srq_ida->max, 69 GFP_KERNEL); 70 if (id < 0) { 71 ibdev_err(&hr_dev->ib_dev, "failed to alloc srq(%d).\n", id); 72 return -ENOMEM; 73 } 74 75 srq->srqn = id; 76 77 return 0; 78 } 79 80 static void free_srqn(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 81 { 82 ida_free(&hr_dev->srq_table.srq_ida.ida, (int)srq->srqn); 83 } 84 85 static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev, 86 struct hns_roce_srq *srq) 87 { 88 struct ib_device *ibdev = &hr_dev->ib_dev; 89 struct hns_roce_cmd_mailbox *mailbox; 90 int ret; 91 92 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 93 if (IS_ERR(mailbox)) { 94 ibdev_err(ibdev, "failed to alloc mailbox for SRQC.\n"); 95 return PTR_ERR(mailbox); 96 } 97 98 ret = hr_dev->hw->write_srqc(srq, mailbox->buf); 99 if (ret) { 100 ibdev_err(ibdev, "failed to write SRQC.\n"); 101 goto err_mbox; 102 } 103 104 ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ, 105 srq->srqn); 106 if (ret) 107 ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret); 108 109 err_mbox: 110 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 111 return ret; 112 } 113 114 static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 115 { 116 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 117 struct ib_device *ibdev = &hr_dev->ib_dev; 118 int ret; 119 120 ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn); 121 if (ret) { 122 ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret); 123 return ret; 124 } 125 126 ret = xa_err(xa_store_irq(&srq_table->xa, srq->srqn, srq, GFP_KERNEL)); 127 if (ret) { 128 ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret); 129 goto err_put; 130 } 131 132 ret = hns_roce_create_srqc(hr_dev, srq); 133 if (ret) 134 goto err_xa; 135 136 return 0; 137 138 err_xa: 139 xa_erase_irq(&srq_table->xa, srq->srqn); 140 err_put: 141 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); 142 143 return ret; 144 } 145 146 static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 147 { 148 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 149 int ret; 150 151 ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_SRQ, 152 srq->srqn); 153 if (ret) 154 dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n", 155 ret, srq->srqn); 156 157 xa_erase_irq(&srq_table->xa, srq->srqn); 158 159 if (refcount_dec_and_test(&srq->refcount)) 160 complete(&srq->free); 161 wait_for_completion(&srq->free); 162 163 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); 164 } 165 166 static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, 167 struct ib_udata *udata, unsigned long addr) 168 { 169 struct hns_roce_idx_que *idx_que = &srq->idx_que; 170 struct ib_device *ibdev = &hr_dev->ib_dev; 171 struct hns_roce_buf_attr buf_attr = {}; 172 int ret; 173 174 srq->idx_que.entry_shift = ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ); 175 176 buf_attr.page_shift = hr_dev->caps.idx_buf_pg_sz + PAGE_SHIFT; 177 buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt, 178 srq->idx_que.entry_shift); 179 buf_attr.region[0].hopnum = hr_dev->caps.idx_hop_num; 180 buf_attr.region_count = 1; 181 182 ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr, 183 hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT, 184 udata, addr); 185 if (ret) { 186 ibdev_err(ibdev, 187 "failed to alloc SRQ idx mtr, ret = %d.\n", ret); 188 return ret; 189 } 190 191 if (!udata) { 192 idx_que->bitmap = bitmap_zalloc(srq->wqe_cnt, GFP_KERNEL); 193 if (!idx_que->bitmap) { 194 ibdev_err(ibdev, "failed to alloc SRQ idx bitmap.\n"); 195 ret = -ENOMEM; 196 goto err_idx_mtr; 197 } 198 } 199 200 idx_que->head = 0; 201 idx_que->tail = 0; 202 203 return 0; 204 err_idx_mtr: 205 hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); 206 207 return ret; 208 } 209 210 static void free_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 211 { 212 struct hns_roce_idx_que *idx_que = &srq->idx_que; 213 214 bitmap_free(idx_que->bitmap); 215 idx_que->bitmap = NULL; 216 hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); 217 } 218 219 static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev, 220 struct hns_roce_srq *srq, 221 struct ib_udata *udata, unsigned long addr) 222 { 223 struct ib_device *ibdev = &hr_dev->ib_dev; 224 struct hns_roce_buf_attr buf_attr = {}; 225 int ret; 226 227 srq->wqe_shift = ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE, 228 HNS_ROCE_SGE_SIZE * 229 srq->max_gs))); 230 231 buf_attr.page_shift = hr_dev->caps.srqwqe_buf_pg_sz + PAGE_SHIFT; 232 buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt, 233 srq->wqe_shift); 234 buf_attr.region[0].hopnum = hr_dev->caps.srqwqe_hop_num; 235 buf_attr.region_count = 1; 236 237 ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr, 238 hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT, 239 udata, addr); 240 if (ret) 241 ibdev_err(ibdev, 242 "failed to alloc SRQ buf mtr, ret = %d.\n", ret); 243 244 return ret; 245 } 246 247 static void free_srq_wqe_buf(struct hns_roce_dev *hr_dev, 248 struct hns_roce_srq *srq) 249 { 250 hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr); 251 } 252 253 static int alloc_srq_wrid(struct hns_roce_srq *srq) 254 { 255 srq->wrid = kvmalloc_array(srq->wqe_cnt, sizeof(u64), GFP_KERNEL); 256 if (!srq->wrid) 257 return -ENOMEM; 258 259 return 0; 260 } 261 262 static void free_srq_wrid(struct hns_roce_srq *srq) 263 { 264 kvfree(srq->wrid); 265 srq->wrid = NULL; 266 } 267 268 static u32 proc_srq_sge(struct hns_roce_dev *dev, struct hns_roce_srq *hr_srq, 269 bool user) 270 { 271 u32 max_sge = dev->caps.max_srq_sges; 272 273 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 274 return max_sge; 275 276 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will 277 * calculate number of max_sge with reserved SGEs when allocating wqe 278 * buf, so there is no need to do this again in kernel. But the number 279 * may exceed the capacity of SGEs recorded in the firmware, so the 280 * kernel driver should just adapt the value accordingly. 281 */ 282 if (user) 283 max_sge = roundup_pow_of_two(max_sge + 1); 284 else 285 hr_srq->rsv_sge = 1; 286 287 return max_sge; 288 } 289 290 static int set_srq_basic_param(struct hns_roce_srq *srq, 291 struct ib_srq_init_attr *init_attr, 292 struct ib_udata *udata) 293 { 294 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); 295 struct ib_srq_attr *attr = &init_attr->attr; 296 u32 max_sge; 297 298 max_sge = proc_srq_sge(hr_dev, srq, !!udata); 299 if (attr->max_wr > hr_dev->caps.max_srq_wrs || 300 attr->max_sge > max_sge || !attr->max_sge) { 301 ibdev_err(&hr_dev->ib_dev, 302 "invalid SRQ attr, depth = %u, sge = %u.\n", 303 attr->max_wr, attr->max_sge); 304 return -EINVAL; 305 } 306 307 attr->max_wr = max_t(u32, attr->max_wr, HNS_ROCE_MIN_SRQ_WQE_NUM); 308 srq->wqe_cnt = roundup_pow_of_two(attr->max_wr); 309 srq->max_gs = roundup_pow_of_two(attr->max_sge + srq->rsv_sge); 310 311 attr->max_wr = srq->wqe_cnt; 312 attr->max_sge = srq->max_gs - srq->rsv_sge; 313 attr->srq_limit = 0; 314 315 return 0; 316 } 317 318 static void set_srq_ext_param(struct hns_roce_srq *srq, 319 struct ib_srq_init_attr *init_attr) 320 { 321 srq->cqn = ib_srq_has_cq(init_attr->srq_type) ? 322 to_hr_cq(init_attr->ext.cq)->cqn : 0; 323 324 srq->xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? 325 to_hr_xrcd(init_attr->ext.xrc.xrcd)->xrcdn : 0; 326 } 327 328 static int set_srq_param(struct hns_roce_srq *srq, 329 struct ib_srq_init_attr *init_attr, 330 struct ib_udata *udata) 331 { 332 int ret; 333 334 ret = set_srq_basic_param(srq, init_attr, udata); 335 if (ret) 336 return ret; 337 338 set_srq_ext_param(srq, init_attr); 339 340 return 0; 341 } 342 343 static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, 344 struct ib_udata *udata) 345 { 346 struct hns_roce_ib_create_srq ucmd = {}; 347 int ret; 348 349 if (udata) { 350 ret = ib_copy_from_udata(&ucmd, udata, 351 min(udata->inlen, sizeof(ucmd))); 352 if (ret) { 353 ibdev_err(&hr_dev->ib_dev, 354 "failed to copy SRQ udata, ret = %d.\n", 355 ret); 356 return ret; 357 } 358 } 359 360 ret = alloc_srq_idx(hr_dev, srq, udata, ucmd.que_addr); 361 if (ret) 362 return ret; 363 364 ret = alloc_srq_wqe_buf(hr_dev, srq, udata, ucmd.buf_addr); 365 if (ret) 366 goto err_idx; 367 368 if (!udata) { 369 ret = alloc_srq_wrid(srq); 370 if (ret) 371 goto err_wqe_buf; 372 } 373 374 return 0; 375 376 err_wqe_buf: 377 free_srq_wqe_buf(hr_dev, srq); 378 err_idx: 379 free_srq_idx(hr_dev, srq); 380 381 return ret; 382 } 383 384 static void free_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 385 { 386 free_srq_wrid(srq); 387 free_srq_wqe_buf(hr_dev, srq); 388 free_srq_idx(hr_dev, srq); 389 } 390 391 static int get_srq_ucmd(struct hns_roce_srq *srq, struct ib_udata *udata, 392 struct hns_roce_ib_create_srq *ucmd) 393 { 394 struct ib_device *ibdev = srq->ibsrq.device; 395 int ret; 396 397 ret = ib_copy_from_udata(ucmd, udata, min(udata->inlen, sizeof(*ucmd))); 398 if (ret) { 399 ibdev_err(ibdev, "failed to copy SRQ udata, ret = %d.\n", ret); 400 return ret; 401 } 402 403 return 0; 404 } 405 406 static void free_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, 407 struct ib_udata *udata) 408 { 409 struct hns_roce_ucontext *uctx; 410 411 if (!(srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB)) 412 return; 413 414 srq->cap_flags &= ~HNS_ROCE_SRQ_CAP_RECORD_DB; 415 if (udata) { 416 uctx = rdma_udata_to_drv_context(udata, 417 struct hns_roce_ucontext, 418 ibucontext); 419 hns_roce_db_unmap_user(uctx, &srq->rdb); 420 } else { 421 hns_roce_free_db(hr_dev, &srq->rdb); 422 } 423 } 424 425 static int alloc_srq_db(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, 426 struct ib_udata *udata, 427 struct hns_roce_ib_create_srq_resp *resp) 428 { 429 struct hns_roce_ib_create_srq ucmd = {}; 430 struct hns_roce_ucontext *uctx; 431 int ret; 432 433 if (udata) { 434 ret = get_srq_ucmd(srq, udata, &ucmd); 435 if (ret) 436 return ret; 437 438 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB) && 439 (ucmd.req_cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB)) { 440 uctx = rdma_udata_to_drv_context(udata, 441 struct hns_roce_ucontext, ibucontext); 442 ret = hns_roce_db_map_user(uctx, ucmd.db_addr, 443 &srq->rdb); 444 if (ret) 445 return ret; 446 447 srq->cap_flags |= HNS_ROCE_RSP_SRQ_CAP_RECORD_DB; 448 } 449 } else { 450 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB) { 451 ret = hns_roce_alloc_db(hr_dev, &srq->rdb, 1); 452 if (ret) 453 return ret; 454 455 *srq->rdb.db_record = 0; 456 srq->cap_flags |= HNS_ROCE_RSP_SRQ_CAP_RECORD_DB; 457 } 458 srq->db_reg = hr_dev->reg_base + SRQ_DB_REG; 459 } 460 461 return 0; 462 } 463 464 int hns_roce_create_srq(struct ib_srq *ib_srq, 465 struct ib_srq_init_attr *init_attr, 466 struct ib_udata *udata) 467 { 468 struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device); 469 struct hns_roce_ib_create_srq_resp resp = {}; 470 struct hns_roce_srq *srq = to_hr_srq(ib_srq); 471 int ret; 472 473 mutex_init(&srq->mutex); 474 spin_lock_init(&srq->lock); 475 476 ret = set_srq_param(srq, init_attr, udata); 477 if (ret) 478 goto err_out; 479 480 ret = alloc_srq_buf(hr_dev, srq, udata); 481 if (ret) 482 goto err_out; 483 484 ret = alloc_srq_db(hr_dev, srq, udata, &resp); 485 if (ret) 486 goto err_srq_buf; 487 488 ret = alloc_srqn(hr_dev, srq); 489 if (ret) 490 goto err_srq_db; 491 492 ret = alloc_srqc(hr_dev, srq); 493 if (ret) 494 goto err_srqn; 495 496 if (udata) { 497 resp.cap_flags = srq->cap_flags; 498 resp.srqn = srq->srqn; 499 if (ib_copy_to_udata(udata, &resp, 500 min(udata->outlen, sizeof(resp)))) { 501 ret = -EFAULT; 502 goto err_srqc; 503 } 504 } 505 506 srq->event = hns_roce_ib_srq_event; 507 refcount_set(&srq->refcount, 1); 508 init_completion(&srq->free); 509 510 return 0; 511 512 err_srqc: 513 free_srqc(hr_dev, srq); 514 err_srqn: 515 free_srqn(hr_dev, srq); 516 err_srq_db: 517 free_srq_db(hr_dev, srq, udata); 518 err_srq_buf: 519 free_srq_buf(hr_dev, srq); 520 err_out: 521 mutex_destroy(&srq->mutex); 522 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT]); 523 524 return ret; 525 } 526 527 int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 528 { 529 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 530 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 531 532 free_srqc(hr_dev, srq); 533 free_srqn(hr_dev, srq); 534 free_srq_db(hr_dev, srq, udata); 535 free_srq_buf(hr_dev, srq); 536 mutex_destroy(&srq->mutex); 537 return 0; 538 } 539 540 void hns_roce_init_srq_table(struct hns_roce_dev *hr_dev) 541 { 542 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 543 struct hns_roce_ida *srq_ida = &srq_table->srq_ida; 544 545 xa_init(&srq_table->xa); 546 547 ida_init(&srq_ida->ida); 548 srq_ida->max = hr_dev->caps.num_srqs - 1; 549 srq_ida->min = hr_dev->caps.reserved_srqs; 550 } 551