1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018 Hisilicon Limited. 4 */ 5 6 #include <linux/pci.h> 7 #include <rdma/ib_umem.h> 8 #include "hns_roce_device.h" 9 #include "hns_roce_cmd.h" 10 #include "hns_roce_hem.h" 11 12 void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type) 13 { 14 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 15 struct hns_roce_srq *srq; 16 17 xa_lock(&srq_table->xa); 18 srq = xa_load(&srq_table->xa, srqn & (hr_dev->caps.num_srqs - 1)); 19 if (srq) 20 refcount_inc(&srq->refcount); 21 xa_unlock(&srq_table->xa); 22 23 if (!srq) { 24 dev_warn(hr_dev->dev, "Async event for bogus SRQ %08x\n", srqn); 25 return; 26 } 27 28 srq->event(srq, event_type); 29 30 if (refcount_dec_and_test(&srq->refcount)) 31 complete(&srq->free); 32 } 33 34 static void hns_roce_ib_srq_event(struct hns_roce_srq *srq, 35 enum hns_roce_event event_type) 36 { 37 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); 38 struct ib_srq *ibsrq = &srq->ibsrq; 39 struct ib_event event; 40 41 if (ibsrq->event_handler) { 42 event.device = ibsrq->device; 43 event.element.srq = ibsrq; 44 switch (event_type) { 45 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 46 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 47 break; 48 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 49 event.event = IB_EVENT_SRQ_ERR; 50 break; 51 default: 52 dev_err(hr_dev->dev, 53 "hns_roce:Unexpected event type 0x%x on SRQ %06lx\n", 54 event_type, srq->srqn); 55 return; 56 } 57 58 ibsrq->event_handler(&event, ibsrq->srq_context); 59 } 60 } 61 62 static int hns_roce_hw_create_srq(struct hns_roce_dev *dev, 63 struct hns_roce_cmd_mailbox *mailbox, 64 unsigned long srq_num) 65 { 66 return hns_roce_cmd_mbox(dev, mailbox->dma, 0, srq_num, 0, 67 HNS_ROCE_CMD_CREATE_SRQ, 68 HNS_ROCE_CMD_TIMEOUT_MSECS); 69 } 70 71 static int hns_roce_hw_destroy_srq(struct hns_roce_dev *dev, 72 struct hns_roce_cmd_mailbox *mailbox, 73 unsigned long srq_num) 74 { 75 return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, srq_num, 76 mailbox ? 0 : 1, HNS_ROCE_CMD_DESTROY_SRQ, 77 HNS_ROCE_CMD_TIMEOUT_MSECS); 78 } 79 80 static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 81 { 82 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 83 struct hns_roce_ida *srq_ida = &hr_dev->srq_table.srq_ida; 84 struct ib_device *ibdev = &hr_dev->ib_dev; 85 struct hns_roce_cmd_mailbox *mailbox; 86 int ret; 87 int id; 88 89 id = ida_alloc_range(&srq_ida->ida, srq_ida->min, srq_ida->max, 90 GFP_KERNEL); 91 if (id < 0) { 92 ibdev_err(ibdev, "failed to alloc srq(%d).\n", id); 93 return -ENOMEM; 94 } 95 srq->srqn = (unsigned long)id; 96 97 ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn); 98 if (ret) { 99 ibdev_err(ibdev, "failed to get SRQC table, ret = %d.\n", ret); 100 goto err_out; 101 } 102 103 ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL)); 104 if (ret) { 105 ibdev_err(ibdev, "failed to store SRQC, ret = %d.\n", ret); 106 goto err_put; 107 } 108 109 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 110 if (IS_ERR_OR_NULL(mailbox)) { 111 ibdev_err(ibdev, "failed to alloc mailbox for SRQC.\n"); 112 ret = -ENOMEM; 113 goto err_xa; 114 } 115 116 ret = hr_dev->hw->write_srqc(srq, mailbox->buf); 117 if (ret) { 118 ibdev_err(ibdev, "failed to write SRQC.\n"); 119 goto err_mbox; 120 } 121 122 ret = hns_roce_hw_create_srq(hr_dev, mailbox, srq->srqn); 123 if (ret) { 124 ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret); 125 goto err_mbox; 126 } 127 128 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 129 130 return 0; 131 132 err_mbox: 133 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 134 err_xa: 135 xa_erase(&srq_table->xa, srq->srqn); 136 err_put: 137 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); 138 err_out: 139 ida_free(&srq_ida->ida, id); 140 141 return ret; 142 } 143 144 static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 145 { 146 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 147 int ret; 148 149 ret = hns_roce_hw_destroy_srq(hr_dev, NULL, srq->srqn); 150 if (ret) 151 dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n", 152 ret, srq->srqn); 153 154 xa_erase(&srq_table->xa, srq->srqn); 155 156 if (refcount_dec_and_test(&srq->refcount)) 157 complete(&srq->free); 158 wait_for_completion(&srq->free); 159 160 hns_roce_table_put(hr_dev, &srq_table->table, srq->srqn); 161 ida_free(&srq_table->srq_ida.ida, (int)srq->srqn); 162 } 163 164 static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, 165 struct ib_udata *udata, unsigned long addr) 166 { 167 struct hns_roce_idx_que *idx_que = &srq->idx_que; 168 struct ib_device *ibdev = &hr_dev->ib_dev; 169 struct hns_roce_buf_attr buf_attr = {}; 170 int ret; 171 172 srq->idx_que.entry_shift = ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ); 173 174 buf_attr.page_shift = hr_dev->caps.idx_buf_pg_sz + PAGE_SHIFT; 175 buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt, 176 srq->idx_que.entry_shift); 177 buf_attr.region[0].hopnum = hr_dev->caps.idx_hop_num; 178 buf_attr.region_count = 1; 179 180 ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr, 181 hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT, 182 udata, addr); 183 if (ret) { 184 ibdev_err(ibdev, 185 "failed to alloc SRQ idx mtr, ret = %d.\n", ret); 186 return ret; 187 } 188 189 if (!udata) { 190 idx_que->bitmap = bitmap_zalloc(srq->wqe_cnt, GFP_KERNEL); 191 if (!idx_que->bitmap) { 192 ibdev_err(ibdev, "failed to alloc SRQ idx bitmap.\n"); 193 ret = -ENOMEM; 194 goto err_idx_mtr; 195 } 196 } 197 198 idx_que->head = 0; 199 idx_que->tail = 0; 200 201 return 0; 202 err_idx_mtr: 203 hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); 204 205 return ret; 206 } 207 208 static void free_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 209 { 210 struct hns_roce_idx_que *idx_que = &srq->idx_que; 211 212 bitmap_free(idx_que->bitmap); 213 idx_que->bitmap = NULL; 214 hns_roce_mtr_destroy(hr_dev, &idx_que->mtr); 215 } 216 217 static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev, 218 struct hns_roce_srq *srq, 219 struct ib_udata *udata, unsigned long addr) 220 { 221 struct ib_device *ibdev = &hr_dev->ib_dev; 222 struct hns_roce_buf_attr buf_attr = {}; 223 int ret; 224 225 srq->wqe_shift = ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE, 226 HNS_ROCE_SGE_SIZE * 227 srq->max_gs))); 228 229 buf_attr.page_shift = hr_dev->caps.srqwqe_buf_pg_sz + PAGE_SHIFT; 230 buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt, 231 srq->wqe_shift); 232 buf_attr.region[0].hopnum = hr_dev->caps.srqwqe_hop_num; 233 buf_attr.region_count = 1; 234 235 ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr, 236 hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT, 237 udata, addr); 238 if (ret) 239 ibdev_err(ibdev, 240 "failed to alloc SRQ buf mtr, ret = %d.\n", ret); 241 242 return ret; 243 } 244 245 static void free_srq_wqe_buf(struct hns_roce_dev *hr_dev, 246 struct hns_roce_srq *srq) 247 { 248 hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr); 249 } 250 251 static int alloc_srq_wrid(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 252 { 253 srq->wrid = kvmalloc_array(srq->wqe_cnt, sizeof(u64), GFP_KERNEL); 254 if (!srq->wrid) 255 return -ENOMEM; 256 257 return 0; 258 } 259 260 static void free_srq_wrid(struct hns_roce_srq *srq) 261 { 262 kvfree(srq->wrid); 263 srq->wrid = NULL; 264 } 265 266 static u32 proc_srq_sge(struct hns_roce_dev *dev, struct hns_roce_srq *hr_srq, 267 bool user) 268 { 269 u32 max_sge = dev->caps.max_srq_sges; 270 271 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 272 return max_sge; 273 274 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will 275 * calculate number of max_sge with reserved SGEs when allocating wqe 276 * buf, so there is no need to do this again in kernel. But the number 277 * may exceed the capacity of SGEs recorded in the firmware, so the 278 * kernel driver should just adapt the value accordingly. 279 */ 280 if (user) 281 max_sge = roundup_pow_of_two(max_sge + 1); 282 else 283 hr_srq->rsv_sge = 1; 284 285 return max_sge; 286 } 287 288 static int set_srq_basic_param(struct hns_roce_srq *srq, 289 struct ib_srq_init_attr *init_attr, 290 struct ib_udata *udata) 291 { 292 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); 293 struct ib_srq_attr *attr = &init_attr->attr; 294 u32 max_sge; 295 296 max_sge = proc_srq_sge(hr_dev, srq, !!udata); 297 if (attr->max_wr > hr_dev->caps.max_srq_wrs || 298 attr->max_sge > max_sge) { 299 ibdev_err(&hr_dev->ib_dev, 300 "invalid SRQ attr, depth = %u, sge = %u.\n", 301 attr->max_wr, attr->max_sge); 302 return -EINVAL; 303 } 304 305 attr->max_wr = max_t(u32, attr->max_wr, HNS_ROCE_MIN_SRQ_WQE_NUM); 306 srq->wqe_cnt = roundup_pow_of_two(attr->max_wr); 307 srq->max_gs = roundup_pow_of_two(attr->max_sge + srq->rsv_sge); 308 309 attr->max_wr = srq->wqe_cnt; 310 attr->max_sge = srq->max_gs - srq->rsv_sge; 311 attr->srq_limit = 0; 312 313 return 0; 314 } 315 316 static void set_srq_ext_param(struct hns_roce_srq *srq, 317 struct ib_srq_init_attr *init_attr) 318 { 319 srq->cqn = ib_srq_has_cq(init_attr->srq_type) ? 320 to_hr_cq(init_attr->ext.cq)->cqn : 0; 321 322 srq->xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? 323 to_hr_xrcd(init_attr->ext.xrc.xrcd)->xrcdn : 0; 324 } 325 326 static int set_srq_param(struct hns_roce_srq *srq, 327 struct ib_srq_init_attr *init_attr, 328 struct ib_udata *udata) 329 { 330 int ret; 331 332 ret = set_srq_basic_param(srq, init_attr, udata); 333 if (ret) 334 return ret; 335 336 set_srq_ext_param(srq, init_attr); 337 338 return 0; 339 } 340 341 static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, 342 struct ib_udata *udata) 343 { 344 struct hns_roce_ib_create_srq ucmd = {}; 345 int ret; 346 347 if (udata) { 348 ret = ib_copy_from_udata(&ucmd, udata, 349 min(udata->inlen, sizeof(ucmd))); 350 if (ret) { 351 ibdev_err(&hr_dev->ib_dev, 352 "failed to copy SRQ udata, ret = %d.\n", 353 ret); 354 return ret; 355 } 356 } 357 358 ret = alloc_srq_idx(hr_dev, srq, udata, ucmd.que_addr); 359 if (ret) 360 return ret; 361 362 ret = alloc_srq_wqe_buf(hr_dev, srq, udata, ucmd.buf_addr); 363 if (ret) 364 goto err_idx; 365 366 if (!udata) { 367 ret = alloc_srq_wrid(hr_dev, srq); 368 if (ret) 369 goto err_wqe_buf; 370 } 371 372 return 0; 373 374 err_wqe_buf: 375 free_srq_wqe_buf(hr_dev, srq); 376 err_idx: 377 free_srq_idx(hr_dev, srq); 378 379 return ret; 380 } 381 382 static void free_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 383 { 384 free_srq_wrid(srq); 385 free_srq_wqe_buf(hr_dev, srq); 386 free_srq_idx(hr_dev, srq); 387 } 388 389 int hns_roce_create_srq(struct ib_srq *ib_srq, 390 struct ib_srq_init_attr *init_attr, 391 struct ib_udata *udata) 392 { 393 struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device); 394 struct hns_roce_ib_create_srq_resp resp = {}; 395 struct hns_roce_srq *srq = to_hr_srq(ib_srq); 396 int ret; 397 398 mutex_init(&srq->mutex); 399 spin_lock_init(&srq->lock); 400 401 ret = set_srq_param(srq, init_attr, udata); 402 if (ret) 403 return ret; 404 405 ret = alloc_srq_buf(hr_dev, srq, udata); 406 if (ret) 407 return ret; 408 409 ret = alloc_srqc(hr_dev, srq); 410 if (ret) 411 goto err_srq_buf; 412 413 if (udata) { 414 resp.srqn = srq->srqn; 415 if (ib_copy_to_udata(udata, &resp, 416 min(udata->outlen, sizeof(resp)))) { 417 ret = -EFAULT; 418 goto err_srqc; 419 } 420 } 421 422 srq->db_reg = hr_dev->reg_base + SRQ_DB_REG; 423 srq->event = hns_roce_ib_srq_event; 424 refcount_set(&srq->refcount, 1); 425 init_completion(&srq->free); 426 427 return 0; 428 429 err_srqc: 430 free_srqc(hr_dev, srq); 431 err_srq_buf: 432 free_srq_buf(hr_dev, srq); 433 434 return ret; 435 } 436 437 int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 438 { 439 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 440 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 441 442 free_srqc(hr_dev, srq); 443 free_srq_buf(hr_dev, srq); 444 return 0; 445 } 446 447 void hns_roce_init_srq_table(struct hns_roce_dev *hr_dev) 448 { 449 struct hns_roce_srq_table *srq_table = &hr_dev->srq_table; 450 struct hns_roce_ida *srq_ida = &srq_table->srq_ida; 451 452 xa_init(&srq_table->xa); 453 454 ida_init(&srq_ida->ida); 455 srq_ida->max = hr_dev->caps.num_srqs - 1; 456 srq_ida->min = hr_dev->caps.reserved_srqs; 457 } 458