1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/mlx5/qp.h> 7 #include <linux/slab.h> 8 #include <rdma/ib_umem.h> 9 #include <rdma/ib_user_verbs.h> 10 #include "mlx5_ib.h" 11 #include "srq.h" 12 13 static void *get_wqe(struct mlx5_ib_srq *srq, int n) 14 { 15 return mlx5_frag_buf_get_wqe(&srq->fbc, n); 16 } 17 18 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type) 19 { 20 struct ib_event event; 21 struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 22 23 if (ibsrq->event_handler) { 24 event.device = ibsrq->device; 25 event.element.srq = ibsrq; 26 switch (type) { 27 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: 28 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 29 break; 30 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: 31 event.event = IB_EVENT_SRQ_ERR; 32 break; 33 default: 34 pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n", 35 type, srq->srqn); 36 return; 37 } 38 39 ibsrq->event_handler(&event, ibsrq->srq_context); 40 } 41 } 42 43 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 44 struct mlx5_srq_attr *in, 45 struct ib_udata *udata, int buf_size) 46 { 47 struct mlx5_ib_dev *dev = to_mdev(pd->device); 48 struct mlx5_ib_create_srq ucmd; 49 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context( 50 udata, struct mlx5_ib_ucontext, ibucontext); 51 int err; 52 u32 uidx = MLX5_IB_DEFAULT_UIDX; 53 54 err = ib_copy_validate_udata_in(udata, ucmd, flags); 55 if (err) 56 return err; 57 58 if (ucmd.reserved0 || ucmd.reserved1) 59 return -EINVAL; 60 61 if (in->type != IB_SRQT_BASIC) { 62 err = get_srq_user_index(ucontext, &ucmd, udata->inlen, &uidx); 63 if (err) 64 return err; 65 } 66 67 srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); 68 69 srq->umem = ib_umem_get(pd->device, ucmd.buf_addr, buf_size, 0); 70 if (IS_ERR(srq->umem)) { 71 mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); 72 err = PTR_ERR(srq->umem); 73 return err; 74 } 75 in->umem = srq->umem; 76 77 err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &srq->db); 78 if (err) { 79 mlx5_ib_dbg(dev, "map doorbell failed\n"); 80 goto err_umem; 81 } 82 83 in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0; 84 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 85 in->type != IB_SRQT_BASIC) 86 in->user_index = uidx; 87 88 return 0; 89 90 err_umem: 91 ib_umem_release(srq->umem); 92 93 return err; 94 } 95 96 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, 97 struct mlx5_srq_attr *in, int buf_size) 98 { 99 int err; 100 int i; 101 struct mlx5_wqe_srq_next_seg *next; 102 103 err = mlx5_db_alloc(dev->mdev, &srq->db); 104 if (err) { 105 mlx5_ib_warn(dev, "alloc dbell rec failed\n"); 106 return err; 107 } 108 109 if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf, 110 dev->mdev->priv.numa_node)) { 111 mlx5_ib_dbg(dev, "buf alloc failed\n"); 112 err = -ENOMEM; 113 goto err_db; 114 } 115 116 mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max), 117 &srq->fbc); 118 119 srq->head = 0; 120 srq->tail = srq->msrq.max - 1; 121 srq->wqe_ctr = 0; 122 123 for (i = 0; i < srq->msrq.max; i++) { 124 next = get_wqe(srq, i); 125 next->next_wqe_index = 126 cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 127 } 128 129 mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); 130 in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL); 131 if (!in->pas) { 132 err = -ENOMEM; 133 goto err_buf; 134 } 135 mlx5_fill_page_frag_array(&srq->buf, in->pas); 136 137 srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL); 138 if (!srq->wrid) { 139 err = -ENOMEM; 140 goto err_in; 141 } 142 srq->wq_sig = 0; 143 144 in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; 145 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 146 in->type != IB_SRQT_BASIC) 147 in->user_index = MLX5_IB_DEFAULT_UIDX; 148 149 return 0; 150 151 err_in: 152 kvfree(in->pas); 153 154 err_buf: 155 mlx5_frag_buf_free(dev->mdev, &srq->buf); 156 157 err_db: 158 mlx5_db_free(dev->mdev, &srq->db); 159 return err; 160 } 161 162 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 163 struct ib_udata *udata) 164 { 165 mlx5_ib_db_unmap_user( 166 rdma_udata_to_drv_context( 167 udata, 168 struct mlx5_ib_ucontext, 169 ibucontext), 170 &srq->db); 171 ib_umem_release(srq->umem); 172 } 173 174 175 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) 176 { 177 kvfree(srq->wrid); 178 mlx5_frag_buf_free(dev->mdev, &srq->buf); 179 mlx5_db_free(dev->mdev, &srq->db); 180 } 181 182 int mlx5_ib_create_srq(struct ib_srq *ib_srq, 183 struct ib_srq_init_attr *init_attr, 184 struct ib_udata *udata) 185 { 186 struct mlx5_ib_dev *dev = to_mdev(ib_srq->device); 187 struct mlx5_ib_srq *srq = to_msrq(ib_srq); 188 size_t desc_size; 189 size_t buf_size; 190 int err; 191 struct mlx5_srq_attr in = {}; 192 __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); 193 __u32 max_sge_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq) / 194 sizeof(struct mlx5_wqe_data_seg); 195 196 if (init_attr->srq_type != IB_SRQT_BASIC && 197 init_attr->srq_type != IB_SRQT_XRC && 198 init_attr->srq_type != IB_SRQT_TM) 199 return -EOPNOTSUPP; 200 201 /* Sanity check SRQ and sge size before proceeding */ 202 if (init_attr->attr.max_wr >= max_srq_wqes || 203 init_attr->attr.max_sge > max_sge_sz) { 204 mlx5_ib_dbg(dev, "max_wr %d,wr_cap %d,max_sge %d, sge_cap:%d\n", 205 init_attr->attr.max_wr, max_srq_wqes, 206 init_attr->attr.max_sge, max_sge_sz); 207 return -EINVAL; 208 } 209 210 err = mlx5_ib_dev_res_cq_init(dev); 211 if (err) 212 return err; 213 214 mutex_init(&srq->mutex); 215 spin_lock_init(&srq->lock); 216 srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 217 srq->msrq.max_gs = init_attr->attr.max_sge; 218 219 desc_size = sizeof(struct mlx5_wqe_srq_next_seg) + 220 srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); 221 if (desc_size == 0 || srq->msrq.max_gs > desc_size) 222 return -EINVAL; 223 224 desc_size = roundup_pow_of_two(desc_size); 225 desc_size = max_t(size_t, 32, desc_size); 226 if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) 227 return -EINVAL; 228 229 srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / 230 sizeof(struct mlx5_wqe_data_seg); 231 srq->msrq.wqe_shift = ilog2(desc_size); 232 buf_size = srq->msrq.max * desc_size; 233 if (buf_size < desc_size) 234 return -EINVAL; 235 236 in.type = init_attr->srq_type; 237 238 if (udata) 239 err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size); 240 else 241 err = create_srq_kernel(dev, srq, &in, buf_size); 242 243 if (err) { 244 mlx5_ib_warn(dev, "create srq %s failed, err %d\n", 245 udata ? "user" : "kernel", err); 246 return err; 247 } 248 249 in.log_size = ilog2(srq->msrq.max); 250 in.wqe_shift = srq->msrq.wqe_shift - 4; 251 if (srq->wq_sig) 252 in.flags |= MLX5_SRQ_FLAG_WQ_SIG; 253 254 if (init_attr->srq_type == IB_SRQT_XRC && init_attr->ext.xrc.xrcd) 255 in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn; 256 else 257 in.xrcd = dev->devr.xrcdn0; 258 259 if (init_attr->srq_type == IB_SRQT_TM) { 260 in.tm_log_list_size = 261 ilog2(init_attr->ext.tag_matching.max_num_tags) + 1; 262 if (in.tm_log_list_size > 263 MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) { 264 mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n"); 265 err = -EINVAL; 266 goto err_usr_kern_srq; 267 } 268 in.flags |= MLX5_SRQ_FLAG_RNDV; 269 } 270 271 if (ib_srq_has_cq(init_attr->srq_type)) 272 in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn; 273 else 274 in.cqn = to_mcq(dev->devr.c0)->mcq.cqn; 275 276 in.pd = to_mpd(ib_srq->pd)->pdn; 277 in.db_record = srq->db.dma; 278 err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); 279 kvfree(in.pas); 280 if (err) { 281 mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err); 282 goto err_usr_kern_srq; 283 } 284 285 mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); 286 287 srq->msrq.event = mlx5_ib_srq_event; 288 srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 289 290 if (udata) { 291 struct mlx5_ib_create_srq_resp resp = { 292 .srqn = srq->msrq.srqn, 293 }; 294 295 if (ib_copy_to_udata(udata, &resp, min(udata->outlen, 296 sizeof(resp)))) { 297 mlx5_ib_dbg(dev, "copy to user failed\n"); 298 err = -EFAULT; 299 goto err_core; 300 } 301 } 302 303 init_attr->attr.max_wr = srq->msrq.max - 1; 304 305 return 0; 306 307 err_core: 308 mlx5_cmd_destroy_srq(dev, &srq->msrq); 309 310 err_usr_kern_srq: 311 if (udata) 312 destroy_srq_user(ib_srq->pd, srq, udata); 313 else 314 destroy_srq_kernel(dev, srq); 315 316 return err; 317 } 318 319 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 320 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 321 { 322 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 323 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 324 int ret; 325 326 /* We don't support resizing SRQs yet */ 327 if (attr_mask & IB_SRQ_MAX_WR) 328 return -EINVAL; 329 330 if (attr_mask & IB_SRQ_LIMIT) { 331 if (attr->srq_limit >= srq->msrq.max) 332 return -EINVAL; 333 334 mutex_lock(&srq->mutex); 335 ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1); 336 mutex_unlock(&srq->mutex); 337 338 if (ret) 339 return ret; 340 } 341 342 return 0; 343 } 344 345 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 346 { 347 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 348 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 349 int ret; 350 struct mlx5_srq_attr *out; 351 352 out = kzalloc_obj(*out); 353 if (!out) 354 return -ENOMEM; 355 356 ret = mlx5_cmd_query_srq(dev, &srq->msrq, out); 357 if (ret) 358 goto out_box; 359 360 srq_attr->srq_limit = out->lwm; 361 srq_attr->max_wr = srq->msrq.max - 1; 362 srq_attr->max_sge = srq->msrq.max_gs; 363 364 out_box: 365 kfree(out); 366 return ret; 367 } 368 369 int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) 370 { 371 struct mlx5_ib_dev *dev = to_mdev(srq->device); 372 struct mlx5_ib_srq *msrq = to_msrq(srq); 373 int ret; 374 375 ret = mlx5_cmd_destroy_srq(dev, &msrq->msrq); 376 if (ret) 377 return ret; 378 379 if (udata) 380 destroy_srq_user(srq->pd, msrq, udata); 381 else 382 destroy_srq_kernel(dev, msrq); 383 return 0; 384 } 385 386 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) 387 { 388 struct mlx5_wqe_srq_next_seg *next; 389 390 /* always called with interrupts disabled. */ 391 spin_lock(&srq->lock); 392 393 next = get_wqe(srq, srq->tail); 394 next->next_wqe_index = cpu_to_be16(wqe_index); 395 srq->tail = wqe_index; 396 397 spin_unlock(&srq->lock); 398 } 399 400 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 401 const struct ib_recv_wr **bad_wr) 402 { 403 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 404 struct mlx5_wqe_srq_next_seg *next; 405 struct mlx5_wqe_data_seg *scat; 406 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 407 struct mlx5_core_dev *mdev = dev->mdev; 408 unsigned long flags; 409 int err = 0; 410 int nreq; 411 int i; 412 413 spin_lock_irqsave(&srq->lock, flags); 414 415 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 416 err = -EIO; 417 *bad_wr = wr; 418 goto out; 419 } 420 421 for (nreq = 0; wr; nreq++, wr = wr->next) { 422 if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 423 err = -EINVAL; 424 *bad_wr = wr; 425 break; 426 } 427 428 if (unlikely(srq->head == srq->tail)) { 429 err = -ENOMEM; 430 *bad_wr = wr; 431 break; 432 } 433 434 srq->wrid[srq->head] = wr->wr_id; 435 436 next = get_wqe(srq, srq->head); 437 srq->head = be16_to_cpu(next->next_wqe_index); 438 scat = (struct mlx5_wqe_data_seg *)(next + 1); 439 440 for (i = 0; i < wr->num_sge; i++) { 441 scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 442 scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 443 scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 444 } 445 446 if (i < srq->msrq.max_avail_gather) { 447 scat[i].byte_count = 0; 448 scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey; 449 scat[i].addr = 0; 450 } 451 } 452 453 if (likely(nreq)) { 454 srq->wqe_ctr += nreq; 455 456 /* Make sure that descriptors are written before 457 * doorbell record. 458 */ 459 wmb(); 460 461 *srq->db.db = cpu_to_be32(srq->wqe_ctr); 462 } 463 out: 464 spin_unlock_irqrestore(&srq->lock, flags); 465 466 return err; 467 } 468