1 /*- 2 * Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include "opt_rss.h" 29 #include "opt_ratelimit.h" 30 31 #include <linux/module.h> 32 #include <dev/mlx5/qp.h> 33 #include <dev/mlx5/srq.h> 34 #include <linux/slab.h> 35 #include <rdma/ib_umem.h> 36 #include <rdma/ib_user_verbs.h> 37 #include <rdma/uverbs_ioctl.h> 38 39 #include <dev/mlx5/mlx5_ib/mlx5_ib.h> 40 41 static void *get_wqe(struct mlx5_ib_srq *srq, int n) 42 { 43 return mlx5_buf_offset(&srq->buf, n << srq->msrq.wqe_shift); 44 } 45 46 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, int type) 47 { 48 struct ib_event event; 49 struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 50 51 if (ibsrq->event_handler) { 52 event.device = ibsrq->device; 53 event.element.srq = ibsrq; 54 switch (type) { 55 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: 56 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 57 break; 58 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: 59 event.event = IB_EVENT_SRQ_ERR; 60 break; 61 default: 62 pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n", 63 type, srq->srqn); 64 return; 65 } 66 67 ibsrq->event_handler(&event, ibsrq->srq_context); 68 } 69 } 70 71 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 72 struct mlx5_srq_attr *in, 73 struct ib_udata *udata, int buf_size) 74 { 75 struct mlx5_ib_dev *dev = to_mdev(pd->device); 76 struct mlx5_ib_create_srq ucmd = {}; 77 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context( 78 udata, struct mlx5_ib_ucontext, ibucontext); 79 size_t ucmdlen; 80 int err; 81 int npages; 82 int page_shift; 83 int ncont; 84 u32 offset; 85 u32 uidx = MLX5_IB_DEFAULT_UIDX; 86 87 ucmdlen = min(udata->inlen, sizeof(ucmd)); 88 89 if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) { 90 mlx5_ib_dbg(dev, "failed copy udata\n"); 91 return -EFAULT; 92 } 93 94 if (ucmd.reserved0 || ucmd.reserved1) 95 return -EINVAL; 96 97 if (udata->inlen > sizeof(ucmd) && 98 !ib_is_udata_cleared(udata, sizeof(ucmd), 99 udata->inlen - sizeof(ucmd))) 100 return -EINVAL; 101 102 if (in->type != IB_SRQT_BASIC) { 103 err = get_srq_user_index(ucontext, &ucmd, udata->inlen, &uidx); 104 if (err) 105 return err; 106 } 107 108 srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); 109 110 srq->umem = ib_umem_get(&ucontext->ibucontext, ucmd.buf_addr, buf_size, 0, 0); 111 if (IS_ERR(srq->umem)) { 112 mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); 113 err = PTR_ERR(srq->umem); 114 return err; 115 } 116 117 mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages, 118 &page_shift, &ncont, NULL); 119 err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, 120 &offset); 121 if (err) { 122 mlx5_ib_warn(dev, "bad offset\n"); 123 goto err_umem; 124 } 125 126 in->pas = mlx5_vzalloc(sizeof(*in->pas) * ncont); 127 if (!in->pas) { 128 err = -ENOMEM; 129 goto err_umem; 130 } 131 132 mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0); 133 134 err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &srq->db); 135 if (err) { 136 mlx5_ib_dbg(dev, "map doorbell failed\n"); 137 goto err_in; 138 } 139 140 in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT; 141 in->page_offset = offset; 142 in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0; 143 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 144 in->type != IB_SRQT_BASIC) 145 in->user_index = uidx; 146 147 return 0; 148 149 err_in: 150 kvfree(in->pas); 151 152 err_umem: 153 ib_umem_release(srq->umem); 154 155 return err; 156 } 157 158 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, 159 struct mlx5_srq_attr *in, int buf_size) 160 { 161 int err; 162 int i; 163 struct mlx5_wqe_srq_next_seg *next; 164 165 err = mlx5_db_alloc(dev->mdev, &srq->db); 166 if (err) { 167 mlx5_ib_warn(dev, "alloc dbell rec failed\n"); 168 return err; 169 } 170 171 if (mlx5_buf_alloc(dev->mdev, buf_size, 2 * PAGE_SIZE, &srq->buf)) { 172 mlx5_ib_dbg(dev, "buf alloc failed\n"); 173 err = -ENOMEM; 174 goto err_db; 175 } 176 177 srq->head = 0; 178 srq->tail = srq->msrq.max - 1; 179 srq->wqe_ctr = 0; 180 181 for (i = 0; i < srq->msrq.max; i++) { 182 next = get_wqe(srq, i); 183 next->next_wqe_index = 184 cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 185 } 186 187 mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); 188 in->pas = mlx5_vzalloc(sizeof(*in->pas) * srq->buf.npages); 189 if (!in->pas) { 190 err = -ENOMEM; 191 goto err_buf; 192 } 193 mlx5_fill_page_array(&srq->buf, in->pas); 194 195 srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL); 196 if (!srq->wrid) { 197 err = -ENOMEM; 198 goto err_in; 199 } 200 srq->wq_sig = 0; 201 202 in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; 203 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 204 in->type != IB_SRQT_BASIC) 205 in->user_index = MLX5_IB_DEFAULT_UIDX; 206 207 return 0; 208 209 err_in: 210 kvfree(in->pas); 211 212 err_buf: 213 mlx5_buf_free(dev->mdev, &srq->buf); 214 215 err_db: 216 mlx5_db_free(dev->mdev, &srq->db); 217 return err; 218 } 219 220 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 221 struct ib_udata *udata) 222 { 223 mlx5_ib_db_unmap_user( 224 rdma_udata_to_drv_context( 225 udata, 226 struct mlx5_ib_ucontext, 227 ibucontext), 228 &srq->db); 229 ib_umem_release(srq->umem); 230 } 231 232 233 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) 234 { 235 kfree(srq->wrid); 236 mlx5_buf_free(dev->mdev, &srq->buf); 237 mlx5_db_free(dev->mdev, &srq->db); 238 } 239 240 int mlx5_ib_create_srq(struct ib_srq *ib_srq, 241 struct ib_srq_init_attr *init_attr, 242 struct ib_udata *udata) 243 { 244 struct mlx5_ib_dev *dev = to_mdev(ib_srq->device); 245 struct mlx5_ib_srq *srq = to_msrq(ib_srq); 246 size_t desc_size; 247 size_t buf_size; 248 int err; 249 struct mlx5_srq_attr in = {}; 250 __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); 251 252 /* Sanity check SRQ size before proceeding */ 253 if (init_attr->attr.max_wr >= max_srq_wqes) { 254 mlx5_ib_dbg(dev, "max_wr %d, cap %d\n", 255 init_attr->attr.max_wr, 256 max_srq_wqes); 257 return -EINVAL; 258 } 259 260 mutex_init(&srq->mutex); 261 spin_lock_init(&srq->lock); 262 srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 263 srq->msrq.max_gs = init_attr->attr.max_sge; 264 265 desc_size = sizeof(struct mlx5_wqe_srq_next_seg) + 266 srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); 267 if (desc_size == 0 || srq->msrq.max_gs > desc_size) 268 return -EINVAL; 269 270 desc_size = roundup_pow_of_two(desc_size); 271 desc_size = max_t(size_t, 32, desc_size); 272 if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) 273 return -EINVAL; 274 275 srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / 276 sizeof(struct mlx5_wqe_data_seg); 277 srq->msrq.wqe_shift = ilog2(desc_size); 278 buf_size = srq->msrq.max * desc_size; 279 if (buf_size < desc_size) 280 return -EINVAL; 281 282 in.type = init_attr->srq_type; 283 284 if (udata) 285 err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size); 286 else 287 err = create_srq_kernel(dev, srq, &in, buf_size); 288 289 if (err) { 290 mlx5_ib_warn(dev, "create srq %s failed, err %d\n", 291 udata ? "user" : "kernel", err); 292 return err; 293 } 294 295 in.log_size = ilog2(srq->msrq.max); 296 in.wqe_shift = srq->msrq.wqe_shift - 4; 297 if (srq->wq_sig) 298 in.flags |= MLX5_SRQ_FLAG_WQ_SIG; 299 300 if (init_attr->srq_type == IB_SRQT_XRC) 301 in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn; 302 else 303 in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn; 304 305 if (ib_srq_has_cq(init_attr->srq_type)) 306 in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn; 307 else 308 in.cqn = to_mcq(dev->devr.c0)->mcq.cqn; 309 310 in.pd = to_mpd(ib_srq->pd)->pdn; 311 in.db_record = srq->db.dma; 312 err = mlx5_core_create_srq(dev->mdev, &srq->msrq, &in); 313 kvfree(in.pas); 314 if (err) { 315 mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err); 316 goto err_usr_kern_srq; 317 } 318 319 mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); 320 321 srq->msrq.event = mlx5_ib_srq_event; 322 srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 323 324 if (udata) 325 if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) { 326 mlx5_ib_dbg(dev, "copy to user failed\n"); 327 err = -EFAULT; 328 goto err_core; 329 } 330 331 init_attr->attr.max_wr = srq->msrq.max - 1; 332 333 return 0; 334 335 err_core: 336 mlx5_core_destroy_srq(dev->mdev, &srq->msrq); 337 338 err_usr_kern_srq: 339 if (udata) 340 destroy_srq_user(ib_srq->pd, srq, udata); 341 else 342 destroy_srq_kernel(dev, srq); 343 344 return err; 345 } 346 347 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 348 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 349 { 350 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 351 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 352 int ret; 353 354 /* We don't support resizing SRQs yet */ 355 if (attr_mask & IB_SRQ_MAX_WR) 356 return -EINVAL; 357 358 if (attr_mask & IB_SRQ_LIMIT) { 359 if (attr->srq_limit >= srq->msrq.max) 360 return -EINVAL; 361 362 mutex_lock(&srq->mutex); 363 ret = mlx5_core_arm_srq(dev->mdev, &srq->msrq, attr->srq_limit, 1); 364 mutex_unlock(&srq->mutex); 365 366 if (ret) 367 return ret; 368 } 369 370 return 0; 371 } 372 373 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 374 { 375 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 376 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 377 int ret; 378 struct mlx5_srq_attr *out; 379 380 out = kzalloc(sizeof(*out), GFP_KERNEL); 381 if (!out) 382 return -ENOMEM; 383 384 ret = mlx5_core_query_srq(dev->mdev, &srq->msrq, out); 385 if (ret) 386 goto out_box; 387 388 srq_attr->srq_limit = out->lwm; 389 srq_attr->max_wr = srq->msrq.max - 1; 390 srq_attr->max_sge = srq->msrq.max_gs; 391 392 out_box: 393 kfree(out); 394 return ret; 395 } 396 397 void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) 398 { 399 struct mlx5_ib_dev *dev = to_mdev(srq->device); 400 struct mlx5_ib_srq *msrq = to_msrq(srq); 401 402 mlx5_core_destroy_srq(dev->mdev, &msrq->msrq); 403 404 if (srq->uobject) { 405 mlx5_ib_db_unmap_user( 406 rdma_udata_to_drv_context( 407 udata, 408 struct mlx5_ib_ucontext, 409 ibucontext), 410 &msrq->db); 411 ib_umem_release(msrq->umem); 412 } else { 413 destroy_srq_kernel(dev, msrq); 414 } 415 } 416 417 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) 418 { 419 struct mlx5_wqe_srq_next_seg *next; 420 421 /* always called with interrupts disabled. */ 422 spin_lock(&srq->lock); 423 424 next = get_wqe(srq, srq->tail); 425 next->next_wqe_index = cpu_to_be16(wqe_index); 426 srq->tail = wqe_index; 427 428 spin_unlock(&srq->lock); 429 } 430 431 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 432 const struct ib_recv_wr **bad_wr) 433 { 434 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 435 struct mlx5_wqe_srq_next_seg *next; 436 struct mlx5_wqe_data_seg *scat; 437 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 438 struct mlx5_core_dev *mdev = dev->mdev; 439 unsigned long flags; 440 int err = 0; 441 int nreq; 442 int i; 443 444 spin_lock_irqsave(&srq->lock, flags); 445 446 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 447 err = -EIO; 448 *bad_wr = wr; 449 goto out; 450 } 451 452 for (nreq = 0; wr; nreq++, wr = wr->next) { 453 if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 454 err = -EINVAL; 455 *bad_wr = wr; 456 break; 457 } 458 459 if (unlikely(srq->head == srq->tail)) { 460 err = -ENOMEM; 461 *bad_wr = wr; 462 break; 463 } 464 465 srq->wrid[srq->head] = wr->wr_id; 466 467 next = get_wqe(srq, srq->head); 468 srq->head = be16_to_cpu(next->next_wqe_index); 469 scat = (struct mlx5_wqe_data_seg *)(next + 1); 470 471 for (i = 0; i < wr->num_sge; i++) { 472 scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 473 scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 474 scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 475 } 476 477 if (i < srq->msrq.max_avail_gather) { 478 scat[i].byte_count = 0; 479 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 480 scat[i].addr = 0; 481 } 482 } 483 484 if (likely(nreq)) { 485 srq->wqe_ctr += nreq; 486 487 /* Make sure that descriptors are written before 488 * doorbell record. 489 */ 490 wmb(); 491 492 *srq->db.db = cpu_to_be32(srq->wqe_ctr); 493 } 494 out: 495 spin_unlock_irqrestore(&srq->lock, flags); 496 497 return err; 498 } 499