qp.c (b7d3622a39fde7658170b7f3cf6c6889bb8db30d) qp.c (e1e66cc26457c2e9412f67618646ec2a441fc409)
1/*
2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 202 unchanged lines hidden (view full) ---

211 sizeof(struct mlx5_wqe_raddr_seg);
212 break;
213
214 case IB_QPT_XRC_TGT:
215 return 0;
216
217 case IB_QPT_UC:
218 size += sizeof(struct mlx5_wqe_ctrl_seg) +
1/*
2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 202 unchanged lines hidden (view full) ---

211 sizeof(struct mlx5_wqe_raddr_seg);
212 break;
213
214 case IB_QPT_XRC_TGT:
215 return 0;
216
217 case IB_QPT_UC:
218 size += sizeof(struct mlx5_wqe_ctrl_seg) +
219 sizeof(struct mlx5_wqe_raddr_seg);
219 sizeof(struct mlx5_wqe_raddr_seg) +
220 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
221 sizeof(struct mlx5_mkey_seg);
220 break;
221
222 case IB_QPT_UD:
223 case IB_QPT_SMI:
224 case IB_QPT_GSI:
225 size += sizeof(struct mlx5_wqe_ctrl_seg) +
226 sizeof(struct mlx5_wqe_datagram_seg);
227 break;

--- 21 unchanged lines hidden (view full) ---

249 return size;
250
251 if (attr->cap.max_inline_data) {
252 inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
253 attr->cap.max_inline_data;
254 }
255
256 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
222 break;
223
224 case IB_QPT_UD:
225 case IB_QPT_SMI:
226 case IB_QPT_GSI:
227 size += sizeof(struct mlx5_wqe_ctrl_seg) +
228 sizeof(struct mlx5_wqe_datagram_seg);
229 break;

--- 21 unchanged lines hidden (view full) ---

251 return size;
252
253 if (attr->cap.max_inline_data) {
254 inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
255 attr->cap.max_inline_data;
256 }
257
258 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
257
258 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
259 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN &&
260 ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
261 return MLX5_SIG_WQE_SIZE;
262 else
263 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
259}
260
261static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
262 struct mlx5_ib_qp *qp)
263{
264 int wqe_size;
265 int wq_size;
266

--- 10 unchanged lines hidden (view full) ---

277 wqe_size, dev->mdev.caps.max_sq_desc_sz);
278 return -EINVAL;
279 }
280
281 qp->max_inline_data = wqe_size - sq_overhead(attr->qp_type) -
282 sizeof(struct mlx5_wqe_inline_seg);
283 attr->cap.max_inline_data = qp->max_inline_data;
284
264}
265
266static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
267 struct mlx5_ib_qp *qp)
268{
269 int wqe_size;
270 int wq_size;
271

--- 10 unchanged lines hidden (view full) ---

282 wqe_size, dev->mdev.caps.max_sq_desc_sz);
283 return -EINVAL;
284 }
285
286 qp->max_inline_data = wqe_size - sq_overhead(attr->qp_type) -
287 sizeof(struct mlx5_wqe_inline_seg);
288 attr->cap.max_inline_data = qp->max_inline_data;
289
290 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
291 qp->signature_en = true;
292
285 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
286 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
287 if (qp->sq.wqe_cnt > dev->mdev.caps.max_wqes) {
288 mlx5_ib_dbg(dev, "wqe count(%d) exceeds limits(%d)\n",
289 qp->sq.wqe_cnt, dev->mdev.caps.max_wqes);
290 return -ENOMEM;
291 }
292 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);

--- 42 unchanged lines hidden (view full) ---

335 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
336 attr->qp_type == MLX5_IB_QPT_REG_UMR ||
337 !attr->cap.max_recv_wr)
338 return 0;
339
340 return 1;
341}
342
293 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
294 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
295 if (qp->sq.wqe_cnt > dev->mdev.caps.max_wqes) {
296 mlx5_ib_dbg(dev, "wqe count(%d) exceeds limits(%d)\n",
297 qp->sq.wqe_cnt, dev->mdev.caps.max_wqes);
298 return -ENOMEM;
299 }
300 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);

--- 42 unchanged lines hidden (view full) ---

343 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
344 attr->qp_type == MLX5_IB_QPT_REG_UMR ||
345 !attr->cap.max_recv_wr)
346 return 0;
347
348 return 1;
349}
350
351static int first_med_uuar(void)
352{
353 return 1;
354}
355
356static int next_uuar(int n)
357{
358 n++;
359
360 while (((n % 4) & 2))
361 n++;
362
363 return n;
364}
365
366static int num_med_uuar(struct mlx5_uuar_info *uuari)
367{
368 int n;
369
370 n = uuari->num_uars * MLX5_NON_FP_BF_REGS_PER_PAGE -
371 uuari->num_low_latency_uuars - 1;
372
373 return n >= 0 ? n : 0;
374}
375
376static int max_uuari(struct mlx5_uuar_info *uuari)
377{
378 return uuari->num_uars * 4;
379}
380
381static int first_hi_uuar(struct mlx5_uuar_info *uuari)
382{
383 int med;
384 int i;
385 int t;
386
387 med = num_med_uuar(uuari);
388 for (t = 0, i = first_med_uuar();; i = next_uuar(i)) {
389 t++;
390 if (t == med)
391 return next_uuar(i);
392 }
393
394 return 0;
395}
396
343static int alloc_high_class_uuar(struct mlx5_uuar_info *uuari)
344{
397static int alloc_high_class_uuar(struct mlx5_uuar_info *uuari)
398{
345 int nuuars = uuari->num_uars * MLX5_BF_REGS_PER_PAGE;
346 int start_uuar;
347 int i;
348
399 int i;
400
349 start_uuar = nuuars - uuari->num_low_latency_uuars;
350 for (i = start_uuar; i < nuuars; i++) {
401 for (i = first_hi_uuar(uuari); i < max_uuari(uuari); i = next_uuar(i)) {
351 if (!test_bit(i, uuari->bitmap)) {
352 set_bit(i, uuari->bitmap);
353 uuari->count[i]++;
354 return i;
355 }
356 }
357
358 return -ENOMEM;
359}
360
361static int alloc_med_class_uuar(struct mlx5_uuar_info *uuari)
362{
402 if (!test_bit(i, uuari->bitmap)) {
403 set_bit(i, uuari->bitmap);
404 uuari->count[i]++;
405 return i;
406 }
407 }
408
409 return -ENOMEM;
410}
411
412static int alloc_med_class_uuar(struct mlx5_uuar_info *uuari)
413{
363 int nuuars = uuari->num_uars * MLX5_BF_REGS_PER_PAGE;
364 int minidx = 1;
365 int uuarn;
366 int end;
414 int minidx = first_med_uuar();
367 int i;
368
415 int i;
416
369 end = nuuars - uuari->num_low_latency_uuars;
370
371 for (i = 1; i < end; i++) {
372 uuarn = i & 3;
373 if (uuarn == 2 || uuarn == 3)
374 continue;
375
417 for (i = first_med_uuar(); i < first_hi_uuar(uuari); i = next_uuar(i)) {
376 if (uuari->count[i] < uuari->count[minidx])
377 minidx = i;
378 }
379
380 uuari->count[minidx]++;
381 return minidx;
382}
383

--- 5 unchanged lines hidden (view full) ---

389 mutex_lock(&uuari->lock);
390 switch (lat) {
391 case MLX5_IB_LATENCY_CLASS_LOW:
392 uuarn = 0;
393 uuari->count[uuarn]++;
394 break;
395
396 case MLX5_IB_LATENCY_CLASS_MEDIUM:
418 if (uuari->count[i] < uuari->count[minidx])
419 minidx = i;
420 }
421
422 uuari->count[minidx]++;
423 return minidx;
424}
425

--- 5 unchanged lines hidden (view full) ---

431 mutex_lock(&uuari->lock);
432 switch (lat) {
433 case MLX5_IB_LATENCY_CLASS_LOW:
434 uuarn = 0;
435 uuari->count[uuarn]++;
436 break;
437
438 case MLX5_IB_LATENCY_CLASS_MEDIUM:
397 uuarn = alloc_med_class_uuar(uuari);
439 if (uuari->ver < 2)
440 uuarn = -ENOMEM;
441 else
442 uuarn = alloc_med_class_uuar(uuari);
398 break;
399
400 case MLX5_IB_LATENCY_CLASS_HIGH:
443 break;
444
445 case MLX5_IB_LATENCY_CLASS_HIGH:
401 uuarn = alloc_high_class_uuar(uuari);
446 if (uuari->ver < 2)
447 uuarn = -ENOMEM;
448 else
449 uuarn = alloc_high_class_uuar(uuari);
402 break;
403
404 case MLX5_IB_LATENCY_CLASS_FAST_PATH:
405 uuarn = 2;
406 break;
407 }
408 mutex_unlock(&uuari->lock);
409

--- 74 unchanged lines hidden (view full) ---

484
485static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
486 struct mlx5_ib_qp *qp, struct ib_udata *udata,
487 struct mlx5_create_qp_mbox_in **in,
488 struct mlx5_ib_create_qp_resp *resp, int *inlen)
489{
490 struct mlx5_ib_ucontext *context;
491 struct mlx5_ib_create_qp ucmd;
450 break;
451
452 case MLX5_IB_LATENCY_CLASS_FAST_PATH:
453 uuarn = 2;
454 break;
455 }
456 mutex_unlock(&uuari->lock);
457

--- 74 unchanged lines hidden (view full) ---

532
533static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
534 struct mlx5_ib_qp *qp, struct ib_udata *udata,
535 struct mlx5_create_qp_mbox_in **in,
536 struct mlx5_ib_create_qp_resp *resp, int *inlen)
537{
538 struct mlx5_ib_ucontext *context;
539 struct mlx5_ib_create_qp ucmd;
492 int page_shift;
540 int page_shift = 0;
493 int uar_index;
494 int npages;
541 int uar_index;
542 int npages;
495 u32 offset;
543 u32 offset = 0;
496 int uuarn;
544 int uuarn;
497 int ncont;
545 int ncont = 0;
498 int err;
499
500 err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
501 if (err) {
502 mlx5_ib_dbg(dev, "copy failed\n");
503 return err;
504 }
505
506 context = to_mucontext(pd->uobject->context);
507 /*
508 * TBD: should come from the verbs when we have the API
509 */
510 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH);
511 if (uuarn < 0) {
512 mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n");
546 int err;
547
548 err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
549 if (err) {
550 mlx5_ib_dbg(dev, "copy failed\n");
551 return err;
552 }
553
554 context = to_mucontext(pd->uobject->context);
555 /*
556 * TBD: should come from the verbs when we have the API
557 */
558 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH);
559 if (uuarn < 0) {
560 mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n");
513 mlx5_ib_dbg(dev, "reverting to high latency\n");
514 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_LOW);
561 mlx5_ib_dbg(dev, "reverting to medium latency\n");
562 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_MEDIUM);
515 if (uuarn < 0) {
563 if (uuarn < 0) {
516 mlx5_ib_dbg(dev, "uuar allocation failed\n");
517 return uuarn;
564 mlx5_ib_dbg(dev, "failed to allocate medium latency UUAR\n");
565 mlx5_ib_dbg(dev, "reverting to high latency\n");
566 uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_LOW);
567 if (uuarn < 0) {
568 mlx5_ib_warn(dev, "uuar allocation failed\n");
569 return uuarn;
570 }
518 }
519 }
520
521 uar_index = uuarn_to_uar_index(&context->uuari, uuarn);
522 mlx5_ib_dbg(dev, "uuarn 0x%x, uar_index 0x%x\n", uuarn, uar_index);
523
524 err = set_user_buf_size(dev, qp, &ucmd);
525 if (err)
526 goto err_uuar;
527
571 }
572 }
573
574 uar_index = uuarn_to_uar_index(&context->uuari, uuarn);
575 mlx5_ib_dbg(dev, "uuarn 0x%x, uar_index 0x%x\n", uuarn, uar_index);
576
577 err = set_user_buf_size(dev, qp, &ucmd);
578 if (err)
579 goto err_uuar;
580
528 qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
529 qp->buf_size, 0, 0);
530 if (IS_ERR(qp->umem)) {
531 mlx5_ib_dbg(dev, "umem_get failed\n");
532 err = PTR_ERR(qp->umem);
533 goto err_uuar;
581 if (ucmd.buf_addr && qp->buf_size) {
582 qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
583 qp->buf_size, 0, 0);
584 if (IS_ERR(qp->umem)) {
585 mlx5_ib_dbg(dev, "umem_get failed\n");
586 err = PTR_ERR(qp->umem);
587 goto err_uuar;
588 }
589 } else {
590 qp->umem = NULL;
534 }
535
591 }
592
536 mlx5_ib_cont_pages(qp->umem, ucmd.buf_addr, &npages, &page_shift,
537 &ncont, NULL);
538 err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, &offset);
539 if (err) {
540 mlx5_ib_warn(dev, "bad offset\n");
541 goto err_umem;
593 if (qp->umem) {
594 mlx5_ib_cont_pages(qp->umem, ucmd.buf_addr, &npages, &page_shift,
595 &ncont, NULL);
596 err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, &offset);
597 if (err) {
598 mlx5_ib_warn(dev, "bad offset\n");
599 goto err_umem;
600 }
601 mlx5_ib_dbg(dev, "addr 0x%llx, size %d, npages %d, page_shift %d, ncont %d, offset %d\n",
602 ucmd.buf_addr, qp->buf_size, npages, page_shift, ncont, offset);
542 }
603 }
543 mlx5_ib_dbg(dev, "addr 0x%llx, size %d, npages %d, page_shift %d, ncont %d, offset %d\n",
544 ucmd.buf_addr, qp->buf_size, npages, page_shift, ncont, offset);
545
546 *inlen = sizeof(**in) + sizeof(*(*in)->pas) * ncont;
547 *in = mlx5_vzalloc(*inlen);
548 if (!*in) {
549 err = -ENOMEM;
550 goto err_umem;
551 }
604
605 *inlen = sizeof(**in) + sizeof(*(*in)->pas) * ncont;
606 *in = mlx5_vzalloc(*inlen);
607 if (!*in) {
608 err = -ENOMEM;
609 goto err_umem;
610 }
552 mlx5_ib_populate_pas(dev, qp->umem, page_shift, (*in)->pas, 0);
611 if (qp->umem)
612 mlx5_ib_populate_pas(dev, qp->umem, page_shift, (*in)->pas, 0);
553 (*in)->ctx.log_pg_sz_remote_qpn =
554 cpu_to_be32((page_shift - MLX5_ADAPTER_PAGE_SHIFT) << 24);
555 (*in)->ctx.params2 = cpu_to_be32(offset << 6);
556
557 (*in)->ctx.qp_counter_set_usr_page = cpu_to_be32(uar_index);
558 resp->uuar_index = uuarn;
559 qp->uuarn = uuarn;
560

--- 14 unchanged lines hidden (view full) ---

575
576err_unmap:
577 mlx5_ib_db_unmap_user(context, &qp->db);
578
579err_free:
580 mlx5_vfree(*in);
581
582err_umem:
613 (*in)->ctx.log_pg_sz_remote_qpn =
614 cpu_to_be32((page_shift - MLX5_ADAPTER_PAGE_SHIFT) << 24);
615 (*in)->ctx.params2 = cpu_to_be32(offset << 6);
616
617 (*in)->ctx.qp_counter_set_usr_page = cpu_to_be32(uar_index);
618 resp->uuar_index = uuarn;
619 qp->uuarn = uuarn;
620

--- 14 unchanged lines hidden (view full) ---

635
636err_unmap:
637 mlx5_ib_db_unmap_user(context, &qp->db);
638
639err_free:
640 mlx5_vfree(*in);
641
642err_umem:
583 ib_umem_release(qp->umem);
643 if (qp->umem)
644 ib_umem_release(qp->umem);
584
585err_uuar:
586 free_uuar(&context->uuari, uuarn);
587 return err;
588}
589
590static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp)
591{
592 struct mlx5_ib_ucontext *context;
593
594 context = to_mucontext(pd->uobject->context);
595 mlx5_ib_db_unmap_user(context, &qp->db);
645
646err_uuar:
647 free_uuar(&context->uuari, uuarn);
648 return err;
649}
650
651static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp)
652{
653 struct mlx5_ib_ucontext *context;
654
655 context = to_mucontext(pd->uobject->context);
656 mlx5_ib_db_unmap_user(context, &qp->db);
596 ib_umem_release(qp->umem);
657 if (qp->umem)
658 ib_umem_release(qp->umem);
597 free_uuar(&context->uuari, qp->uuarn);
598}
599
600static int create_kernel_qp(struct mlx5_ib_dev *dev,
601 struct ib_qp_init_attr *init_attr,
602 struct mlx5_ib_qp *qp,
603 struct mlx5_create_qp_mbox_in **in, int *inlen)
604{
605 enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW;
606 struct mlx5_uuar_info *uuari;
607 int uar_index;
608 int uuarn;
609 int err;
610
611 uuari = &dev->mdev.priv.uuari;
659 free_uuar(&context->uuari, qp->uuarn);
660}
661
662static int create_kernel_qp(struct mlx5_ib_dev *dev,
663 struct ib_qp_init_attr *init_attr,
664 struct mlx5_ib_qp *qp,
665 struct mlx5_create_qp_mbox_in **in, int *inlen)
666{
667 enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW;
668 struct mlx5_uuar_info *uuari;
669 int uar_index;
670 int uuarn;
671 int err;
672
673 uuari = &dev->mdev.priv.uuari;
612 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
613 qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK;
674 if (init_attr->create_flags & ~IB_QP_CREATE_SIGNATURE_EN)
675 return -EINVAL;
614
615 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
616 lc = MLX5_IB_LATENCY_CLASS_FAST_PATH;
617
618 uuarn = alloc_uuar(uuari, lc);
619 if (uuarn < 0) {
620 mlx5_ib_dbg(dev, "\n");
621 return -ENOMEM;

--- 989 unchanged lines hidden (view full) ---

1611 int port;
1612
1613 mutex_lock(&qp->mutex);
1614
1615 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
1616 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1617
1618 if (ibqp->qp_type != MLX5_IB_QPT_REG_UMR &&
676
677 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
678 lc = MLX5_IB_LATENCY_CLASS_FAST_PATH;
679
680 uuarn = alloc_uuar(uuari, lc);
681 if (uuarn < 0) {
682 mlx5_ib_dbg(dev, "\n");
683 return -ENOMEM;

--- 989 unchanged lines hidden (view full) ---

1673 int port;
1674
1675 mutex_lock(&qp->mutex);
1676
1677 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
1678 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1679
1680 if (ibqp->qp_type != MLX5_IB_QPT_REG_UMR &&
1619 !ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask))
1681 !ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask,
1682 IB_LINK_LAYER_UNSPECIFIED))
1620 goto out;
1621
1622 if ((attr_mask & IB_QP_PORT) &&
1623 (attr->port_num == 0 || attr->port_num > dev->mdev.caps.num_ports))
1624 goto out;
1625
1626 if (attr_mask & IB_QP_PKEY_INDEX) {
1627 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;

--- 579 unchanged lines hidden (view full) ---

2207
2208 /* Make sure that descriptors are written before
2209 * updating doorbell record and ringing the doorbell
2210 */
2211 wmb();
2212
2213 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
2214
1683 goto out;
1684
1685 if ((attr_mask & IB_QP_PORT) &&
1686 (attr->port_num == 0 || attr->port_num > dev->mdev.caps.num_ports))
1687 goto out;
1688
1689 if (attr_mask & IB_QP_PKEY_INDEX) {
1690 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;

--- 579 unchanged lines hidden (view full) ---

2270
2271 /* Make sure that descriptors are written before
2272 * updating doorbell record and ringing the doorbell
2273 */
2274 wmb();
2275
2276 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
2277
2278 /* Make sure doorbell record is visible to the HCA before
2279 * we hit doorbell */
2280 wmb();
2281
2215 if (bf->need_lock)
2216 spin_lock(&bf->lock);
2217
2218 /* TBD enable WC */
2219 if (0 && nreq == 1 && bf->uuarn && inl && size > 1 && size <= bf->buf_size / 16) {
2220 mlx5_bf_copy(bf->reg + bf->offset, (u64 *)ctrl, ALIGN(size * 16, 64), qp);
2221 /* wc_wmb(); */
2222 } else {

--- 291 unchanged lines hidden --- 		2282 if (bf->need_lock)
2283 spin_lock(&bf->lock);
2284
2285 /* TBD enable WC */
2286 if (0 && nreq == 1 && bf->uuarn && inl && size > 1 && size <= bf->buf_size / 16) {
2287 mlx5_bf_copy(bf->reg + bf->offset, (u64 *)ctrl, ALIGN(size * 16, 64), qp);
2288 /* wc_wmb(); */
2289 } else {

--- 291 unchanged lines hidden ---