Lines Matching full:lnk
523 void smc_wr_remember_qp_attr(struct smc_link *lnk) in smc_wr_remember_qp_attr() argument
525 struct ib_qp_attr *attr = &lnk->qp_attr; in smc_wr_remember_qp_attr()
530 ib_query_qp(lnk->roce_qp, attr, in smc_wr_remember_qp_attr()
550 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, in smc_wr_remember_qp_attr()
551 lnk->qp_attr.cap.max_send_wr); in smc_wr_remember_qp_attr()
552 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, in smc_wr_remember_qp_attr()
553 lnk->qp_attr.cap.max_recv_wr); in smc_wr_remember_qp_attr()
556 static void smc_wr_init_sge(struct smc_link *lnk) in smc_wr_init_sge() argument
558 int sges_per_buf = (lnk->lgr->smc_version == SMC_V2) ? 2 : 1; in smc_wr_init_sge()
559 bool send_inline = (lnk->qp_attr.cap.max_inline_data > SMC_WR_TX_SIZE); in smc_wr_init_sge()
562 for (i = 0; i < lnk->wr_tx_cnt; i++) { in smc_wr_init_sge()
563 lnk->wr_tx_sges[i].addr = send_inline ? (uintptr_t)(&lnk->wr_tx_bufs[i]) : in smc_wr_init_sge()
564 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; in smc_wr_init_sge()
565 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; in smc_wr_init_sge()
566 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
567 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey = in smc_wr_init_sge()
568 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
569 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey = in smc_wr_init_sge()
570 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
571 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey = in smc_wr_init_sge()
572 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
573 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey = in smc_wr_init_sge()
574 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
575 lnk->wr_tx_ibs[i].next = NULL; in smc_wr_init_sge()
576 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; in smc_wr_init_sge()
577 lnk->wr_tx_ibs[i].num_sge = 1; in smc_wr_init_sge()
578 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; in smc_wr_init_sge()
579 lnk->wr_tx_ibs[i].send_flags = in smc_wr_init_sge()
582 lnk->wr_tx_ibs[i].send_flags |= IB_SEND_INLINE; in smc_wr_init_sge()
583 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE; in smc_wr_init_sge()
584 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE; in smc_wr_init_sge()
585 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list = in smc_wr_init_sge()
586 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge; in smc_wr_init_sge()
587 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list = in smc_wr_init_sge()
588 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge; in smc_wr_init_sge()
591 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_init_sge()
592 lnk->wr_tx_v2_sge->addr = lnk->wr_tx_v2_dma_addr; in smc_wr_init_sge()
593 lnk->wr_tx_v2_sge->length = SMC_WR_BUF_V2_SIZE; in smc_wr_init_sge()
594 lnk->wr_tx_v2_sge->lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
596 lnk->wr_tx_v2_ib->next = NULL; in smc_wr_init_sge()
597 lnk->wr_tx_v2_ib->sg_list = lnk->wr_tx_v2_sge; in smc_wr_init_sge()
598 lnk->wr_tx_v2_ib->num_sge = 1; in smc_wr_init_sge()
599 lnk->wr_tx_v2_ib->opcode = IB_WR_SEND; in smc_wr_init_sge()
600 lnk->wr_tx_v2_ib->send_flags = in smc_wr_init_sge()
610 for (i = 0; i < lnk->wr_rx_cnt; i++) { in smc_wr_init_sge()
613 lnk->wr_rx_sges[x].addr = in smc_wr_init_sge()
614 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; in smc_wr_init_sge()
615 lnk->wr_rx_sges[x].length = SMC_WR_TX_SIZE; in smc_wr_init_sge()
616 lnk->wr_rx_sges[x].lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
617 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_init_sge()
618 lnk->wr_rx_sges[x + 1].addr = in smc_wr_init_sge()
619 lnk->wr_rx_v2_dma_addr + SMC_WR_TX_SIZE; in smc_wr_init_sge()
620 lnk->wr_rx_sges[x + 1].length = in smc_wr_init_sge()
622 lnk->wr_rx_sges[x + 1].lkey = in smc_wr_init_sge()
623 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
625 lnk->wr_rx_ibs[i].next = NULL; in smc_wr_init_sge()
626 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[x]; in smc_wr_init_sge()
627 lnk->wr_rx_ibs[i].num_sge = sges_per_buf; in smc_wr_init_sge()
629 lnk->wr_reg.wr.next = NULL; in smc_wr_init_sge()
630 lnk->wr_reg.wr.num_sge = 0; in smc_wr_init_sge()
631 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; in smc_wr_init_sge()
632 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; in smc_wr_init_sge()
633 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; in smc_wr_init_sge()
636 void smc_wr_free_link(struct smc_link *lnk) in smc_wr_free_link() argument
640 if (!lnk->smcibdev) in smc_wr_free_link()
642 ibdev = lnk->smcibdev->ibdev; in smc_wr_free_link()
644 smc_wr_drain_cq(lnk); in smc_wr_free_link()
645 smc_wr_wakeup_reg_wait(lnk); in smc_wr_free_link()
646 smc_wr_wakeup_tx_wait(lnk); in smc_wr_free_link()
648 smc_wr_tx_wait_no_pending_sends(lnk); in smc_wr_free_link()
649 percpu_ref_kill(&lnk->wr_reg_refs); in smc_wr_free_link()
650 wait_for_completion(&lnk->reg_ref_comp); in smc_wr_free_link()
651 percpu_ref_kill(&lnk->wr_tx_refs); in smc_wr_free_link()
652 wait_for_completion(&lnk->tx_ref_comp); in smc_wr_free_link()
654 if (lnk->wr_rx_dma_addr) { in smc_wr_free_link()
655 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, in smc_wr_free_link()
656 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_free_link()
658 lnk->wr_rx_dma_addr = 0; in smc_wr_free_link()
660 if (lnk->wr_rx_v2_dma_addr) { in smc_wr_free_link()
661 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, in smc_wr_free_link()
664 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_free_link()
666 if (lnk->wr_tx_dma_addr) { in smc_wr_free_link()
667 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, in smc_wr_free_link()
668 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, in smc_wr_free_link()
670 lnk->wr_tx_dma_addr = 0; in smc_wr_free_link()
672 if (lnk->wr_tx_v2_dma_addr) { in smc_wr_free_link()
673 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, in smc_wr_free_link()
676 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_free_link()
691 void smc_wr_free_link_mem(struct smc_link *lnk) in smc_wr_free_link_mem() argument
693 kfree(lnk->wr_tx_v2_ib); in smc_wr_free_link_mem()
694 lnk->wr_tx_v2_ib = NULL; in smc_wr_free_link_mem()
695 kfree(lnk->wr_tx_v2_sge); in smc_wr_free_link_mem()
696 lnk->wr_tx_v2_sge = NULL; in smc_wr_free_link_mem()
697 kfree(lnk->wr_tx_v2_pend); in smc_wr_free_link_mem()
698 lnk->wr_tx_v2_pend = NULL; in smc_wr_free_link_mem()
699 kfree(lnk->wr_tx_compl); in smc_wr_free_link_mem()
700 lnk->wr_tx_compl = NULL; in smc_wr_free_link_mem()
701 kfree(lnk->wr_tx_pends); in smc_wr_free_link_mem()
702 lnk->wr_tx_pends = NULL; in smc_wr_free_link_mem()
703 bitmap_free(lnk->wr_tx_mask); in smc_wr_free_link_mem()
704 lnk->wr_tx_mask = NULL; in smc_wr_free_link_mem()
705 kfree(lnk->wr_tx_sges); in smc_wr_free_link_mem()
706 lnk->wr_tx_sges = NULL; in smc_wr_free_link_mem()
707 kfree(lnk->wr_tx_rdma_sges); in smc_wr_free_link_mem()
708 lnk->wr_tx_rdma_sges = NULL; in smc_wr_free_link_mem()
709 kfree(lnk->wr_rx_sges); in smc_wr_free_link_mem()
710 lnk->wr_rx_sges = NULL; in smc_wr_free_link_mem()
711 kfree(lnk->wr_tx_rdmas); in smc_wr_free_link_mem()
712 lnk->wr_tx_rdmas = NULL; in smc_wr_free_link_mem()
713 kfree(lnk->wr_rx_ibs); in smc_wr_free_link_mem()
714 lnk->wr_rx_ibs = NULL; in smc_wr_free_link_mem()
715 kfree(lnk->wr_tx_ibs); in smc_wr_free_link_mem()
716 lnk->wr_tx_ibs = NULL; in smc_wr_free_link_mem()
717 kfree(lnk->wr_tx_bufs); in smc_wr_free_link_mem()
718 lnk->wr_tx_bufs = NULL; in smc_wr_free_link_mem()
719 kfree(lnk->wr_rx_bufs); in smc_wr_free_link_mem()
720 lnk->wr_rx_bufs = NULL; in smc_wr_free_link_mem()
853 struct smc_link *lnk = container_of(ref, struct smc_link, wr_tx_refs); in smcr_wr_tx_refs_free() local
855 complete(&lnk->tx_ref_comp); in smcr_wr_tx_refs_free()
860 struct smc_link *lnk = container_of(ref, struct smc_link, wr_reg_refs); in smcr_wr_reg_refs_free() local
862 complete(&lnk->reg_ref_comp); in smcr_wr_reg_refs_free()
865 int smc_wr_create_link(struct smc_link *lnk) in smc_wr_create_link() argument
867 struct ib_device *ibdev = lnk->smcibdev->ibdev; in smc_wr_create_link()
870 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); in smc_wr_create_link()
871 lnk->wr_rx_id = 0; in smc_wr_create_link()
872 lnk->wr_rx_dma_addr = ib_dma_map_single( in smc_wr_create_link()
873 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_create_link()
875 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { in smc_wr_create_link()
876 lnk->wr_rx_dma_addr = 0; in smc_wr_create_link()
880 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_create_link()
881 lnk->wr_rx_v2_dma_addr = ib_dma_map_single(ibdev, in smc_wr_create_link()
882 lnk->lgr->wr_rx_buf_v2, SMC_WR_BUF_V2_SIZE, in smc_wr_create_link()
884 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_v2_dma_addr)) { in smc_wr_create_link()
885 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_create_link()
889 lnk->wr_tx_v2_dma_addr = ib_dma_map_single(ibdev, in smc_wr_create_link()
890 lnk->lgr->wr_tx_buf_v2, SMC_WR_BUF_V2_SIZE, in smc_wr_create_link()
892 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_v2_dma_addr)) { in smc_wr_create_link()
893 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_create_link()
898 lnk->wr_tx_dma_addr = ib_dma_map_single( in smc_wr_create_link()
899 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, in smc_wr_create_link()
901 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { in smc_wr_create_link()
905 smc_wr_init_sge(lnk); in smc_wr_create_link()
906 bitmap_zero(lnk->wr_tx_mask, SMC_WR_BUF_CNT); in smc_wr_create_link()
907 init_waitqueue_head(&lnk->wr_tx_wait); in smc_wr_create_link()
908 rc = percpu_ref_init(&lnk->wr_tx_refs, smcr_wr_tx_refs_free, 0, GFP_KERNEL); in smc_wr_create_link()
911 init_completion(&lnk->tx_ref_comp); in smc_wr_create_link()
912 init_waitqueue_head(&lnk->wr_reg_wait); in smc_wr_create_link()
913 rc = percpu_ref_init(&lnk->wr_reg_refs, smcr_wr_reg_refs_free, 0, GFP_KERNEL); in smc_wr_create_link()
916 init_completion(&lnk->reg_ref_comp); in smc_wr_create_link()
917 init_waitqueue_head(&lnk->wr_rx_empty_wait); in smc_wr_create_link()
921 if (lnk->wr_rx_v2_dma_addr) { in smc_wr_create_link()
922 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, in smc_wr_create_link()
925 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_create_link()
927 if (lnk->wr_tx_v2_dma_addr) { in smc_wr_create_link()
928 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, in smc_wr_create_link()
931 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_create_link()
933 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, in smc_wr_create_link()
934 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_create_link()
936 lnk->wr_rx_dma_addr = 0; in smc_wr_create_link()