1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 8 /* Copyright (c) 2008-2019, IBM Corporation */ 9 10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */ 11 12 #include <linux/vmalloc.h> 13 #include <net/addrconf.h> 14 #include <rdma/erdma-abi.h> 15 #include <rdma/ib_umem.h> 16 #include <rdma/uverbs_ioctl.h> 17 18 #include "erdma.h" 19 #include "erdma_cm.h" 20 #include "erdma_verbs.h" 21 22 static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg, 23 u64 *addr0, u64 *addr1) 24 { 25 struct erdma_mtt *mtt = mem->mtt; 26 27 if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) { 28 *addr0 = mtt->buf_dma; 29 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK, 30 ERDMA_MR_MTT_1LEVEL); 31 } else { 32 *addr0 = mtt->buf[0]; 33 memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1)); 34 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK, 35 ERDMA_MR_MTT_0LEVEL); 36 } 37 } 38 39 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) 40 { 41 struct erdma_dev *dev = to_edev(qp->ibqp.device); 42 struct erdma_pd *pd = to_epd(qp->ibqp.pd); 43 struct erdma_cmdq_create_qp_req req; 44 struct erdma_uqp *user_qp; 45 u64 resp0, resp1; 46 int err; 47 48 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 49 CMDQ_OPCODE_CREATE_QP); 50 51 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK, 52 ilog2(qp->attrs.sq_size)) | 53 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp)); 54 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK, 55 ilog2(qp->attrs.rq_size)) | 56 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn); 57 58 if (rdma_is_kernel_res(&qp->ibqp.res)) { 59 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT; 60 61 req.sq_cqn_mtt_cfg = 62 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 63 pgsz_range) | 64 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 65 req.rq_cqn_mtt_cfg = 66 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 67 pgsz_range) | 68 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 69 70 req.sq_mtt_cfg = 71 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) | 72 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) | 73 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK, 74 ERDMA_MR_MTT_0LEVEL); 75 req.rq_mtt_cfg = req.sq_mtt_cfg; 76 77 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; 78 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; 79 req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr + 80 (qp->attrs.sq_size << SQEBB_SHIFT); 81 req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr + 82 (qp->attrs.rq_size << RQE_SHIFT); 83 } else { 84 user_qp = &qp->user_qp; 85 req.sq_cqn_mtt_cfg = FIELD_PREP( 86 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 87 ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT); 88 req.sq_cqn_mtt_cfg |= 89 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 90 91 req.rq_cqn_mtt_cfg = FIELD_PREP( 92 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 93 ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT); 94 req.rq_cqn_mtt_cfg |= 95 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 96 97 req.sq_mtt_cfg = user_qp->sq_mem.page_offset; 98 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 99 user_qp->sq_mem.mtt_nents); 100 101 req.rq_mtt_cfg = user_qp->rq_mem.page_offset; 102 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 103 user_qp->rq_mem.mtt_nents); 104 105 assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg, 106 &req.sq_buf_addr, req.sq_mtt_entry); 107 assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg, 108 &req.rq_buf_addr, req.rq_mtt_entry); 109 110 req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr; 111 req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr; 112 113 if (uctx->ext_db.enable) { 114 req.sq_cqn_mtt_cfg |= 115 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1); 116 req.db_cfg = 117 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK, 118 uctx->ext_db.sdb_off) | 119 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK, 120 uctx->ext_db.rdb_off); 121 } 122 } 123 124 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, 125 &resp1); 126 if (!err) 127 qp->attrs.cookie = 128 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0); 129 130 return err; 131 } 132 133 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr) 134 { 135 struct erdma_pd *pd = to_epd(mr->ibmr.pd); 136 struct erdma_cmdq_reg_mr_req req; 137 u32 mtt_level; 138 139 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR); 140 141 if (mr->type == ERDMA_MR_TYPE_FRMR || 142 mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) { 143 if (mr->mem.mtt->continuous) { 144 req.phy_addr[0] = mr->mem.mtt->buf_dma; 145 mtt_level = ERDMA_MR_MTT_1LEVEL; 146 } else { 147 req.phy_addr[0] = sg_dma_address(mr->mem.mtt->sglist); 148 mtt_level = mr->mem.mtt->level; 149 } 150 } else { 151 memcpy(req.phy_addr, mr->mem.mtt->buf, 152 MTT_SIZE(mr->mem.page_cnt)); 153 mtt_level = ERDMA_MR_MTT_0LEVEL; 154 } 155 156 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) | 157 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) | 158 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8); 159 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) | 160 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) | 161 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access); 162 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK, 163 ilog2(mr->mem.page_size)) | 164 FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) | 165 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt); 166 167 if (mr->type == ERDMA_MR_TYPE_DMA) 168 goto post_cmd; 169 170 if (mr->type == ERDMA_MR_TYPE_NORMAL) { 171 req.start_va = mr->mem.va; 172 req.size = mr->mem.len; 173 } 174 175 if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) { 176 req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1); 177 req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK, 178 PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT); 179 req.size_h = upper_32_bits(mr->mem.len); 180 req.mtt_cnt_h = mr->mem.page_cnt >> 20; 181 } 182 183 post_cmd: 184 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 185 } 186 187 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) 188 { 189 struct erdma_dev *dev = to_edev(cq->ibcq.device); 190 struct erdma_cmdq_create_cq_req req; 191 struct erdma_mem *mem; 192 u32 page_size; 193 194 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 195 CMDQ_OPCODE_CREATE_CQ); 196 197 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) | 198 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth)); 199 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn); 200 201 if (rdma_is_kernel_res(&cq->ibcq.res)) { 202 page_size = SZ_32M; 203 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 204 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT); 205 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr); 206 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr); 207 208 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) | 209 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, 210 ERDMA_MR_MTT_0LEVEL); 211 212 req.first_page_offset = 0; 213 req.cq_db_info_addr = 214 cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT); 215 } else { 216 mem = &cq->user_cq.qbuf_mem; 217 req.cfg0 |= 218 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 219 ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT); 220 if (mem->mtt_nents == 1) { 221 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]); 222 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]); 223 req.cfg1 |= 224 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, 225 ERDMA_MR_MTT_0LEVEL); 226 } else { 227 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma); 228 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma); 229 req.cfg1 |= 230 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, 231 ERDMA_MR_MTT_1LEVEL); 232 } 233 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 234 mem->mtt_nents); 235 236 req.first_page_offset = mem->page_offset; 237 req.cq_db_info_addr = cq->user_cq.db_info_dma_addr; 238 239 if (uctx->ext_db.enable) { 240 req.cfg1 |= FIELD_PREP( 241 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1); 242 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK, 243 uctx->ext_db.cdb_off); 244 } 245 } 246 247 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 248 } 249 250 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb) 251 { 252 int idx; 253 unsigned long flags; 254 255 spin_lock_irqsave(&res_cb->lock, flags); 256 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap, 257 res_cb->next_alloc_idx); 258 if (idx == res_cb->max_cap) { 259 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap); 260 if (idx == res_cb->max_cap) { 261 res_cb->next_alloc_idx = 1; 262 spin_unlock_irqrestore(&res_cb->lock, flags); 263 return -ENOSPC; 264 } 265 } 266 267 set_bit(idx, res_cb->bitmap); 268 res_cb->next_alloc_idx = idx + 1; 269 spin_unlock_irqrestore(&res_cb->lock, flags); 270 271 return idx; 272 } 273 274 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx) 275 { 276 unsigned long flags; 277 u32 used; 278 279 spin_lock_irqsave(&res_cb->lock, flags); 280 used = __test_and_clear_bit(idx, res_cb->bitmap); 281 spin_unlock_irqrestore(&res_cb->lock, flags); 282 WARN_ON(!used); 283 } 284 285 static struct rdma_user_mmap_entry * 286 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address, 287 u32 size, u8 mmap_flag, u64 *mmap_offset) 288 { 289 struct erdma_user_mmap_entry *entry = 290 kzalloc(sizeof(*entry), GFP_KERNEL); 291 int ret; 292 293 if (!entry) 294 return NULL; 295 296 entry->address = (u64)address; 297 entry->mmap_flag = mmap_flag; 298 299 size = PAGE_ALIGN(size); 300 301 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry, 302 size); 303 if (ret) { 304 kfree(entry); 305 return NULL; 306 } 307 308 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 309 310 return &entry->rdma_entry; 311 } 312 313 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 314 struct ib_udata *unused) 315 { 316 struct erdma_dev *dev = to_edev(ibdev); 317 318 memset(attr, 0, sizeof(*attr)); 319 320 attr->max_mr_size = dev->attrs.max_mr_size; 321 attr->vendor_id = PCI_VENDOR_ID_ALIBABA; 322 attr->vendor_part_id = dev->pdev->device; 323 attr->hw_ver = dev->pdev->revision; 324 attr->max_qp = dev->attrs.max_qp - 1; 325 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr); 326 attr->max_qp_rd_atom = dev->attrs.max_ord; 327 attr->max_qp_init_rd_atom = dev->attrs.max_ird; 328 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird; 329 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS; 330 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 331 ibdev->local_dma_lkey = dev->attrs.local_dma_key; 332 attr->max_send_sge = dev->attrs.max_send_sge; 333 attr->max_recv_sge = dev->attrs.max_recv_sge; 334 attr->max_sge_rd = dev->attrs.max_sge_rd; 335 attr->max_cq = dev->attrs.max_cq - 1; 336 attr->max_cqe = dev->attrs.max_cqe; 337 attr->max_mr = dev->attrs.max_mr; 338 attr->max_pd = dev->attrs.max_pd; 339 attr->max_mw = dev->attrs.max_mw; 340 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA; 341 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT; 342 343 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC) 344 attr->atomic_cap = IB_ATOMIC_GLOB; 345 346 attr->fw_ver = dev->attrs.fw_version; 347 348 if (dev->netdev) 349 addrconf_addr_eui48((u8 *)&attr->sys_image_guid, 350 dev->netdev->dev_addr); 351 352 return 0; 353 } 354 355 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx, 356 union ib_gid *gid) 357 { 358 struct erdma_dev *dev = to_edev(ibdev); 359 360 memset(gid, 0, sizeof(*gid)); 361 ether_addr_copy(gid->raw, dev->attrs.peer_addr); 362 363 return 0; 364 } 365 366 int erdma_query_port(struct ib_device *ibdev, u32 port, 367 struct ib_port_attr *attr) 368 { 369 struct erdma_dev *dev = to_edev(ibdev); 370 struct net_device *ndev = dev->netdev; 371 372 memset(attr, 0, sizeof(*attr)); 373 374 attr->gid_tbl_len = 1; 375 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP; 376 attr->max_msg_sz = -1; 377 378 if (!ndev) 379 goto out; 380 381 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width); 382 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu); 383 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu); 384 if (netif_running(ndev) && netif_carrier_ok(ndev)) 385 dev->state = IB_PORT_ACTIVE; 386 else 387 dev->state = IB_PORT_DOWN; 388 attr->state = dev->state; 389 390 out: 391 if (dev->state == IB_PORT_ACTIVE) 392 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 393 else 394 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 395 396 return 0; 397 } 398 399 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port, 400 struct ib_port_immutable *port_immutable) 401 { 402 port_immutable->gid_tbl_len = 1; 403 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 404 405 return 0; 406 } 407 408 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 409 { 410 struct erdma_pd *pd = to_epd(ibpd); 411 struct erdma_dev *dev = to_edev(ibpd->device); 412 int pdn; 413 414 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]); 415 if (pdn < 0) 416 return pdn; 417 418 pd->pdn = pdn; 419 420 return 0; 421 } 422 423 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 424 { 425 struct erdma_pd *pd = to_epd(ibpd); 426 struct erdma_dev *dev = to_edev(ibpd->device); 427 428 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn); 429 430 return 0; 431 } 432 433 static void erdma_flush_worker(struct work_struct *work) 434 { 435 struct delayed_work *dwork = to_delayed_work(work); 436 struct erdma_qp *qp = 437 container_of(dwork, struct erdma_qp, reflush_dwork); 438 struct erdma_cmdq_reflush_req req; 439 440 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 441 CMDQ_OPCODE_REFLUSH); 442 req.qpn = QP_ID(qp); 443 req.sq_pi = qp->kern_qp.sq_pi; 444 req.rq_pi = qp->kern_qp.rq_pi; 445 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL); 446 } 447 448 static int erdma_qp_validate_cap(struct erdma_dev *dev, 449 struct ib_qp_init_attr *attrs) 450 { 451 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) || 452 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) || 453 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) || 454 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) || 455 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) || 456 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) { 457 return -EINVAL; 458 } 459 460 return 0; 461 } 462 463 static int erdma_qp_validate_attr(struct erdma_dev *dev, 464 struct ib_qp_init_attr *attrs) 465 { 466 if (attrs->qp_type != IB_QPT_RC) 467 return -EOPNOTSUPP; 468 469 if (attrs->srq) 470 return -EOPNOTSUPP; 471 472 if (!attrs->send_cq || !attrs->recv_cq) 473 return -EOPNOTSUPP; 474 475 return 0; 476 } 477 478 static void free_kernel_qp(struct erdma_qp *qp) 479 { 480 struct erdma_dev *dev = qp->dev; 481 482 vfree(qp->kern_qp.swr_tbl); 483 vfree(qp->kern_qp.rwr_tbl); 484 485 if (qp->kern_qp.sq_buf) 486 dma_free_coherent( 487 &dev->pdev->dev, 488 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 489 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 490 491 if (qp->kern_qp.rq_buf) 492 dma_free_coherent( 493 &dev->pdev->dev, 494 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 495 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 496 } 497 498 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, 499 struct ib_qp_init_attr *attrs) 500 { 501 struct erdma_kqp *kqp = &qp->kern_qp; 502 int size; 503 504 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) 505 kqp->sig_all = 1; 506 507 kqp->sq_pi = 0; 508 kqp->sq_ci = 0; 509 kqp->rq_pi = 0; 510 kqp->rq_ci = 0; 511 kqp->hw_sq_db = 512 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); 513 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; 514 515 kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64)); 516 kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64)); 517 if (!kqp->swr_tbl || !kqp->rwr_tbl) 518 goto err_out; 519 520 size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 521 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 522 &kqp->sq_buf_dma_addr, GFP_KERNEL); 523 if (!kqp->sq_buf) 524 goto err_out; 525 526 size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 527 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 528 &kqp->rq_buf_dma_addr, GFP_KERNEL); 529 if (!kqp->rq_buf) 530 goto err_out; 531 532 kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT); 533 kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT); 534 535 return 0; 536 537 err_out: 538 free_kernel_qp(qp); 539 return -ENOMEM; 540 } 541 542 static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem) 543 { 544 struct erdma_mtt *mtt = mem->mtt; 545 struct ib_block_iter biter; 546 u32 idx = 0; 547 548 while (mtt->low_level) 549 mtt = mtt->low_level; 550 551 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) 552 mtt->buf[idx++] = rdma_block_iter_dma_address(&biter); 553 } 554 555 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev, 556 size_t size) 557 { 558 struct erdma_mtt *mtt; 559 560 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL); 561 if (!mtt) 562 return ERR_PTR(-ENOMEM); 563 564 mtt->size = size; 565 mtt->buf = kzalloc(mtt->size, GFP_KERNEL); 566 if (!mtt->buf) 567 goto err_free_mtt; 568 569 mtt->continuous = true; 570 mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size, 571 DMA_TO_DEVICE); 572 if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma)) 573 goto err_free_mtt_buf; 574 575 return mtt; 576 577 err_free_mtt_buf: 578 kfree(mtt->buf); 579 580 err_free_mtt: 581 kfree(mtt); 582 583 return ERR_PTR(-ENOMEM); 584 } 585 586 static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev, 587 struct erdma_mtt *mtt) 588 { 589 dma_unmap_sg(&dev->pdev->dev, mtt->sglist, mtt->nsg, DMA_TO_DEVICE); 590 vfree(mtt->sglist); 591 } 592 593 static void erdma_destroy_scatter_mtt(struct erdma_dev *dev, 594 struct erdma_mtt *mtt) 595 { 596 erdma_destroy_mtt_buf_sg(dev, mtt); 597 vfree(mtt->buf); 598 kfree(mtt); 599 } 600 601 static void erdma_init_middle_mtt(struct erdma_mtt *mtt, 602 struct erdma_mtt *low_mtt) 603 { 604 struct scatterlist *sg; 605 u32 idx = 0, i; 606 607 for_each_sg(low_mtt->sglist, sg, low_mtt->nsg, i) 608 mtt->buf[idx++] = sg_dma_address(sg); 609 } 610 611 static int erdma_create_mtt_buf_sg(struct erdma_dev *dev, struct erdma_mtt *mtt) 612 { 613 struct scatterlist *sglist; 614 void *buf = mtt->buf; 615 u32 npages, i, nsg; 616 struct page *pg; 617 618 /* Failed if buf is not page aligned */ 619 if ((uintptr_t)buf & ~PAGE_MASK) 620 return -EINVAL; 621 622 npages = DIV_ROUND_UP(mtt->size, PAGE_SIZE); 623 sglist = vzalloc(npages * sizeof(*sglist)); 624 if (!sglist) 625 return -ENOMEM; 626 627 sg_init_table(sglist, npages); 628 for (i = 0; i < npages; i++) { 629 pg = vmalloc_to_page(buf); 630 if (!pg) 631 goto err; 632 sg_set_page(&sglist[i], pg, PAGE_SIZE, 0); 633 buf += PAGE_SIZE; 634 } 635 636 nsg = dma_map_sg(&dev->pdev->dev, sglist, npages, DMA_TO_DEVICE); 637 if (!nsg) 638 goto err; 639 640 mtt->sglist = sglist; 641 mtt->nsg = nsg; 642 643 return 0; 644 err: 645 vfree(sglist); 646 647 return -ENOMEM; 648 } 649 650 static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev, 651 size_t size) 652 { 653 struct erdma_mtt *mtt; 654 int ret = -ENOMEM; 655 656 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL); 657 if (!mtt) 658 return NULL; 659 660 mtt->size = ALIGN(size, PAGE_SIZE); 661 mtt->buf = vzalloc(mtt->size); 662 mtt->continuous = false; 663 if (!mtt->buf) 664 goto err_free_mtt; 665 666 ret = erdma_create_mtt_buf_sg(dev, mtt); 667 if (ret) 668 goto err_free_mtt_buf; 669 670 ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, nsg:%u\n", 671 mtt->size, mtt->nsg); 672 673 return mtt; 674 675 err_free_mtt_buf: 676 vfree(mtt->buf); 677 678 err_free_mtt: 679 kfree(mtt); 680 681 return ERR_PTR(ret); 682 } 683 684 static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size, 685 bool force_continuous) 686 { 687 struct erdma_mtt *mtt, *tmp_mtt; 688 int ret, level = 0; 689 690 ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size, 691 force_continuous); 692 693 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA)) 694 force_continuous = true; 695 696 if (force_continuous) 697 return erdma_create_cont_mtt(dev, size); 698 699 mtt = erdma_create_scatter_mtt(dev, size); 700 if (IS_ERR(mtt)) 701 return mtt; 702 level = 1; 703 704 /* convergence the mtt table. */ 705 while (mtt->nsg != 1 && level <= 3) { 706 tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->nsg)); 707 if (IS_ERR(tmp_mtt)) { 708 ret = PTR_ERR(tmp_mtt); 709 goto err_free_mtt; 710 } 711 erdma_init_middle_mtt(tmp_mtt, mtt); 712 tmp_mtt->low_level = mtt; 713 mtt = tmp_mtt; 714 level++; 715 } 716 717 if (level > 3) { 718 ret = -ENOMEM; 719 goto err_free_mtt; 720 } 721 722 mtt->level = level; 723 ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n", 724 mtt->level, mtt->sglist[0].dma_address); 725 726 return mtt; 727 err_free_mtt: 728 while (mtt) { 729 tmp_mtt = mtt->low_level; 730 erdma_destroy_scatter_mtt(dev, mtt); 731 mtt = tmp_mtt; 732 } 733 734 return ERR_PTR(ret); 735 } 736 737 static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt) 738 { 739 struct erdma_mtt *tmp_mtt; 740 741 if (mtt->continuous) { 742 dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size, 743 DMA_TO_DEVICE); 744 kfree(mtt->buf); 745 kfree(mtt); 746 } else { 747 while (mtt) { 748 tmp_mtt = mtt->low_level; 749 erdma_destroy_scatter_mtt(dev, mtt); 750 mtt = tmp_mtt; 751 } 752 } 753 } 754 755 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, 756 u64 start, u64 len, int access, u64 virt, 757 unsigned long req_page_size, bool force_continuous) 758 { 759 int ret = 0; 760 761 mem->umem = ib_umem_get(&dev->ibdev, start, len, access); 762 if (IS_ERR(mem->umem)) { 763 ret = PTR_ERR(mem->umem); 764 mem->umem = NULL; 765 return ret; 766 } 767 768 mem->va = virt; 769 mem->len = len; 770 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt); 771 mem->page_offset = start & (mem->page_size - 1); 772 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size); 773 mem->page_cnt = mem->mtt_nents; 774 mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), 775 force_continuous); 776 if (IS_ERR(mem->mtt)) { 777 ret = PTR_ERR(mem->mtt); 778 goto error_ret; 779 } 780 781 erdma_fill_bottom_mtt(dev, mem); 782 783 return 0; 784 785 error_ret: 786 if (mem->umem) { 787 ib_umem_release(mem->umem); 788 mem->umem = NULL; 789 } 790 791 return ret; 792 } 793 794 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem) 795 { 796 if (mem->mtt) 797 erdma_destroy_mtt(dev, mem->mtt); 798 799 if (mem->umem) { 800 ib_umem_release(mem->umem); 801 mem->umem = NULL; 802 } 803 } 804 805 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, 806 u64 dbrecords_va, 807 struct erdma_user_dbrecords_page **dbr_page, 808 dma_addr_t *dma_addr) 809 { 810 struct erdma_user_dbrecords_page *page = NULL; 811 int rv = 0; 812 813 mutex_lock(&ctx->dbrecords_page_mutex); 814 815 list_for_each_entry(page, &ctx->dbrecords_page_list, list) 816 if (page->va == (dbrecords_va & PAGE_MASK)) 817 goto found; 818 819 page = kmalloc(sizeof(*page), GFP_KERNEL); 820 if (!page) { 821 rv = -ENOMEM; 822 goto out; 823 } 824 825 page->va = (dbrecords_va & PAGE_MASK); 826 page->refcnt = 0; 827 828 page->umem = ib_umem_get(ctx->ibucontext.device, 829 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); 830 if (IS_ERR(page->umem)) { 831 rv = PTR_ERR(page->umem); 832 kfree(page); 833 goto out; 834 } 835 836 list_add(&page->list, &ctx->dbrecords_page_list); 837 838 found: 839 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) + 840 (dbrecords_va & ~PAGE_MASK); 841 *dbr_page = page; 842 page->refcnt++; 843 844 out: 845 mutex_unlock(&ctx->dbrecords_page_mutex); 846 return rv; 847 } 848 849 static void 850 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, 851 struct erdma_user_dbrecords_page **dbr_page) 852 { 853 if (!ctx || !(*dbr_page)) 854 return; 855 856 mutex_lock(&ctx->dbrecords_page_mutex); 857 if (--(*dbr_page)->refcnt == 0) { 858 list_del(&(*dbr_page)->list); 859 ib_umem_release((*dbr_page)->umem); 860 kfree(*dbr_page); 861 } 862 863 *dbr_page = NULL; 864 mutex_unlock(&ctx->dbrecords_page_mutex); 865 } 866 867 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, 868 u64 va, u32 len, u64 db_info_va) 869 { 870 dma_addr_t db_info_dma_addr; 871 u32 rq_offset; 872 int ret; 873 874 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) + 875 qp->attrs.rq_size * RQE_SIZE)) 876 return -EINVAL; 877 878 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va, 879 qp->attrs.sq_size << SQEBB_SHIFT, 0, va, 880 (SZ_1M - SZ_4K), true); 881 if (ret) 882 return ret; 883 884 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE); 885 qp->user_qp.rq_offset = rq_offset; 886 887 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset, 888 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset, 889 (SZ_1M - SZ_4K), true); 890 if (ret) 891 goto put_sq_mtt; 892 893 ret = erdma_map_user_dbrecords(uctx, db_info_va, 894 &qp->user_qp.user_dbr_page, 895 &db_info_dma_addr); 896 if (ret) 897 goto put_rq_mtt; 898 899 qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr; 900 qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE; 901 902 return 0; 903 904 put_rq_mtt: 905 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem); 906 907 put_sq_mtt: 908 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem); 909 910 return ret; 911 } 912 913 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx) 914 { 915 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem); 916 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem); 917 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page); 918 } 919 920 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 921 struct ib_udata *udata) 922 { 923 struct erdma_qp *qp = to_eqp(ibqp); 924 struct erdma_dev *dev = to_edev(ibqp->device); 925 struct erdma_ucontext *uctx = rdma_udata_to_drv_context( 926 udata, struct erdma_ucontext, ibucontext); 927 struct erdma_ureq_create_qp ureq; 928 struct erdma_uresp_create_qp uresp; 929 int ret; 930 931 ret = erdma_qp_validate_cap(dev, attrs); 932 if (ret) 933 goto err_out; 934 935 ret = erdma_qp_validate_attr(dev, attrs); 936 if (ret) 937 goto err_out; 938 939 qp->scq = to_ecq(attrs->send_cq); 940 qp->rcq = to_ecq(attrs->recv_cq); 941 qp->dev = dev; 942 qp->attrs.cc = dev->attrs.cc; 943 944 init_rwsem(&qp->state_lock); 945 kref_init(&qp->ref); 946 init_completion(&qp->safe_free); 947 948 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp, 949 XA_LIMIT(1, dev->attrs.max_qp - 1), 950 &dev->next_alloc_qpn, GFP_KERNEL); 951 if (ret < 0) { 952 ret = -ENOMEM; 953 goto err_out; 954 } 955 956 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr * 957 ERDMA_MAX_WQEBB_PER_SQE); 958 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr); 959 960 if (uctx) { 961 ret = ib_copy_from_udata(&ureq, udata, 962 min(sizeof(ureq), udata->inlen)); 963 if (ret) 964 goto err_out_xa; 965 966 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len, 967 ureq.db_record_va); 968 if (ret) 969 goto err_out_xa; 970 971 memset(&uresp, 0, sizeof(uresp)); 972 973 uresp.num_sqe = qp->attrs.sq_size; 974 uresp.num_rqe = qp->attrs.rq_size; 975 uresp.qp_id = QP_ID(qp); 976 uresp.rq_offset = qp->user_qp.rq_offset; 977 978 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 979 if (ret) 980 goto err_out_cmd; 981 } else { 982 init_kernel_qp(dev, qp, attrs); 983 } 984 985 qp->attrs.max_send_sge = attrs->cap.max_send_sge; 986 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge; 987 qp->attrs.state = ERDMA_QP_STATE_IDLE; 988 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker); 989 990 ret = create_qp_cmd(uctx, qp); 991 if (ret) 992 goto err_out_cmd; 993 994 spin_lock_init(&qp->lock); 995 996 return 0; 997 998 err_out_cmd: 999 if (uctx) 1000 free_user_qp(qp, uctx); 1001 else 1002 free_kernel_qp(qp); 1003 err_out_xa: 1004 xa_erase(&dev->qp_xa, QP_ID(qp)); 1005 err_out: 1006 return ret; 1007 } 1008 1009 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag) 1010 { 1011 int stag_idx; 1012 1013 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]); 1014 if (stag_idx < 0) 1015 return stag_idx; 1016 1017 /* For now, we always let key field be zero. */ 1018 *stag = (stag_idx << 8); 1019 1020 return 0; 1021 } 1022 1023 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc) 1024 { 1025 struct erdma_dev *dev = to_edev(ibpd->device); 1026 struct erdma_mr *mr; 1027 u32 stag; 1028 int ret; 1029 1030 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1031 if (!mr) 1032 return ERR_PTR(-ENOMEM); 1033 1034 ret = erdma_create_stag(dev, &stag); 1035 if (ret) 1036 goto out_free; 1037 1038 mr->type = ERDMA_MR_TYPE_DMA; 1039 1040 mr->ibmr.lkey = stag; 1041 mr->ibmr.rkey = stag; 1042 mr->ibmr.pd = ibpd; 1043 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc); 1044 ret = regmr_cmd(dev, mr); 1045 if (ret) 1046 goto out_remove_stag; 1047 1048 return &mr->ibmr; 1049 1050 out_remove_stag: 1051 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 1052 mr->ibmr.lkey >> 8); 1053 1054 out_free: 1055 kfree(mr); 1056 1057 return ERR_PTR(ret); 1058 } 1059 1060 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 1061 u32 max_num_sg) 1062 { 1063 struct erdma_mr *mr; 1064 struct erdma_dev *dev = to_edev(ibpd->device); 1065 int ret; 1066 u32 stag; 1067 1068 if (mr_type != IB_MR_TYPE_MEM_REG) 1069 return ERR_PTR(-EOPNOTSUPP); 1070 1071 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT) 1072 return ERR_PTR(-EINVAL); 1073 1074 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1075 if (!mr) 1076 return ERR_PTR(-ENOMEM); 1077 1078 ret = erdma_create_stag(dev, &stag); 1079 if (ret) 1080 goto out_free; 1081 1082 mr->type = ERDMA_MR_TYPE_FRMR; 1083 1084 mr->ibmr.lkey = stag; 1085 mr->ibmr.rkey = stag; 1086 mr->ibmr.pd = ibpd; 1087 /* update it in FRMR. */ 1088 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR | 1089 ERDMA_MR_ACC_RW; 1090 1091 mr->mem.page_size = PAGE_SIZE; /* update it later. */ 1092 mr->mem.page_cnt = max_num_sg; 1093 mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true); 1094 if (IS_ERR(mr->mem.mtt)) { 1095 ret = PTR_ERR(mr->mem.mtt); 1096 goto out_remove_stag; 1097 } 1098 1099 ret = regmr_cmd(dev, mr); 1100 if (ret) 1101 goto out_destroy_mtt; 1102 1103 return &mr->ibmr; 1104 1105 out_destroy_mtt: 1106 erdma_destroy_mtt(dev, mr->mem.mtt); 1107 1108 out_remove_stag: 1109 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 1110 mr->ibmr.lkey >> 8); 1111 1112 out_free: 1113 kfree(mr); 1114 1115 return ERR_PTR(ret); 1116 } 1117 1118 static int erdma_set_page(struct ib_mr *ibmr, u64 addr) 1119 { 1120 struct erdma_mr *mr = to_emr(ibmr); 1121 1122 if (mr->mem.mtt_nents >= mr->mem.page_cnt) 1123 return -1; 1124 1125 mr->mem.mtt->buf[mr->mem.mtt_nents] = addr; 1126 mr->mem.mtt_nents++; 1127 1128 return 0; 1129 } 1130 1131 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1132 unsigned int *sg_offset) 1133 { 1134 struct erdma_mr *mr = to_emr(ibmr); 1135 int num; 1136 1137 mr->mem.mtt_nents = 0; 1138 1139 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset, 1140 erdma_set_page); 1141 1142 return num; 1143 } 1144 1145 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1146 u64 virt, int access, struct ib_udata *udata) 1147 { 1148 struct erdma_mr *mr = NULL; 1149 struct erdma_dev *dev = to_edev(ibpd->device); 1150 u32 stag; 1151 int ret; 1152 1153 if (!len || len > dev->attrs.max_mr_size) 1154 return ERR_PTR(-EINVAL); 1155 1156 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1157 if (!mr) 1158 return ERR_PTR(-ENOMEM); 1159 1160 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt, 1161 SZ_2G - SZ_4K, false); 1162 if (ret) 1163 goto err_out_free; 1164 1165 ret = erdma_create_stag(dev, &stag); 1166 if (ret) 1167 goto err_out_put_mtt; 1168 1169 mr->ibmr.lkey = mr->ibmr.rkey = stag; 1170 mr->ibmr.pd = ibpd; 1171 mr->mem.va = virt; 1172 mr->mem.len = len; 1173 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access); 1174 mr->valid = 1; 1175 mr->type = ERDMA_MR_TYPE_NORMAL; 1176 1177 ret = regmr_cmd(dev, mr); 1178 if (ret) 1179 goto err_out_mr; 1180 1181 return &mr->ibmr; 1182 1183 err_out_mr: 1184 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 1185 mr->ibmr.lkey >> 8); 1186 1187 err_out_put_mtt: 1188 put_mtt_entries(dev, &mr->mem); 1189 1190 err_out_free: 1191 kfree(mr); 1192 1193 return ERR_PTR(ret); 1194 } 1195 1196 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1197 { 1198 struct erdma_mr *mr; 1199 struct erdma_dev *dev = to_edev(ibmr->device); 1200 struct erdma_cmdq_dereg_mr_req req; 1201 int ret; 1202 1203 mr = to_emr(ibmr); 1204 1205 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1206 CMDQ_OPCODE_DEREG_MR); 1207 1208 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) | 1209 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF); 1210 1211 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1212 if (ret) 1213 return ret; 1214 1215 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); 1216 1217 put_mtt_entries(dev, &mr->mem); 1218 1219 kfree(mr); 1220 return 0; 1221 } 1222 1223 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1224 { 1225 struct erdma_cq *cq = to_ecq(ibcq); 1226 struct erdma_dev *dev = to_edev(ibcq->device); 1227 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1228 udata, struct erdma_ucontext, ibucontext); 1229 int err; 1230 struct erdma_cmdq_destroy_cq_req req; 1231 1232 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1233 CMDQ_OPCODE_DESTROY_CQ); 1234 req.cqn = cq->cqn; 1235 1236 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1237 if (err) 1238 return err; 1239 1240 if (rdma_is_kernel_res(&cq->ibcq.res)) { 1241 dma_free_coherent(&dev->pdev->dev, 1242 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1243 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1244 } else { 1245 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1246 put_mtt_entries(dev, &cq->user_cq.qbuf_mem); 1247 } 1248 1249 xa_erase(&dev->cq_xa, cq->cqn); 1250 1251 return 0; 1252 } 1253 1254 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1255 { 1256 struct erdma_qp *qp = to_eqp(ibqp); 1257 struct erdma_dev *dev = to_edev(ibqp->device); 1258 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1259 udata, struct erdma_ucontext, ibucontext); 1260 struct erdma_qp_attrs qp_attrs; 1261 int err; 1262 struct erdma_cmdq_destroy_qp_req req; 1263 1264 down_write(&qp->state_lock); 1265 qp_attrs.state = ERDMA_QP_STATE_ERROR; 1266 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE); 1267 up_write(&qp->state_lock); 1268 1269 cancel_delayed_work_sync(&qp->reflush_dwork); 1270 1271 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1272 CMDQ_OPCODE_DESTROY_QP); 1273 req.qpn = QP_ID(qp); 1274 1275 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1276 if (err) 1277 return err; 1278 1279 erdma_qp_put(qp); 1280 wait_for_completion(&qp->safe_free); 1281 1282 if (rdma_is_kernel_res(&qp->ibqp.res)) { 1283 vfree(qp->kern_qp.swr_tbl); 1284 vfree(qp->kern_qp.rwr_tbl); 1285 dma_free_coherent( 1286 &dev->pdev->dev, 1287 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 1288 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 1289 dma_free_coherent( 1290 &dev->pdev->dev, 1291 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 1292 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 1293 } else { 1294 put_mtt_entries(dev, &qp->user_qp.sq_mem); 1295 put_mtt_entries(dev, &qp->user_qp.rq_mem); 1296 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page); 1297 } 1298 1299 if (qp->cep) 1300 erdma_cep_put(qp->cep); 1301 xa_erase(&dev->qp_xa, QP_ID(qp)); 1302 1303 return 0; 1304 } 1305 1306 void erdma_qp_get_ref(struct ib_qp *ibqp) 1307 { 1308 erdma_qp_get(to_eqp(ibqp)); 1309 } 1310 1311 void erdma_qp_put_ref(struct ib_qp *ibqp) 1312 { 1313 erdma_qp_put(to_eqp(ibqp)); 1314 } 1315 1316 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma) 1317 { 1318 struct rdma_user_mmap_entry *rdma_entry; 1319 struct erdma_user_mmap_entry *entry; 1320 pgprot_t prot; 1321 int err; 1322 1323 rdma_entry = rdma_user_mmap_entry_get(ctx, vma); 1324 if (!rdma_entry) 1325 return -EINVAL; 1326 1327 entry = to_emmap(rdma_entry); 1328 1329 switch (entry->mmap_flag) { 1330 case ERDMA_MMAP_IO_NC: 1331 /* map doorbell. */ 1332 prot = pgprot_device(vma->vm_page_prot); 1333 break; 1334 default: 1335 err = -EINVAL; 1336 goto put_entry; 1337 } 1338 1339 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE, 1340 prot, rdma_entry); 1341 1342 put_entry: 1343 rdma_user_mmap_entry_put(rdma_entry); 1344 return err; 1345 } 1346 1347 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 1348 { 1349 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry); 1350 1351 kfree(entry); 1352 } 1353 1354 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx, 1355 bool ext_db_en) 1356 { 1357 struct erdma_cmdq_ext_db_req req = {}; 1358 u64 val0, val1; 1359 int ret; 1360 1361 /* 1362 * CAP_SYS_RAWIO is required if hardware does not support extend 1363 * doorbell mechanism. 1364 */ 1365 if (!ext_db_en && !capable(CAP_SYS_RAWIO)) 1366 return -EPERM; 1367 1368 if (!ext_db_en) { 1369 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET; 1370 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET; 1371 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET; 1372 return 0; 1373 } 1374 1375 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1376 CMDQ_OPCODE_ALLOC_DB); 1377 1378 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) | 1379 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) | 1380 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1); 1381 1382 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1); 1383 if (ret) 1384 return ret; 1385 1386 ctx->ext_db.enable = true; 1387 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB); 1388 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB); 1389 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB); 1390 1391 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT); 1392 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT); 1393 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT); 1394 1395 return 0; 1396 } 1397 1398 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx) 1399 { 1400 struct erdma_cmdq_ext_db_req req = {}; 1401 int ret; 1402 1403 if (!ctx->ext_db.enable) 1404 return; 1405 1406 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1407 CMDQ_OPCODE_FREE_DB); 1408 1409 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) | 1410 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) | 1411 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1); 1412 1413 req.sdb_off = ctx->ext_db.sdb_off; 1414 req.rdb_off = ctx->ext_db.rdb_off; 1415 req.cdb_off = ctx->ext_db.cdb_off; 1416 1417 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1418 if (ret) 1419 ibdev_err_ratelimited(&dev->ibdev, 1420 "free db resources failed %d", ret); 1421 } 1422 1423 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx) 1424 { 1425 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry); 1426 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry); 1427 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry); 1428 } 1429 1430 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) 1431 { 1432 struct erdma_ucontext *ctx = to_ectx(ibctx); 1433 struct erdma_dev *dev = to_edev(ibctx->device); 1434 int ret; 1435 struct erdma_uresp_alloc_ctx uresp = {}; 1436 1437 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) { 1438 ret = -ENOMEM; 1439 goto err_out; 1440 } 1441 1442 if (udata->outlen < sizeof(uresp)) { 1443 ret = -EINVAL; 1444 goto err_out; 1445 } 1446 1447 INIT_LIST_HEAD(&ctx->dbrecords_page_list); 1448 mutex_init(&ctx->dbrecords_page_mutex); 1449 1450 ret = alloc_db_resources(dev, ctx, 1451 !!(dev->attrs.cap_flags & 1452 ERDMA_DEV_CAP_FLAGS_EXTEND_DB)); 1453 if (ret) 1454 goto err_out; 1455 1456 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert( 1457 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb); 1458 if (!ctx->sq_db_mmap_entry) { 1459 ret = -ENOMEM; 1460 goto err_free_ext_db; 1461 } 1462 1463 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert( 1464 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb); 1465 if (!ctx->rq_db_mmap_entry) { 1466 ret = -EINVAL; 1467 goto err_put_mmap_entries; 1468 } 1469 1470 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert( 1471 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb); 1472 if (!ctx->cq_db_mmap_entry) { 1473 ret = -EINVAL; 1474 goto err_put_mmap_entries; 1475 } 1476 1477 uresp.dev_id = dev->pdev->device; 1478 1479 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1480 if (ret) 1481 goto err_put_mmap_entries; 1482 1483 return 0; 1484 1485 err_put_mmap_entries: 1486 erdma_uctx_user_mmap_entries_remove(ctx); 1487 1488 err_free_ext_db: 1489 free_db_resources(dev, ctx); 1490 1491 err_out: 1492 atomic_dec(&dev->num_ctx); 1493 return ret; 1494 } 1495 1496 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx) 1497 { 1498 struct erdma_dev *dev = to_edev(ibctx->device); 1499 struct erdma_ucontext *ctx = to_ectx(ibctx); 1500 1501 erdma_uctx_user_mmap_entries_remove(ctx); 1502 free_db_resources(dev, ctx); 1503 atomic_dec(&dev->num_ctx); 1504 } 1505 1506 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = { 1507 [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE, 1508 [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE, 1509 [IB_QPS_RTR] = ERDMA_QP_STATE_RTR, 1510 [IB_QPS_RTS] = ERDMA_QP_STATE_RTS, 1511 [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING, 1512 [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE, 1513 [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR 1514 }; 1515 1516 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1517 struct ib_udata *udata) 1518 { 1519 struct erdma_qp_attrs new_attrs; 1520 enum erdma_qp_attr_mask erdma_attr_mask = 0; 1521 struct erdma_qp *qp = to_eqp(ibqp); 1522 int ret = 0; 1523 1524 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1525 return -EOPNOTSUPP; 1526 1527 memset(&new_attrs, 0, sizeof(new_attrs)); 1528 1529 if (attr_mask & IB_QP_STATE) { 1530 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state]; 1531 1532 erdma_attr_mask |= ERDMA_QP_ATTR_STATE; 1533 } 1534 1535 down_write(&qp->state_lock); 1536 1537 ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask); 1538 1539 up_write(&qp->state_lock); 1540 1541 return ret; 1542 } 1543 1544 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 1545 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 1546 { 1547 struct erdma_qp *qp; 1548 struct erdma_dev *dev; 1549 1550 if (ibqp && qp_attr && qp_init_attr) { 1551 qp = to_eqp(ibqp); 1552 dev = to_edev(ibqp->device); 1553 } else { 1554 return -EINVAL; 1555 } 1556 1557 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1558 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1559 1560 qp_attr->cap.max_send_wr = qp->attrs.sq_size; 1561 qp_attr->cap.max_recv_wr = qp->attrs.rq_size; 1562 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge; 1563 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge; 1564 1565 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu); 1566 qp_attr->max_rd_atomic = qp->attrs.irq_size; 1567 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size; 1568 1569 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | 1570 IB_ACCESS_REMOTE_WRITE | 1571 IB_ACCESS_REMOTE_READ; 1572 1573 qp_init_attr->cap = qp_attr->cap; 1574 1575 return 0; 1576 } 1577 1578 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, 1579 struct erdma_ureq_create_cq *ureq) 1580 { 1581 int ret; 1582 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1583 1584 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va, 1585 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K, 1586 true); 1587 if (ret) 1588 return ret; 1589 1590 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va, 1591 &cq->user_cq.user_dbr_page, 1592 &cq->user_cq.db_info_dma_addr); 1593 if (ret) 1594 put_mtt_entries(dev, &cq->user_cq.qbuf_mem); 1595 1596 return ret; 1597 } 1598 1599 static int erdma_init_kernel_cq(struct erdma_cq *cq) 1600 { 1601 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1602 1603 cq->kern_cq.qbuf = 1604 dma_alloc_coherent(&dev->pdev->dev, 1605 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1606 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); 1607 if (!cq->kern_cq.qbuf) 1608 return -ENOMEM; 1609 1610 cq->kern_cq.db_record = 1611 (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT)); 1612 spin_lock_init(&cq->kern_cq.lock); 1613 /* use default cqdb addr */ 1614 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET; 1615 1616 return 0; 1617 } 1618 1619 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1620 struct ib_udata *udata) 1621 { 1622 struct erdma_cq *cq = to_ecq(ibcq); 1623 struct erdma_dev *dev = to_edev(ibcq->device); 1624 unsigned int depth = attr->cqe; 1625 int ret; 1626 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1627 udata, struct erdma_ucontext, ibucontext); 1628 1629 if (depth > dev->attrs.max_cqe) 1630 return -EINVAL; 1631 1632 depth = roundup_pow_of_two(depth); 1633 cq->ibcq.cqe = depth; 1634 cq->depth = depth; 1635 cq->assoc_eqn = attr->comp_vector + 1; 1636 1637 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq, 1638 XA_LIMIT(1, dev->attrs.max_cq - 1), 1639 &dev->next_alloc_cqn, GFP_KERNEL); 1640 if (ret < 0) 1641 return ret; 1642 1643 if (!rdma_is_kernel_res(&ibcq->res)) { 1644 struct erdma_ureq_create_cq ureq; 1645 struct erdma_uresp_create_cq uresp; 1646 1647 ret = ib_copy_from_udata(&ureq, udata, 1648 min(udata->inlen, sizeof(ureq))); 1649 if (ret) 1650 goto err_out_xa; 1651 1652 ret = erdma_init_user_cq(ctx, cq, &ureq); 1653 if (ret) 1654 goto err_out_xa; 1655 1656 uresp.cq_id = cq->cqn; 1657 uresp.num_cqe = depth; 1658 1659 ret = ib_copy_to_udata(udata, &uresp, 1660 min(sizeof(uresp), udata->outlen)); 1661 if (ret) 1662 goto err_free_res; 1663 } else { 1664 ret = erdma_init_kernel_cq(cq); 1665 if (ret) 1666 goto err_out_xa; 1667 } 1668 1669 ret = create_cq_cmd(ctx, cq); 1670 if (ret) 1671 goto err_free_res; 1672 1673 return 0; 1674 1675 err_free_res: 1676 if (!rdma_is_kernel_res(&ibcq->res)) { 1677 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1678 put_mtt_entries(dev, &cq->user_cq.qbuf_mem); 1679 } else { 1680 dma_free_coherent(&dev->pdev->dev, 1681 WARPPED_BUFSIZE(depth << CQE_SHIFT), 1682 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1683 } 1684 1685 err_out_xa: 1686 xa_erase(&dev->cq_xa, cq->cqn); 1687 1688 return ret; 1689 } 1690 1691 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu) 1692 { 1693 struct erdma_cmdq_config_mtu_req req; 1694 1695 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1696 CMDQ_OPCODE_CONF_MTU); 1697 req.mtu = mtu; 1698 1699 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1700 } 1701 1702 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason) 1703 { 1704 struct ib_event event; 1705 1706 event.device = &dev->ibdev; 1707 event.element.port_num = 1; 1708 event.event = reason; 1709 1710 ib_dispatch_event(&event); 1711 } 1712