1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 8 /* Copyright (c) 2008-2019, IBM Corporation */ 9 10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */ 11 12 #include <linux/vmalloc.h> 13 #include <net/addrconf.h> 14 #include <rdma/erdma-abi.h> 15 #include <rdma/ib_umem.h> 16 #include <rdma/uverbs_ioctl.h> 17 18 #include "erdma.h" 19 #include "erdma_cm.h" 20 #include "erdma_verbs.h" 21 22 static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg, 23 u64 *addr0, u64 *addr1) 24 { 25 struct erdma_mtt *mtt = mem->mtt; 26 27 if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) { 28 *addr0 = mtt->buf_dma; 29 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK, 30 ERDMA_MR_MTT_1LEVEL); 31 } else { 32 *addr0 = mtt->buf[0]; 33 memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1)); 34 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK, 35 ERDMA_MR_MTT_0LEVEL); 36 } 37 } 38 39 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) 40 { 41 struct erdma_dev *dev = to_edev(qp->ibqp.device); 42 struct erdma_pd *pd = to_epd(qp->ibqp.pd); 43 struct erdma_cmdq_create_qp_req req; 44 struct erdma_uqp *user_qp; 45 u64 resp0, resp1; 46 int err; 47 48 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 49 CMDQ_OPCODE_CREATE_QP); 50 51 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK, 52 ilog2(qp->attrs.sq_size)) | 53 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp)); 54 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK, 55 ilog2(qp->attrs.rq_size)) | 56 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn); 57 58 if (qp->ibqp.qp_type == IB_QPT_RC) 59 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK, 60 ERDMA_QPT_RC); 61 else 62 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK, 63 ERDMA_QPT_UD); 64 65 if (rdma_is_kernel_res(&qp->ibqp.res)) { 66 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT; 67 68 req.sq_cqn_mtt_cfg = 69 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 70 pgsz_range) | 71 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 72 req.rq_cqn_mtt_cfg = 73 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 74 pgsz_range) | 75 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 76 77 req.sq_mtt_cfg = 78 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) | 79 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) | 80 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK, 81 ERDMA_MR_MTT_0LEVEL); 82 req.rq_mtt_cfg = req.sq_mtt_cfg; 83 84 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; 85 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; 86 req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma; 87 req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma; 88 } else { 89 user_qp = &qp->user_qp; 90 req.sq_cqn_mtt_cfg = FIELD_PREP( 91 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 92 ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT); 93 req.sq_cqn_mtt_cfg |= 94 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 95 96 req.rq_cqn_mtt_cfg = FIELD_PREP( 97 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 98 ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT); 99 req.rq_cqn_mtt_cfg |= 100 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 101 102 req.sq_mtt_cfg = user_qp->sq_mem.page_offset; 103 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 104 user_qp->sq_mem.mtt_nents); 105 106 req.rq_mtt_cfg = user_qp->rq_mem.page_offset; 107 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 108 user_qp->rq_mem.mtt_nents); 109 110 assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg, 111 &req.sq_buf_addr, req.sq_mtt_entry); 112 assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg, 113 &req.rq_buf_addr, req.rq_mtt_entry); 114 115 req.sq_dbrec_dma = user_qp->sq_dbrec_dma; 116 req.rq_dbrec_dma = user_qp->rq_dbrec_dma; 117 118 if (uctx->ext_db.enable) { 119 req.sq_cqn_mtt_cfg |= 120 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1); 121 req.db_cfg = 122 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK, 123 uctx->ext_db.sdb_off) | 124 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK, 125 uctx->ext_db.rdb_off); 126 } 127 } 128 129 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1, 130 true); 131 if (!err && erdma_device_iwarp(dev)) 132 qp->attrs.iwarp.cookie = 133 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0); 134 135 return err; 136 } 137 138 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr) 139 { 140 struct erdma_pd *pd = to_epd(mr->ibmr.pd); 141 u32 mtt_level = ERDMA_MR_MTT_0LEVEL; 142 struct erdma_cmdq_reg_mr_req req; 143 144 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR); 145 146 if (mr->type == ERDMA_MR_TYPE_FRMR || 147 mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) { 148 if (mr->mem.mtt->continuous) { 149 req.phy_addr[0] = mr->mem.mtt->buf_dma; 150 mtt_level = ERDMA_MR_MTT_1LEVEL; 151 } else { 152 req.phy_addr[0] = mr->mem.mtt->dma_addrs[0]; 153 mtt_level = mr->mem.mtt->level; 154 } 155 } else if (mr->type != ERDMA_MR_TYPE_DMA) { 156 memcpy(req.phy_addr, mr->mem.mtt->buf, 157 MTT_SIZE(mr->mem.page_cnt)); 158 } 159 160 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) | 161 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) | 162 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8); 163 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) | 164 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) | 165 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access); 166 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK, 167 ilog2(mr->mem.page_size)) | 168 FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) | 169 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt); 170 171 if (mr->type == ERDMA_MR_TYPE_DMA) 172 goto post_cmd; 173 174 if (mr->type == ERDMA_MR_TYPE_NORMAL) { 175 req.start_va = mr->mem.va; 176 req.size = mr->mem.len; 177 } 178 179 if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) { 180 req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1); 181 req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK, 182 PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT); 183 req.size_h = upper_32_bits(mr->mem.len); 184 req.mtt_cnt_h = mr->mem.page_cnt >> 20; 185 } 186 187 post_cmd: 188 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 189 true); 190 } 191 192 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) 193 { 194 struct erdma_dev *dev = to_edev(cq->ibcq.device); 195 struct erdma_cmdq_create_cq_req req; 196 struct erdma_mem *mem; 197 u32 page_size; 198 199 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 200 CMDQ_OPCODE_CREATE_CQ); 201 202 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) | 203 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth)); 204 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn); 205 206 if (rdma_is_kernel_res(&cq->ibcq.res)) { 207 page_size = SZ_32M; 208 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 209 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT); 210 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr); 211 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr); 212 213 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) | 214 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, 215 ERDMA_MR_MTT_0LEVEL); 216 217 req.first_page_offset = 0; 218 req.cq_dbrec_dma = cq->kern_cq.dbrec_dma; 219 } else { 220 mem = &cq->user_cq.qbuf_mem; 221 req.cfg0 |= 222 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 223 ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT); 224 if (mem->mtt_nents == 1) { 225 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]); 226 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]); 227 req.cfg1 |= 228 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, 229 ERDMA_MR_MTT_0LEVEL); 230 } else { 231 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma); 232 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma); 233 req.cfg1 |= 234 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, 235 ERDMA_MR_MTT_1LEVEL); 236 } 237 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 238 mem->mtt_nents); 239 240 req.first_page_offset = mem->page_offset; 241 req.cq_dbrec_dma = cq->user_cq.dbrec_dma; 242 243 if (uctx->ext_db.enable) { 244 req.cfg1 |= FIELD_PREP( 245 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1); 246 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK, 247 uctx->ext_db.cdb_off); 248 } 249 } 250 251 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 252 true); 253 } 254 255 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb) 256 { 257 int idx; 258 unsigned long flags; 259 260 spin_lock_irqsave(&res_cb->lock, flags); 261 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap, 262 res_cb->next_alloc_idx); 263 if (idx == res_cb->max_cap) { 264 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap); 265 if (idx == res_cb->max_cap) { 266 res_cb->next_alloc_idx = 1; 267 spin_unlock_irqrestore(&res_cb->lock, flags); 268 return -ENOSPC; 269 } 270 } 271 272 set_bit(idx, res_cb->bitmap); 273 res_cb->next_alloc_idx = idx + 1; 274 spin_unlock_irqrestore(&res_cb->lock, flags); 275 276 return idx; 277 } 278 279 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx) 280 { 281 unsigned long flags; 282 u32 used; 283 284 spin_lock_irqsave(&res_cb->lock, flags); 285 used = __test_and_clear_bit(idx, res_cb->bitmap); 286 spin_unlock_irqrestore(&res_cb->lock, flags); 287 WARN_ON(!used); 288 } 289 290 static struct rdma_user_mmap_entry * 291 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address, 292 u32 size, u8 mmap_flag, u64 *mmap_offset) 293 { 294 struct erdma_user_mmap_entry *entry = kzalloc_obj(*entry); 295 int ret; 296 297 if (!entry) 298 return NULL; 299 300 entry->address = (u64)address; 301 entry->mmap_flag = mmap_flag; 302 303 size = PAGE_ALIGN(size); 304 305 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry, 306 size); 307 if (ret) { 308 kfree(entry); 309 return NULL; 310 } 311 312 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 313 314 return &entry->rdma_entry; 315 } 316 317 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 318 struct ib_udata *unused) 319 { 320 struct erdma_dev *dev = to_edev(ibdev); 321 322 memset(attr, 0, sizeof(*attr)); 323 324 attr->max_mr_size = dev->attrs.max_mr_size; 325 attr->vendor_id = PCI_VENDOR_ID_ALIBABA; 326 attr->vendor_part_id = dev->pdev->device; 327 attr->hw_ver = dev->pdev->revision; 328 attr->max_qp = dev->attrs.max_qp - 1; 329 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr); 330 attr->max_qp_rd_atom = dev->attrs.max_ord; 331 attr->max_qp_init_rd_atom = dev->attrs.max_ird; 332 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird; 333 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS; 334 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 335 ibdev->local_dma_lkey = dev->attrs.local_dma_key; 336 attr->max_send_sge = dev->attrs.max_send_sge; 337 attr->max_recv_sge = dev->attrs.max_recv_sge; 338 attr->max_sge_rd = dev->attrs.max_sge_rd; 339 attr->max_cq = dev->attrs.max_cq - 1; 340 attr->max_cqe = dev->attrs.max_cqe; 341 attr->max_mr = dev->attrs.max_mr; 342 attr->max_pd = dev->attrs.max_pd; 343 attr->max_mw = dev->attrs.max_mw; 344 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA; 345 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT; 346 347 if (erdma_device_rocev2(dev)) { 348 attr->max_pkeys = ERDMA_MAX_PKEYS; 349 attr->max_ah = dev->attrs.max_ah; 350 } 351 352 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC) 353 attr->atomic_cap = IB_ATOMIC_GLOB; 354 355 attr->fw_ver = dev->attrs.fw_version; 356 357 if (dev->netdev) 358 addrconf_addr_eui48((u8 *)&attr->sys_image_guid, 359 dev->netdev->dev_addr); 360 361 return 0; 362 } 363 364 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx, 365 union ib_gid *gid) 366 { 367 struct erdma_dev *dev = to_edev(ibdev); 368 369 memset(gid, 0, sizeof(*gid)); 370 ether_addr_copy(gid->raw, dev->attrs.peer_addr); 371 372 return 0; 373 } 374 375 int erdma_query_port(struct ib_device *ibdev, u32 port, 376 struct ib_port_attr *attr) 377 { 378 struct erdma_dev *dev = to_edev(ibdev); 379 struct net_device *ndev = dev->netdev; 380 381 memset(attr, 0, sizeof(*attr)); 382 383 if (erdma_device_iwarp(dev)) { 384 attr->gid_tbl_len = 1; 385 } else { 386 attr->gid_tbl_len = dev->attrs.max_gid; 387 attr->ip_gids = true; 388 attr->pkey_tbl_len = ERDMA_MAX_PKEYS; 389 } 390 391 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP; 392 attr->max_msg_sz = -1; 393 394 if (!ndev) 395 goto out; 396 397 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width); 398 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu); 399 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu); 400 attr->state = ib_get_curr_port_state(ndev); 401 402 out: 403 if (attr->state == IB_PORT_ACTIVE) 404 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 405 else 406 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 407 408 return 0; 409 } 410 411 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port, 412 struct ib_port_immutable *port_immutable) 413 { 414 struct erdma_dev *dev = to_edev(ibdev); 415 416 if (erdma_device_iwarp(dev)) { 417 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 418 port_immutable->gid_tbl_len = 1; 419 } else { 420 port_immutable->core_cap_flags = 421 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 422 port_immutable->max_mad_size = IB_MGMT_MAD_SIZE; 423 port_immutable->gid_tbl_len = dev->attrs.max_gid; 424 port_immutable->pkey_tbl_len = ERDMA_MAX_PKEYS; 425 } 426 427 return 0; 428 } 429 430 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 431 { 432 struct erdma_pd *pd = to_epd(ibpd); 433 struct erdma_dev *dev = to_edev(ibpd->device); 434 int pdn; 435 436 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]); 437 if (pdn < 0) 438 return pdn; 439 440 pd->pdn = pdn; 441 442 return 0; 443 } 444 445 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 446 { 447 struct erdma_pd *pd = to_epd(ibpd); 448 struct erdma_dev *dev = to_edev(ibpd->device); 449 450 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn); 451 452 return 0; 453 } 454 455 static void erdma_flush_worker(struct work_struct *work) 456 { 457 struct delayed_work *dwork = to_delayed_work(work); 458 struct erdma_qp *qp = 459 container_of(dwork, struct erdma_qp, reflush_dwork); 460 struct erdma_cmdq_reflush_req req; 461 462 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 463 CMDQ_OPCODE_REFLUSH); 464 req.qpn = QP_ID(qp); 465 req.sq_pi = qp->kern_qp.sq_pi; 466 req.rq_pi = qp->kern_qp.rq_pi; 467 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL, 468 true); 469 } 470 471 static int erdma_qp_validate_cap(struct erdma_dev *dev, 472 struct ib_qp_init_attr *attrs) 473 { 474 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) || 475 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) || 476 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) || 477 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) || 478 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) || 479 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) { 480 return -EINVAL; 481 } 482 483 return 0; 484 } 485 486 static int erdma_qp_validate_attr(struct erdma_dev *dev, 487 struct ib_qp_init_attr *attrs) 488 { 489 if (erdma_device_iwarp(dev) && attrs->qp_type != IB_QPT_RC) 490 return -EOPNOTSUPP; 491 492 if (erdma_device_rocev2(dev) && attrs->qp_type != IB_QPT_RC && 493 attrs->qp_type != IB_QPT_UD && attrs->qp_type != IB_QPT_GSI) 494 return -EOPNOTSUPP; 495 496 if (attrs->srq) 497 return -EOPNOTSUPP; 498 499 if (!attrs->send_cq || !attrs->recv_cq) 500 return -EOPNOTSUPP; 501 502 return 0; 503 } 504 505 static void free_kernel_qp(struct erdma_qp *qp) 506 { 507 struct erdma_dev *dev = qp->dev; 508 509 vfree(qp->kern_qp.swr_tbl); 510 vfree(qp->kern_qp.rwr_tbl); 511 512 if (qp->kern_qp.sq_buf) 513 dma_free_coherent(&dev->pdev->dev, 514 qp->attrs.sq_size << SQEBB_SHIFT, 515 qp->kern_qp.sq_buf, 516 qp->kern_qp.sq_buf_dma_addr); 517 518 if (qp->kern_qp.sq_dbrec) 519 dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, 520 qp->kern_qp.sq_dbrec_dma); 521 522 if (qp->kern_qp.rq_buf) 523 dma_free_coherent(&dev->pdev->dev, 524 qp->attrs.rq_size << RQE_SHIFT, 525 qp->kern_qp.rq_buf, 526 qp->kern_qp.rq_buf_dma_addr); 527 528 if (qp->kern_qp.rq_dbrec) 529 dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, 530 qp->kern_qp.rq_dbrec_dma); 531 } 532 533 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, 534 struct ib_qp_init_attr *attrs) 535 { 536 struct erdma_kqp *kqp = &qp->kern_qp; 537 int size; 538 539 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) 540 kqp->sig_all = 1; 541 542 kqp->sq_pi = 0; 543 kqp->sq_ci = 0; 544 kqp->rq_pi = 0; 545 kqp->rq_ci = 0; 546 kqp->hw_sq_db = 547 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); 548 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; 549 550 kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64)); 551 kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64)); 552 if (!kqp->swr_tbl || !kqp->rwr_tbl) 553 goto err_out; 554 555 size = qp->attrs.sq_size << SQEBB_SHIFT; 556 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 557 &kqp->sq_buf_dma_addr, GFP_KERNEL); 558 if (!kqp->sq_buf) 559 goto err_out; 560 561 kqp->sq_dbrec = 562 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); 563 if (!kqp->sq_dbrec) 564 goto err_out; 565 566 size = qp->attrs.rq_size << RQE_SHIFT; 567 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 568 &kqp->rq_buf_dma_addr, GFP_KERNEL); 569 if (!kqp->rq_buf) 570 goto err_out; 571 572 kqp->rq_dbrec = 573 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); 574 if (!kqp->rq_dbrec) 575 goto err_out; 576 577 return 0; 578 579 err_out: 580 free_kernel_qp(qp); 581 return -ENOMEM; 582 } 583 584 static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem) 585 { 586 struct erdma_mtt *mtt = mem->mtt; 587 struct ib_block_iter biter; 588 u32 idx = 0; 589 590 while (mtt->low_level) 591 mtt = mtt->low_level; 592 593 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) 594 mtt->buf[idx++] = rdma_block_iter_dma_address(&biter); 595 } 596 597 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev, 598 size_t size) 599 { 600 struct erdma_mtt *mtt; 601 602 mtt = kzalloc_obj(*mtt); 603 if (!mtt) 604 return ERR_PTR(-ENOMEM); 605 606 mtt->size = size; 607 mtt->buf = kzalloc(mtt->size, GFP_KERNEL); 608 if (!mtt->buf) 609 goto err_free_mtt; 610 611 mtt->continuous = true; 612 mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size, 613 DMA_TO_DEVICE); 614 if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma)) 615 goto err_free_mtt_buf; 616 617 return mtt; 618 619 err_free_mtt_buf: 620 kfree(mtt->buf); 621 622 err_free_mtt: 623 kfree(mtt); 624 625 return ERR_PTR(-ENOMEM); 626 } 627 628 static void erdma_unmap_page_list(struct erdma_dev *dev, dma_addr_t *pg_dma, 629 u32 npages) 630 { 631 u32 i; 632 633 for (i = 0; i < npages; i++) 634 dma_unmap_page(&dev->pdev->dev, pg_dma[i], PAGE_SIZE, 635 DMA_TO_DEVICE); 636 } 637 638 static void erdma_destroy_mtt_buf_dma_addrs(struct erdma_dev *dev, 639 struct erdma_mtt *mtt) 640 { 641 erdma_unmap_page_list(dev, mtt->dma_addrs, mtt->npages); 642 vfree(mtt->dma_addrs); 643 } 644 645 static void erdma_destroy_scatter_mtt(struct erdma_dev *dev, 646 struct erdma_mtt *mtt) 647 { 648 erdma_destroy_mtt_buf_dma_addrs(dev, mtt); 649 vfree(mtt->buf); 650 kfree(mtt); 651 } 652 653 static void erdma_init_middle_mtt(struct erdma_mtt *mtt, 654 struct erdma_mtt *low_mtt) 655 { 656 dma_addr_t *pg_addr = mtt->buf; 657 u32 i; 658 659 for (i = 0; i < low_mtt->npages; i++) 660 pg_addr[i] = low_mtt->dma_addrs[i]; 661 } 662 663 static u32 vmalloc_to_dma_addrs(struct erdma_dev *dev, dma_addr_t **dma_addrs, 664 void *buf, u64 len) 665 { 666 dma_addr_t *pg_dma; 667 struct page *pg; 668 u32 npages, i; 669 void *addr; 670 671 npages = (PAGE_ALIGN((u64)buf + len) - PAGE_ALIGN_DOWN((u64)buf)) >> 672 PAGE_SHIFT; 673 pg_dma = vcalloc(npages, sizeof(*pg_dma)); 674 if (!pg_dma) 675 return 0; 676 677 addr = buf; 678 for (i = 0; i < npages; i++) { 679 pg = vmalloc_to_page(addr); 680 if (!pg) 681 goto err; 682 683 pg_dma[i] = dma_map_page(&dev->pdev->dev, pg, 0, PAGE_SIZE, 684 DMA_TO_DEVICE); 685 if (dma_mapping_error(&dev->pdev->dev, pg_dma[i])) 686 goto err; 687 688 addr += PAGE_SIZE; 689 } 690 691 *dma_addrs = pg_dma; 692 693 return npages; 694 err: 695 erdma_unmap_page_list(dev, pg_dma, i); 696 vfree(pg_dma); 697 698 return 0; 699 } 700 701 static int erdma_create_mtt_buf_dma_addrs(struct erdma_dev *dev, 702 struct erdma_mtt *mtt) 703 { 704 dma_addr_t *addrs; 705 u32 npages; 706 707 /* Failed if buf is not page aligned */ 708 if ((uintptr_t)mtt->buf & ~PAGE_MASK) 709 return -EINVAL; 710 711 npages = vmalloc_to_dma_addrs(dev, &addrs, mtt->buf, mtt->size); 712 if (!npages) 713 return -ENOMEM; 714 715 mtt->dma_addrs = addrs; 716 mtt->npages = npages; 717 718 return 0; 719 } 720 721 static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev, 722 size_t size) 723 { 724 struct erdma_mtt *mtt; 725 int ret = -ENOMEM; 726 727 mtt = kzalloc_obj(*mtt); 728 if (!mtt) 729 return ERR_PTR(-ENOMEM); 730 731 mtt->size = ALIGN(size, PAGE_SIZE); 732 mtt->buf = vzalloc(mtt->size); 733 mtt->continuous = false; 734 if (!mtt->buf) 735 goto err_free_mtt; 736 737 ret = erdma_create_mtt_buf_dma_addrs(dev, mtt); 738 if (ret) 739 goto err_free_mtt_buf; 740 741 ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, npages:%u\n", 742 mtt->size, mtt->npages); 743 744 return mtt; 745 746 err_free_mtt_buf: 747 vfree(mtt->buf); 748 749 err_free_mtt: 750 kfree(mtt); 751 752 return ERR_PTR(ret); 753 } 754 755 static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size, 756 bool force_continuous) 757 { 758 struct erdma_mtt *mtt, *tmp_mtt; 759 int ret, level = 0; 760 761 ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size, 762 force_continuous); 763 764 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA)) 765 force_continuous = true; 766 767 if (force_continuous) 768 return erdma_create_cont_mtt(dev, size); 769 770 mtt = erdma_create_scatter_mtt(dev, size); 771 if (IS_ERR(mtt)) 772 return mtt; 773 level = 1; 774 775 /* convergence the mtt table. */ 776 while (mtt->npages != 1 && level <= 3) { 777 tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->npages)); 778 if (IS_ERR(tmp_mtt)) { 779 ret = PTR_ERR(tmp_mtt); 780 goto err_free_mtt; 781 } 782 erdma_init_middle_mtt(tmp_mtt, mtt); 783 tmp_mtt->low_level = mtt; 784 mtt = tmp_mtt; 785 level++; 786 } 787 788 if (level > 3) { 789 ret = -ENOMEM; 790 goto err_free_mtt; 791 } 792 793 mtt->level = level; 794 ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n", 795 mtt->level, mtt->dma_addrs[0]); 796 797 return mtt; 798 err_free_mtt: 799 while (mtt) { 800 tmp_mtt = mtt->low_level; 801 erdma_destroy_scatter_mtt(dev, mtt); 802 mtt = tmp_mtt; 803 } 804 805 return ERR_PTR(ret); 806 } 807 808 static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt) 809 { 810 struct erdma_mtt *tmp_mtt; 811 812 if (mtt->continuous) { 813 dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size, 814 DMA_TO_DEVICE); 815 kfree(mtt->buf); 816 kfree(mtt); 817 } else { 818 while (mtt) { 819 tmp_mtt = mtt->low_level; 820 erdma_destroy_scatter_mtt(dev, mtt); 821 mtt = tmp_mtt; 822 } 823 } 824 } 825 826 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, 827 u64 start, u64 len, int access, u64 virt, 828 unsigned long req_page_size, bool force_continuous) 829 { 830 int ret = 0; 831 832 mem->umem = ib_umem_get(&dev->ibdev, start, len, access); 833 if (IS_ERR(mem->umem)) { 834 ret = PTR_ERR(mem->umem); 835 mem->umem = NULL; 836 return ret; 837 } 838 839 mem->va = virt; 840 mem->len = len; 841 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt); 842 mem->page_offset = start & (mem->page_size - 1); 843 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size); 844 mem->page_cnt = mem->mtt_nents; 845 mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), 846 force_continuous); 847 if (IS_ERR(mem->mtt)) { 848 ret = PTR_ERR(mem->mtt); 849 goto error_ret; 850 } 851 852 erdma_fill_bottom_mtt(dev, mem); 853 854 return 0; 855 856 error_ret: 857 if (mem->umem) { 858 ib_umem_release(mem->umem); 859 mem->umem = NULL; 860 } 861 862 return ret; 863 } 864 865 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem) 866 { 867 if (mem->mtt) 868 erdma_destroy_mtt(dev, mem->mtt); 869 870 if (mem->umem) { 871 ib_umem_release(mem->umem); 872 mem->umem = NULL; 873 } 874 } 875 876 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, 877 u64 dbrecords_va, 878 struct erdma_user_dbrecords_page **dbr_page, 879 dma_addr_t *dma_addr) 880 { 881 struct erdma_user_dbrecords_page *page = NULL; 882 int rv = 0; 883 884 mutex_lock(&ctx->dbrecords_page_mutex); 885 886 list_for_each_entry(page, &ctx->dbrecords_page_list, list) 887 if (page->va == (dbrecords_va & PAGE_MASK)) 888 goto found; 889 890 page = kmalloc_obj(*page); 891 if (!page) { 892 rv = -ENOMEM; 893 goto out; 894 } 895 896 page->va = (dbrecords_va & PAGE_MASK); 897 page->refcnt = 0; 898 899 page->umem = ib_umem_get(ctx->ibucontext.device, 900 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); 901 if (IS_ERR(page->umem)) { 902 rv = PTR_ERR(page->umem); 903 kfree(page); 904 goto out; 905 } 906 907 list_add(&page->list, &ctx->dbrecords_page_list); 908 909 found: 910 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) + 911 (dbrecords_va & ~PAGE_MASK); 912 *dbr_page = page; 913 page->refcnt++; 914 915 out: 916 mutex_unlock(&ctx->dbrecords_page_mutex); 917 return rv; 918 } 919 920 static void 921 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, 922 struct erdma_user_dbrecords_page **dbr_page) 923 { 924 if (!ctx || !(*dbr_page)) 925 return; 926 927 mutex_lock(&ctx->dbrecords_page_mutex); 928 if (--(*dbr_page)->refcnt == 0) { 929 list_del(&(*dbr_page)->list); 930 ib_umem_release((*dbr_page)->umem); 931 kfree(*dbr_page); 932 } 933 934 *dbr_page = NULL; 935 mutex_unlock(&ctx->dbrecords_page_mutex); 936 } 937 938 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, 939 u64 va, u32 len, u64 dbrec_va) 940 { 941 dma_addr_t dbrec_dma; 942 u32 rq_offset; 943 int ret; 944 945 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) + 946 qp->attrs.rq_size * RQE_SIZE)) 947 return -EINVAL; 948 949 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va, 950 qp->attrs.sq_size << SQEBB_SHIFT, 0, va, 951 (SZ_1M - SZ_4K), true); 952 if (ret) 953 return ret; 954 955 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE); 956 qp->user_qp.rq_offset = rq_offset; 957 958 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset, 959 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset, 960 (SZ_1M - SZ_4K), true); 961 if (ret) 962 goto put_sq_mtt; 963 964 ret = erdma_map_user_dbrecords(uctx, dbrec_va, 965 &qp->user_qp.user_dbr_page, 966 &dbrec_dma); 967 if (ret) 968 goto put_rq_mtt; 969 970 qp->user_qp.sq_dbrec_dma = dbrec_dma; 971 qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE; 972 973 return 0; 974 975 put_rq_mtt: 976 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem); 977 978 put_sq_mtt: 979 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem); 980 981 return ret; 982 } 983 984 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx) 985 { 986 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem); 987 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem); 988 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page); 989 } 990 991 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 992 struct ib_udata *udata) 993 { 994 struct erdma_qp *qp = to_eqp(ibqp); 995 struct erdma_dev *dev = to_edev(ibqp->device); 996 struct erdma_ucontext *uctx = rdma_udata_to_drv_context( 997 udata, struct erdma_ucontext, ibucontext); 998 struct erdma_ureq_create_qp ureq; 999 struct erdma_uresp_create_qp uresp; 1000 void *old_entry; 1001 int ret = 0; 1002 1003 ret = erdma_qp_validate_cap(dev, attrs); 1004 if (ret) 1005 goto err_out; 1006 1007 ret = erdma_qp_validate_attr(dev, attrs); 1008 if (ret) 1009 goto err_out; 1010 1011 qp->scq = to_ecq(attrs->send_cq); 1012 qp->rcq = to_ecq(attrs->recv_cq); 1013 qp->dev = dev; 1014 qp->attrs.cc = dev->attrs.cc; 1015 1016 init_rwsem(&qp->state_lock); 1017 kref_init(&qp->ref); 1018 init_completion(&qp->safe_free); 1019 1020 if (qp->ibqp.qp_type == IB_QPT_GSI) { 1021 old_entry = xa_store(&dev->qp_xa, 1, qp, GFP_KERNEL); 1022 if (xa_is_err(old_entry)) 1023 ret = xa_err(old_entry); 1024 else 1025 qp->ibqp.qp_num = 1; 1026 } else { 1027 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp, 1028 XA_LIMIT(1, dev->attrs.max_qp - 1), 1029 &dev->next_alloc_qpn, GFP_KERNEL); 1030 } 1031 1032 if (ret < 0) { 1033 ret = -ENOMEM; 1034 goto err_out; 1035 } 1036 1037 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr * 1038 ERDMA_MAX_WQEBB_PER_SQE); 1039 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr); 1040 1041 if (uctx) { 1042 ret = ib_copy_from_udata(&ureq, udata, 1043 min(sizeof(ureq), udata->inlen)); 1044 if (ret) 1045 goto err_out_xa; 1046 1047 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len, 1048 ureq.db_record_va); 1049 if (ret) 1050 goto err_out_xa; 1051 1052 memset(&uresp, 0, sizeof(uresp)); 1053 1054 uresp.num_sqe = qp->attrs.sq_size; 1055 uresp.num_rqe = qp->attrs.rq_size; 1056 uresp.qp_id = QP_ID(qp); 1057 uresp.rq_offset = qp->user_qp.rq_offset; 1058 1059 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1060 if (ret) 1061 goto err_out_cmd; 1062 } else { 1063 ret = init_kernel_qp(dev, qp, attrs); 1064 if (ret) 1065 goto err_out_xa; 1066 } 1067 1068 qp->attrs.max_send_sge = attrs->cap.max_send_sge; 1069 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge; 1070 1071 if (erdma_device_iwarp(qp->dev)) 1072 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE; 1073 else 1074 qp->attrs.rocev2.state = ERDMA_QPS_ROCEV2_RESET; 1075 1076 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker); 1077 1078 ret = create_qp_cmd(uctx, qp); 1079 if (ret) 1080 goto err_out_cmd; 1081 1082 spin_lock_init(&qp->lock); 1083 1084 return 0; 1085 1086 err_out_cmd: 1087 if (uctx) 1088 free_user_qp(qp, uctx); 1089 else 1090 free_kernel_qp(qp); 1091 err_out_xa: 1092 xa_erase(&dev->qp_xa, QP_ID(qp)); 1093 err_out: 1094 return ret; 1095 } 1096 1097 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag) 1098 { 1099 int stag_idx; 1100 1101 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]); 1102 if (stag_idx < 0) 1103 return stag_idx; 1104 1105 /* For now, we always let key field be zero. */ 1106 *stag = (stag_idx << 8); 1107 1108 return 0; 1109 } 1110 1111 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc) 1112 { 1113 struct erdma_dev *dev = to_edev(ibpd->device); 1114 struct erdma_mr *mr; 1115 u32 stag; 1116 int ret; 1117 1118 mr = kzalloc_obj(*mr); 1119 if (!mr) 1120 return ERR_PTR(-ENOMEM); 1121 1122 ret = erdma_create_stag(dev, &stag); 1123 if (ret) 1124 goto out_free; 1125 1126 mr->type = ERDMA_MR_TYPE_DMA; 1127 1128 mr->ibmr.lkey = stag; 1129 mr->ibmr.rkey = stag; 1130 mr->ibmr.pd = ibpd; 1131 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc); 1132 ret = regmr_cmd(dev, mr); 1133 if (ret) 1134 goto out_remove_stag; 1135 1136 return &mr->ibmr; 1137 1138 out_remove_stag: 1139 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 1140 mr->ibmr.lkey >> 8); 1141 1142 out_free: 1143 kfree(mr); 1144 1145 return ERR_PTR(ret); 1146 } 1147 1148 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 1149 u32 max_num_sg) 1150 { 1151 struct erdma_mr *mr; 1152 struct erdma_dev *dev = to_edev(ibpd->device); 1153 int ret; 1154 u32 stag; 1155 1156 if (mr_type != IB_MR_TYPE_MEM_REG) 1157 return ERR_PTR(-EOPNOTSUPP); 1158 1159 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT) 1160 return ERR_PTR(-EINVAL); 1161 1162 mr = kzalloc_obj(*mr); 1163 if (!mr) 1164 return ERR_PTR(-ENOMEM); 1165 1166 ret = erdma_create_stag(dev, &stag); 1167 if (ret) 1168 goto out_free; 1169 1170 mr->type = ERDMA_MR_TYPE_FRMR; 1171 1172 mr->ibmr.lkey = stag; 1173 mr->ibmr.rkey = stag; 1174 mr->ibmr.pd = ibpd; 1175 /* update it in FRMR. */ 1176 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR | 1177 ERDMA_MR_ACC_RW; 1178 1179 mr->mem.page_size = PAGE_SIZE; /* update it later. */ 1180 mr->mem.page_cnt = max_num_sg; 1181 mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true); 1182 if (IS_ERR(mr->mem.mtt)) { 1183 ret = PTR_ERR(mr->mem.mtt); 1184 goto out_remove_stag; 1185 } 1186 1187 ret = regmr_cmd(dev, mr); 1188 if (ret) 1189 goto out_destroy_mtt; 1190 1191 return &mr->ibmr; 1192 1193 out_destroy_mtt: 1194 erdma_destroy_mtt(dev, mr->mem.mtt); 1195 1196 out_remove_stag: 1197 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 1198 mr->ibmr.lkey >> 8); 1199 1200 out_free: 1201 kfree(mr); 1202 1203 return ERR_PTR(ret); 1204 } 1205 1206 static int erdma_set_page(struct ib_mr *ibmr, u64 addr) 1207 { 1208 struct erdma_mr *mr = to_emr(ibmr); 1209 1210 if (mr->mem.mtt_nents >= mr->mem.page_cnt) 1211 return -1; 1212 1213 mr->mem.mtt->buf[mr->mem.mtt_nents] = addr; 1214 mr->mem.mtt_nents++; 1215 1216 return 0; 1217 } 1218 1219 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1220 unsigned int *sg_offset) 1221 { 1222 struct erdma_mr *mr = to_emr(ibmr); 1223 int num; 1224 1225 mr->mem.mtt_nents = 0; 1226 1227 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset, 1228 erdma_set_page); 1229 1230 return num; 1231 } 1232 1233 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 1234 u64 virt, int access, struct ib_dmah *dmah, 1235 struct ib_udata *udata) 1236 { 1237 struct erdma_mr *mr = NULL; 1238 struct erdma_dev *dev = to_edev(ibpd->device); 1239 u32 stag; 1240 int ret; 1241 1242 if (dmah) 1243 return ERR_PTR(-EOPNOTSUPP); 1244 1245 if (!len || len > dev->attrs.max_mr_size) 1246 return ERR_PTR(-EINVAL); 1247 1248 mr = kzalloc_obj(*mr); 1249 if (!mr) 1250 return ERR_PTR(-ENOMEM); 1251 1252 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt, 1253 SZ_2G - SZ_4K, false); 1254 if (ret) 1255 goto err_out_free; 1256 1257 ret = erdma_create_stag(dev, &stag); 1258 if (ret) 1259 goto err_out_put_mtt; 1260 1261 mr->ibmr.lkey = mr->ibmr.rkey = stag; 1262 mr->ibmr.pd = ibpd; 1263 mr->mem.va = virt; 1264 mr->mem.len = len; 1265 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access); 1266 mr->valid = 1; 1267 mr->type = ERDMA_MR_TYPE_NORMAL; 1268 1269 ret = regmr_cmd(dev, mr); 1270 if (ret) 1271 goto err_out_mr; 1272 1273 return &mr->ibmr; 1274 1275 err_out_mr: 1276 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 1277 mr->ibmr.lkey >> 8); 1278 1279 err_out_put_mtt: 1280 put_mtt_entries(dev, &mr->mem); 1281 1282 err_out_free: 1283 kfree(mr); 1284 1285 return ERR_PTR(ret); 1286 } 1287 1288 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1289 { 1290 struct erdma_mr *mr; 1291 struct erdma_dev *dev = to_edev(ibmr->device); 1292 struct erdma_cmdq_dereg_mr_req req; 1293 int ret; 1294 1295 mr = to_emr(ibmr); 1296 1297 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1298 CMDQ_OPCODE_DEREG_MR); 1299 1300 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) | 1301 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF); 1302 1303 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 1304 true); 1305 if (ret) 1306 return ret; 1307 1308 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); 1309 1310 put_mtt_entries(dev, &mr->mem); 1311 1312 kfree(mr); 1313 return 0; 1314 } 1315 1316 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1317 { 1318 struct erdma_cq *cq = to_ecq(ibcq); 1319 struct erdma_dev *dev = to_edev(ibcq->device); 1320 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1321 udata, struct erdma_ucontext, ibucontext); 1322 int err; 1323 struct erdma_cmdq_destroy_cq_req req; 1324 1325 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1326 CMDQ_OPCODE_DESTROY_CQ); 1327 req.cqn = cq->cqn; 1328 1329 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 1330 true); 1331 if (err) 1332 return err; 1333 1334 if (rdma_is_kernel_res(&cq->ibcq.res)) { 1335 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, 1336 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1337 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, 1338 cq->kern_cq.dbrec_dma); 1339 } else { 1340 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1341 put_mtt_entries(dev, &cq->user_cq.qbuf_mem); 1342 } 1343 1344 xa_erase(&dev->cq_xa, cq->cqn); 1345 1346 return 0; 1347 } 1348 1349 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1350 { 1351 struct erdma_qp *qp = to_eqp(ibqp); 1352 struct erdma_dev *dev = to_edev(ibqp->device); 1353 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1354 udata, struct erdma_ucontext, ibucontext); 1355 struct erdma_cmdq_destroy_qp_req req; 1356 union erdma_mod_qp_params params; 1357 int err; 1358 1359 down_write(&qp->state_lock); 1360 if (erdma_device_iwarp(dev)) { 1361 params.iwarp.state = ERDMA_QPS_IWARP_ERROR; 1362 erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp, 1363 ERDMA_QPA_IWARP_STATE); 1364 } else { 1365 params.rocev2.state = ERDMA_QPS_ROCEV2_ERROR; 1366 erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2, 1367 ERDMA_QPA_ROCEV2_STATE); 1368 } 1369 up_write(&qp->state_lock); 1370 1371 cancel_delayed_work_sync(&qp->reflush_dwork); 1372 1373 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1374 CMDQ_OPCODE_DESTROY_QP); 1375 req.qpn = QP_ID(qp); 1376 1377 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 1378 true); 1379 if (err) 1380 return err; 1381 1382 erdma_qp_put(qp); 1383 wait_for_completion(&qp->safe_free); 1384 1385 if (rdma_is_kernel_res(&qp->ibqp.res)) { 1386 free_kernel_qp(qp); 1387 } else { 1388 put_mtt_entries(dev, &qp->user_qp.sq_mem); 1389 put_mtt_entries(dev, &qp->user_qp.rq_mem); 1390 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page); 1391 } 1392 1393 if (qp->cep) 1394 erdma_cep_put(qp->cep); 1395 xa_erase(&dev->qp_xa, QP_ID(qp)); 1396 1397 return 0; 1398 } 1399 1400 void erdma_qp_get_ref(struct ib_qp *ibqp) 1401 { 1402 erdma_qp_get(to_eqp(ibqp)); 1403 } 1404 1405 void erdma_qp_put_ref(struct ib_qp *ibqp) 1406 { 1407 erdma_qp_put(to_eqp(ibqp)); 1408 } 1409 1410 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma) 1411 { 1412 struct rdma_user_mmap_entry *rdma_entry; 1413 struct erdma_user_mmap_entry *entry; 1414 pgprot_t prot; 1415 int err; 1416 1417 rdma_entry = rdma_user_mmap_entry_get(ctx, vma); 1418 if (!rdma_entry) 1419 return -EINVAL; 1420 1421 entry = to_emmap(rdma_entry); 1422 1423 switch (entry->mmap_flag) { 1424 case ERDMA_MMAP_IO_NC: 1425 /* map doorbell. */ 1426 prot = pgprot_device(vma->vm_page_prot); 1427 break; 1428 default: 1429 err = -EINVAL; 1430 goto put_entry; 1431 } 1432 1433 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE, 1434 prot, rdma_entry); 1435 1436 put_entry: 1437 rdma_user_mmap_entry_put(rdma_entry); 1438 return err; 1439 } 1440 1441 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 1442 { 1443 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry); 1444 1445 kfree(entry); 1446 } 1447 1448 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx, 1449 bool ext_db_en) 1450 { 1451 struct erdma_cmdq_ext_db_req req = {}; 1452 u64 val0, val1; 1453 int ret; 1454 1455 /* 1456 * CAP_SYS_RAWIO is required if hardware does not support extend 1457 * doorbell mechanism. 1458 */ 1459 if (!ext_db_en && !capable(CAP_SYS_RAWIO)) 1460 return -EPERM; 1461 1462 if (!ext_db_en) { 1463 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET; 1464 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET; 1465 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET; 1466 return 0; 1467 } 1468 1469 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1470 CMDQ_OPCODE_ALLOC_DB); 1471 1472 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) | 1473 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) | 1474 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1); 1475 1476 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1, 1477 true); 1478 if (ret) 1479 return ret; 1480 1481 ctx->ext_db.enable = true; 1482 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB); 1483 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB); 1484 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB); 1485 1486 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT); 1487 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT); 1488 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT); 1489 1490 return 0; 1491 } 1492 1493 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx) 1494 { 1495 struct erdma_cmdq_ext_db_req req = {}; 1496 int ret; 1497 1498 if (!ctx->ext_db.enable) 1499 return; 1500 1501 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1502 CMDQ_OPCODE_FREE_DB); 1503 1504 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) | 1505 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) | 1506 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1); 1507 1508 req.sdb_off = ctx->ext_db.sdb_off; 1509 req.rdb_off = ctx->ext_db.rdb_off; 1510 req.cdb_off = ctx->ext_db.cdb_off; 1511 1512 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 1513 true); 1514 if (ret) 1515 ibdev_err_ratelimited(&dev->ibdev, 1516 "free db resources failed %d", ret); 1517 } 1518 1519 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx) 1520 { 1521 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry); 1522 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry); 1523 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry); 1524 } 1525 1526 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) 1527 { 1528 struct erdma_ucontext *ctx = to_ectx(ibctx); 1529 struct erdma_dev *dev = to_edev(ibctx->device); 1530 int ret; 1531 struct erdma_uresp_alloc_ctx uresp = {}; 1532 1533 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) { 1534 ret = -ENOMEM; 1535 goto err_out; 1536 } 1537 1538 if (udata->outlen < sizeof(uresp)) { 1539 ret = -EINVAL; 1540 goto err_out; 1541 } 1542 1543 INIT_LIST_HEAD(&ctx->dbrecords_page_list); 1544 mutex_init(&ctx->dbrecords_page_mutex); 1545 1546 ret = alloc_db_resources(dev, ctx, 1547 !!(dev->attrs.cap_flags & 1548 ERDMA_DEV_CAP_FLAGS_EXTEND_DB)); 1549 if (ret) 1550 goto err_out; 1551 1552 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert( 1553 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb); 1554 if (!ctx->sq_db_mmap_entry) { 1555 ret = -ENOMEM; 1556 goto err_free_ext_db; 1557 } 1558 1559 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert( 1560 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb); 1561 if (!ctx->rq_db_mmap_entry) { 1562 ret = -EINVAL; 1563 goto err_put_mmap_entries; 1564 } 1565 1566 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert( 1567 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb); 1568 if (!ctx->cq_db_mmap_entry) { 1569 ret = -EINVAL; 1570 goto err_put_mmap_entries; 1571 } 1572 1573 uresp.dev_id = dev->pdev->device; 1574 1575 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1576 if (ret) 1577 goto err_put_mmap_entries; 1578 1579 return 0; 1580 1581 err_put_mmap_entries: 1582 erdma_uctx_user_mmap_entries_remove(ctx); 1583 1584 err_free_ext_db: 1585 free_db_resources(dev, ctx); 1586 1587 err_out: 1588 atomic_dec(&dev->num_ctx); 1589 return ret; 1590 } 1591 1592 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx) 1593 { 1594 struct erdma_dev *dev = to_edev(ibctx->device); 1595 struct erdma_ucontext *ctx = to_ectx(ibctx); 1596 1597 erdma_uctx_user_mmap_entries_remove(ctx); 1598 free_db_resources(dev, ctx); 1599 atomic_dec(&dev->num_ctx); 1600 } 1601 1602 static void erdma_attr_to_av(const struct rdma_ah_attr *ah_attr, 1603 struct erdma_av *av, u16 sport) 1604 { 1605 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr); 1606 1607 av->port = rdma_ah_get_port_num(ah_attr); 1608 av->sgid_index = grh->sgid_index; 1609 av->hop_limit = grh->hop_limit; 1610 av->traffic_class = grh->traffic_class; 1611 av->sl = rdma_ah_get_sl(ah_attr); 1612 1613 av->flow_label = grh->flow_label; 1614 av->udp_sport = sport; 1615 1616 ether_addr_copy(av->dmac, ah_attr->roce.dmac); 1617 memcpy(av->dgid, grh->dgid.raw, ERDMA_ROCEV2_GID_SIZE); 1618 1619 if (ipv6_addr_v4mapped((struct in6_addr *)&grh->dgid)) 1620 av->ntype = ERDMA_NETWORK_TYPE_IPV4; 1621 else 1622 av->ntype = ERDMA_NETWORK_TYPE_IPV6; 1623 } 1624 1625 static void erdma_av_to_attr(struct erdma_av *av, struct rdma_ah_attr *ah_attr) 1626 { 1627 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE; 1628 1629 rdma_ah_set_sl(ah_attr, av->sl); 1630 rdma_ah_set_port_num(ah_attr, av->port); 1631 rdma_ah_set_ah_flags(ah_attr, IB_AH_GRH); 1632 1633 rdma_ah_set_grh(ah_attr, NULL, av->flow_label, av->sgid_index, 1634 av->hop_limit, av->traffic_class); 1635 rdma_ah_set_dgid_raw(ah_attr, av->dgid); 1636 } 1637 1638 static int ib_qps_to_erdma_qps[ERDMA_PROTO_COUNT][IB_QPS_ERR + 1] = { 1639 [ERDMA_PROTO_IWARP] = { 1640 [IB_QPS_RESET] = ERDMA_QPS_IWARP_IDLE, 1641 [IB_QPS_INIT] = ERDMA_QPS_IWARP_IDLE, 1642 [IB_QPS_RTR] = ERDMA_QPS_IWARP_RTR, 1643 [IB_QPS_RTS] = ERDMA_QPS_IWARP_RTS, 1644 [IB_QPS_SQD] = ERDMA_QPS_IWARP_CLOSING, 1645 [IB_QPS_SQE] = ERDMA_QPS_IWARP_TERMINATE, 1646 [IB_QPS_ERR] = ERDMA_QPS_IWARP_ERROR, 1647 }, 1648 [ERDMA_PROTO_ROCEV2] = { 1649 [IB_QPS_RESET] = ERDMA_QPS_ROCEV2_RESET, 1650 [IB_QPS_INIT] = ERDMA_QPS_ROCEV2_INIT, 1651 [IB_QPS_RTR] = ERDMA_QPS_ROCEV2_RTR, 1652 [IB_QPS_RTS] = ERDMA_QPS_ROCEV2_RTS, 1653 [IB_QPS_SQD] = ERDMA_QPS_ROCEV2_SQD, 1654 [IB_QPS_SQE] = ERDMA_QPS_ROCEV2_SQE, 1655 [IB_QPS_ERR] = ERDMA_QPS_ROCEV2_ERROR, 1656 }, 1657 }; 1658 1659 static int erdma_qps_to_ib_qps[ERDMA_PROTO_COUNT][ERDMA_QPS_ROCEV2_COUNT] = { 1660 [ERDMA_PROTO_IWARP] = { 1661 [ERDMA_QPS_IWARP_IDLE] = IB_QPS_INIT, 1662 [ERDMA_QPS_IWARP_RTR] = IB_QPS_RTR, 1663 [ERDMA_QPS_IWARP_RTS] = IB_QPS_RTS, 1664 [ERDMA_QPS_IWARP_CLOSING] = IB_QPS_ERR, 1665 [ERDMA_QPS_IWARP_TERMINATE] = IB_QPS_ERR, 1666 [ERDMA_QPS_IWARP_ERROR] = IB_QPS_ERR, 1667 }, 1668 [ERDMA_PROTO_ROCEV2] = { 1669 [ERDMA_QPS_ROCEV2_RESET] = IB_QPS_RESET, 1670 [ERDMA_QPS_ROCEV2_INIT] = IB_QPS_INIT, 1671 [ERDMA_QPS_ROCEV2_RTR] = IB_QPS_RTR, 1672 [ERDMA_QPS_ROCEV2_RTS] = IB_QPS_RTS, 1673 [ERDMA_QPS_ROCEV2_SQD] = IB_QPS_SQD, 1674 [ERDMA_QPS_ROCEV2_SQE] = IB_QPS_SQE, 1675 [ERDMA_QPS_ROCEV2_ERROR] = IB_QPS_ERR, 1676 }, 1677 }; 1678 1679 static inline enum erdma_qps_iwarp ib_to_iwarp_qps(enum ib_qp_state state) 1680 { 1681 return ib_qps_to_erdma_qps[ERDMA_PROTO_IWARP][state]; 1682 } 1683 1684 static inline enum erdma_qps_rocev2 ib_to_rocev2_qps(enum ib_qp_state state) 1685 { 1686 return ib_qps_to_erdma_qps[ERDMA_PROTO_ROCEV2][state]; 1687 } 1688 1689 static inline enum ib_qp_state iwarp_to_ib_qps(enum erdma_qps_iwarp state) 1690 { 1691 return erdma_qps_to_ib_qps[ERDMA_PROTO_IWARP][state]; 1692 } 1693 1694 static inline enum ib_qp_state rocev2_to_ib_qps(enum erdma_qps_rocev2 state) 1695 { 1696 return erdma_qps_to_ib_qps[ERDMA_PROTO_ROCEV2][state]; 1697 } 1698 1699 static int erdma_check_qp_attrs(struct erdma_qp *qp, struct ib_qp_attr *attr, 1700 int attr_mask) 1701 { 1702 enum ib_qp_state cur_state, nxt_state; 1703 struct erdma_dev *dev = qp->dev; 1704 int ret = -EINVAL; 1705 1706 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) { 1707 ret = -EOPNOTSUPP; 1708 goto out; 1709 } 1710 1711 if ((attr_mask & IB_QP_PORT) && 1712 !rdma_is_port_valid(&dev->ibdev, attr->port_num)) 1713 goto out; 1714 1715 if (erdma_device_rocev2(dev)) { 1716 cur_state = (attr_mask & IB_QP_CUR_STATE) ? 1717 attr->cur_qp_state : 1718 rocev2_to_ib_qps(qp->attrs.rocev2.state); 1719 1720 nxt_state = (attr_mask & IB_QP_STATE) ? attr->qp_state : 1721 cur_state; 1722 1723 if (!ib_modify_qp_is_ok(cur_state, nxt_state, qp->ibqp.qp_type, 1724 attr_mask)) 1725 goto out; 1726 1727 if ((attr_mask & IB_QP_AV) && 1728 erdma_check_gid_attr( 1729 rdma_ah_read_grh(&attr->ah_attr)->sgid_attr)) 1730 goto out; 1731 1732 if ((attr_mask & IB_QP_PKEY_INDEX) && 1733 attr->pkey_index >= ERDMA_MAX_PKEYS) 1734 goto out; 1735 } 1736 1737 return 0; 1738 1739 out: 1740 return ret; 1741 } 1742 1743 static void erdma_init_mod_qp_params_rocev2( 1744 struct erdma_qp *qp, struct erdma_mod_qp_params_rocev2 *params, 1745 int *erdma_attr_mask, struct ib_qp_attr *attr, int ib_attr_mask) 1746 { 1747 enum erdma_qpa_mask_rocev2 to_modify_attrs = 0; 1748 enum erdma_qps_rocev2 cur_state, nxt_state; 1749 u16 udp_sport; 1750 1751 if (ib_attr_mask & IB_QP_CUR_STATE) 1752 cur_state = ib_to_rocev2_qps(attr->cur_qp_state); 1753 else 1754 cur_state = qp->attrs.rocev2.state; 1755 1756 if (ib_attr_mask & IB_QP_STATE) 1757 nxt_state = ib_to_rocev2_qps(attr->qp_state); 1758 else 1759 nxt_state = cur_state; 1760 1761 to_modify_attrs |= ERDMA_QPA_ROCEV2_STATE; 1762 params->state = nxt_state; 1763 1764 if (ib_attr_mask & IB_QP_QKEY) { 1765 to_modify_attrs |= ERDMA_QPA_ROCEV2_QKEY; 1766 params->qkey = attr->qkey; 1767 } 1768 1769 if (ib_attr_mask & IB_QP_SQ_PSN) { 1770 to_modify_attrs |= ERDMA_QPA_ROCEV2_SQ_PSN; 1771 params->sq_psn = attr->sq_psn; 1772 } 1773 1774 if (ib_attr_mask & IB_QP_RQ_PSN) { 1775 to_modify_attrs |= ERDMA_QPA_ROCEV2_RQ_PSN; 1776 params->rq_psn = attr->rq_psn; 1777 } 1778 1779 if (ib_attr_mask & IB_QP_DEST_QPN) { 1780 to_modify_attrs |= ERDMA_QPA_ROCEV2_DST_QPN; 1781 params->dst_qpn = attr->dest_qp_num; 1782 } 1783 1784 if (ib_attr_mask & IB_QP_AV) { 1785 to_modify_attrs |= ERDMA_QPA_ROCEV2_AV; 1786 udp_sport = rdma_get_udp_sport(attr->ah_attr.grh.flow_label, 1787 QP_ID(qp), params->dst_qpn); 1788 erdma_attr_to_av(&attr->ah_attr, ¶ms->av, udp_sport); 1789 } 1790 1791 *erdma_attr_mask = to_modify_attrs; 1792 } 1793 1794 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1795 struct ib_udata *udata) 1796 { 1797 struct erdma_qp *qp = to_eqp(ibqp); 1798 union erdma_mod_qp_params params; 1799 int ret = 0, erdma_attr_mask = 0; 1800 1801 down_write(&qp->state_lock); 1802 1803 ret = erdma_check_qp_attrs(qp, attr, attr_mask); 1804 if (ret) 1805 goto out; 1806 1807 if (erdma_device_iwarp(qp->dev)) { 1808 if (attr_mask & IB_QP_STATE) { 1809 erdma_attr_mask |= ERDMA_QPA_IWARP_STATE; 1810 params.iwarp.state = ib_to_iwarp_qps(attr->qp_state); 1811 } 1812 1813 ret = erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp, 1814 erdma_attr_mask); 1815 } else { 1816 erdma_init_mod_qp_params_rocev2( 1817 qp, ¶ms.rocev2, &erdma_attr_mask, attr, attr_mask); 1818 1819 ret = erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2, 1820 erdma_attr_mask); 1821 } 1822 1823 out: 1824 up_write(&qp->state_lock); 1825 return ret; 1826 } 1827 1828 static enum ib_qp_state query_qp_state(struct erdma_qp *qp) 1829 { 1830 if (erdma_device_iwarp(qp->dev)) 1831 return iwarp_to_ib_qps(qp->attrs.iwarp.state); 1832 else 1833 return rocev2_to_ib_qps(qp->attrs.rocev2.state); 1834 } 1835 1836 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 1837 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 1838 { 1839 struct erdma_cmdq_query_qp_req_rocev2 req; 1840 struct erdma_dev *dev; 1841 struct erdma_qp *qp; 1842 u64 resp0, resp1; 1843 int ret; 1844 1845 if (ibqp && qp_attr && qp_init_attr) { 1846 qp = to_eqp(ibqp); 1847 dev = to_edev(ibqp->device); 1848 } else { 1849 return -EINVAL; 1850 } 1851 1852 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1853 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1854 1855 qp_attr->cap.max_send_wr = qp->attrs.sq_size; 1856 qp_attr->cap.max_recv_wr = qp->attrs.rq_size; 1857 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge; 1858 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge; 1859 1860 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu); 1861 qp_attr->max_rd_atomic = qp->attrs.irq_size; 1862 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size; 1863 1864 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | 1865 IB_ACCESS_REMOTE_WRITE | 1866 IB_ACCESS_REMOTE_READ; 1867 1868 qp_init_attr->cap = qp_attr->cap; 1869 1870 if (erdma_device_rocev2(dev)) { 1871 /* Query hardware to get some attributes */ 1872 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1873 CMDQ_OPCODE_QUERY_QP); 1874 req.qpn = QP_ID(qp); 1875 1876 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, 1877 &resp1, true); 1878 if (ret) 1879 return ret; 1880 1881 qp_attr->sq_psn = 1882 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK, resp0); 1883 qp_attr->rq_psn = 1884 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK, resp0); 1885 qp_attr->qp_state = rocev2_to_ib_qps(FIELD_GET( 1886 ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK, resp0)); 1887 qp_attr->cur_qp_state = qp_attr->qp_state; 1888 qp_attr->sq_draining = FIELD_GET( 1889 ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK, resp0); 1890 1891 qp_attr->pkey_index = 0; 1892 qp_attr->dest_qp_num = qp->attrs.rocev2.dst_qpn; 1893 1894 if (qp->ibqp.qp_type == IB_QPT_RC) 1895 erdma_av_to_attr(&qp->attrs.rocev2.av, 1896 &qp_attr->ah_attr); 1897 } else { 1898 qp_attr->qp_state = query_qp_state(qp); 1899 qp_attr->cur_qp_state = qp_attr->qp_state; 1900 } 1901 1902 return 0; 1903 } 1904 1905 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, 1906 struct erdma_ureq_create_cq *ureq) 1907 { 1908 int ret; 1909 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1910 1911 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va, 1912 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K, 1913 true); 1914 if (ret) 1915 return ret; 1916 1917 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va, 1918 &cq->user_cq.user_dbr_page, 1919 &cq->user_cq.dbrec_dma); 1920 if (ret) 1921 put_mtt_entries(dev, &cq->user_cq.qbuf_mem); 1922 1923 return ret; 1924 } 1925 1926 static int erdma_init_kernel_cq(struct erdma_cq *cq) 1927 { 1928 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1929 1930 cq->kern_cq.qbuf = 1931 dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, 1932 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); 1933 if (!cq->kern_cq.qbuf) 1934 return -ENOMEM; 1935 1936 cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, 1937 &cq->kern_cq.dbrec_dma); 1938 if (!cq->kern_cq.dbrec) 1939 goto err_out; 1940 1941 spin_lock_init(&cq->kern_cq.lock); 1942 /* use default cqdb addr */ 1943 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET; 1944 1945 return 0; 1946 1947 err_out: 1948 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, 1949 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1950 1951 return -ENOMEM; 1952 } 1953 1954 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1955 struct uverbs_attr_bundle *attrs) 1956 { 1957 struct ib_udata *udata = &attrs->driver_udata; 1958 struct erdma_cq *cq = to_ecq(ibcq); 1959 struct erdma_dev *dev = to_edev(ibcq->device); 1960 unsigned int depth = attr->cqe; 1961 int ret; 1962 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1963 udata, struct erdma_ucontext, ibucontext); 1964 1965 if (depth > dev->attrs.max_cqe) 1966 return -EINVAL; 1967 1968 depth = roundup_pow_of_two(depth); 1969 cq->ibcq.cqe = depth; 1970 cq->depth = depth; 1971 cq->assoc_eqn = attr->comp_vector + 1; 1972 1973 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq, 1974 XA_LIMIT(1, dev->attrs.max_cq - 1), 1975 &dev->next_alloc_cqn, GFP_KERNEL); 1976 if (ret < 0) 1977 return ret; 1978 1979 if (!rdma_is_kernel_res(&ibcq->res)) { 1980 struct erdma_ureq_create_cq ureq; 1981 struct erdma_uresp_create_cq uresp; 1982 1983 ret = ib_copy_from_udata(&ureq, udata, 1984 min(udata->inlen, sizeof(ureq))); 1985 if (ret) 1986 goto err_out_xa; 1987 1988 ret = erdma_init_user_cq(ctx, cq, &ureq); 1989 if (ret) 1990 goto err_out_xa; 1991 1992 uresp.cq_id = cq->cqn; 1993 uresp.num_cqe = depth; 1994 1995 ret = ib_copy_to_udata(udata, &uresp, 1996 min(sizeof(uresp), udata->outlen)); 1997 if (ret) 1998 goto err_free_res; 1999 } else { 2000 ret = erdma_init_kernel_cq(cq); 2001 if (ret) 2002 goto err_out_xa; 2003 } 2004 2005 ret = create_cq_cmd(ctx, cq); 2006 if (ret) 2007 goto err_free_res; 2008 2009 return 0; 2010 2011 err_free_res: 2012 if (!rdma_is_kernel_res(&ibcq->res)) { 2013 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 2014 put_mtt_entries(dev, &cq->user_cq.qbuf_mem); 2015 } else { 2016 dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT, 2017 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 2018 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, 2019 cq->kern_cq.dbrec_dma); 2020 } 2021 2022 err_out_xa: 2023 xa_erase(&dev->cq_xa, cq->cqn); 2024 2025 return ret; 2026 } 2027 2028 void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext) 2029 { 2030 } 2031 2032 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu) 2033 { 2034 struct erdma_cmdq_config_mtu_req req; 2035 2036 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 2037 CMDQ_OPCODE_CONF_MTU); 2038 req.mtu = mtu; 2039 2040 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); 2041 } 2042 2043 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason) 2044 { 2045 struct ib_event event; 2046 2047 event.device = &dev->ibdev; 2048 event.element.port_num = 1; 2049 event.event = reason; 2050 2051 ib_dispatch_event(&event); 2052 } 2053 2054 enum counters { 2055 ERDMA_STATS_TX_REQS_CNT, 2056 ERDMA_STATS_TX_PACKETS_CNT, 2057 ERDMA_STATS_TX_BYTES_CNT, 2058 ERDMA_STATS_TX_DISABLE_DROP_CNT, 2059 ERDMA_STATS_TX_BPS_METER_DROP_CNT, 2060 ERDMA_STATS_TX_PPS_METER_DROP_CNT, 2061 2062 ERDMA_STATS_RX_PACKETS_CNT, 2063 ERDMA_STATS_RX_BYTES_CNT, 2064 ERDMA_STATS_RX_DISABLE_DROP_CNT, 2065 ERDMA_STATS_RX_BPS_METER_DROP_CNT, 2066 ERDMA_STATS_RX_PPS_METER_DROP_CNT, 2067 2068 ERDMA_STATS_MAX 2069 }; 2070 2071 static const struct rdma_stat_desc erdma_descs[] = { 2072 [ERDMA_STATS_TX_REQS_CNT].name = "tx_reqs_cnt", 2073 [ERDMA_STATS_TX_PACKETS_CNT].name = "tx_packets_cnt", 2074 [ERDMA_STATS_TX_BYTES_CNT].name = "tx_bytes_cnt", 2075 [ERDMA_STATS_TX_DISABLE_DROP_CNT].name = "tx_disable_drop_cnt", 2076 [ERDMA_STATS_TX_BPS_METER_DROP_CNT].name = "tx_bps_limit_drop_cnt", 2077 [ERDMA_STATS_TX_PPS_METER_DROP_CNT].name = "tx_pps_limit_drop_cnt", 2078 [ERDMA_STATS_RX_PACKETS_CNT].name = "rx_packets_cnt", 2079 [ERDMA_STATS_RX_BYTES_CNT].name = "rx_bytes_cnt", 2080 [ERDMA_STATS_RX_DISABLE_DROP_CNT].name = "rx_disable_drop_cnt", 2081 [ERDMA_STATS_RX_BPS_METER_DROP_CNT].name = "rx_bps_limit_drop_cnt", 2082 [ERDMA_STATS_RX_PPS_METER_DROP_CNT].name = "rx_pps_limit_drop_cnt", 2083 }; 2084 2085 struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device, 2086 u32 port_num) 2087 { 2088 return rdma_alloc_hw_stats_struct(erdma_descs, ERDMA_STATS_MAX, 2089 RDMA_HW_STATS_DEFAULT_LIFESPAN); 2090 } 2091 2092 static int erdma_query_hw_stats(struct erdma_dev *dev, 2093 struct rdma_hw_stats *stats) 2094 { 2095 struct erdma_cmdq_query_stats_resp *resp; 2096 struct erdma_cmdq_query_req req; 2097 dma_addr_t dma_addr; 2098 int err; 2099 2100 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 2101 CMDQ_OPCODE_GET_STATS); 2102 2103 resp = dma_pool_zalloc(dev->resp_pool, GFP_KERNEL, &dma_addr); 2104 if (!resp) 2105 return -ENOMEM; 2106 2107 req.target_addr = dma_addr; 2108 req.target_length = ERDMA_HW_RESP_SIZE; 2109 2110 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 2111 true); 2112 if (err) 2113 goto out; 2114 2115 if (resp->hdr.magic != ERDMA_HW_RESP_MAGIC) { 2116 err = -EINVAL; 2117 goto out; 2118 } 2119 2120 memcpy(&stats->value[0], &resp->tx_req_cnt, 2121 sizeof(u64) * stats->num_counters); 2122 2123 out: 2124 dma_pool_free(dev->resp_pool, resp, dma_addr); 2125 2126 return err; 2127 } 2128 2129 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, 2130 u32 port, int index) 2131 { 2132 struct erdma_dev *dev = to_edev(ibdev); 2133 int ret; 2134 2135 if (port == 0) 2136 return 0; 2137 2138 ret = erdma_query_hw_stats(dev, stats); 2139 if (ret) 2140 return ret; 2141 2142 return stats->num_counters; 2143 } 2144 2145 enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev, u32 port_num) 2146 { 2147 return IB_LINK_LAYER_ETHERNET; 2148 } 2149 2150 static int erdma_set_gid(struct erdma_dev *dev, u8 op, u32 idx, 2151 const union ib_gid *gid) 2152 { 2153 struct erdma_cmdq_set_gid_req req; 2154 u8 ntype; 2155 2156 req.cfg = FIELD_PREP(ERDMA_CMD_SET_GID_SGID_IDX_MASK, idx) | 2157 FIELD_PREP(ERDMA_CMD_SET_GID_OP_MASK, op); 2158 2159 if (op == ERDMA_SET_GID_OP_ADD) { 2160 if (ipv6_addr_v4mapped((struct in6_addr *)gid)) 2161 ntype = ERDMA_NETWORK_TYPE_IPV4; 2162 else 2163 ntype = ERDMA_NETWORK_TYPE_IPV6; 2164 2165 req.cfg |= FIELD_PREP(ERDMA_CMD_SET_GID_NTYPE_MASK, ntype); 2166 2167 memcpy(&req.gid, gid, ERDMA_ROCEV2_GID_SIZE); 2168 } 2169 2170 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 2171 CMDQ_OPCODE_SET_GID); 2172 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 2173 true); 2174 } 2175 2176 int erdma_add_gid(const struct ib_gid_attr *attr, void **context) 2177 { 2178 struct erdma_dev *dev = to_edev(attr->device); 2179 int ret; 2180 2181 ret = erdma_check_gid_attr(attr); 2182 if (ret) 2183 return ret; 2184 2185 return erdma_set_gid(dev, ERDMA_SET_GID_OP_ADD, attr->index, 2186 &attr->gid); 2187 } 2188 2189 int erdma_del_gid(const struct ib_gid_attr *attr, void **context) 2190 { 2191 return erdma_set_gid(to_edev(attr->device), ERDMA_SET_GID_OP_DEL, 2192 attr->index, NULL); 2193 } 2194 2195 int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) 2196 { 2197 if (index >= ERDMA_MAX_PKEYS) 2198 return -EINVAL; 2199 2200 *pkey = ERDMA_DEFAULT_PKEY; 2201 return 0; 2202 } 2203 2204 void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av) 2205 { 2206 av_cfg->cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_AV_FL_MASK, av->flow_label) | 2207 FIELD_PREP(ERDMA_CMD_CREATE_AV_NTYPE_MASK, av->ntype); 2208 2209 av_cfg->traffic_class = av->traffic_class; 2210 av_cfg->hop_limit = av->hop_limit; 2211 av_cfg->sl = av->sl; 2212 2213 av_cfg->udp_sport = av->udp_sport; 2214 av_cfg->sgid_index = av->sgid_index; 2215 2216 ether_addr_copy(av_cfg->dmac, av->dmac); 2217 memcpy(av_cfg->dgid, av->dgid, ERDMA_ROCEV2_GID_SIZE); 2218 } 2219 2220 int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 2221 struct ib_udata *udata) 2222 { 2223 const struct ib_global_route *grh = 2224 rdma_ah_read_grh(init_attr->ah_attr); 2225 struct erdma_dev *dev = to_edev(ibah->device); 2226 struct erdma_pd *pd = to_epd(ibah->pd); 2227 struct erdma_ah *ah = to_eah(ibah); 2228 struct erdma_cmdq_create_ah_req req; 2229 u32 udp_sport; 2230 int ret; 2231 2232 ret = erdma_check_gid_attr(grh->sgid_attr); 2233 if (ret) 2234 return ret; 2235 2236 ret = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_AH]); 2237 if (ret < 0) 2238 return ret; 2239 2240 ah->ahn = ret; 2241 2242 if (grh->flow_label) 2243 udp_sport = rdma_flow_label_to_udp_sport(grh->flow_label); 2244 else 2245 udp_sport = 2246 IB_ROCE_UDP_ENCAP_VALID_PORT_MIN + (ah->ahn & 0x3FFF); 2247 2248 erdma_attr_to_av(init_attr->ah_attr, &ah->av, udp_sport); 2249 2250 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 2251 CMDQ_OPCODE_CREATE_AH); 2252 2253 req.pdn = pd->pdn; 2254 req.ahn = ah->ahn; 2255 erdma_set_av_cfg(&req.av_cfg, &ah->av); 2256 2257 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 2258 init_attr->flags & RDMA_CREATE_AH_SLEEPABLE); 2259 if (ret) { 2260 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn); 2261 return ret; 2262 } 2263 2264 return 0; 2265 } 2266 2267 int erdma_destroy_ah(struct ib_ah *ibah, u32 flags) 2268 { 2269 struct erdma_dev *dev = to_edev(ibah->device); 2270 struct erdma_pd *pd = to_epd(ibah->pd); 2271 struct erdma_ah *ah = to_eah(ibah); 2272 struct erdma_cmdq_destroy_ah_req req; 2273 int ret; 2274 2275 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 2276 CMDQ_OPCODE_DESTROY_AH); 2277 2278 req.pdn = pd->pdn; 2279 req.ahn = ah->ahn; 2280 2281 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, 2282 flags & RDMA_DESTROY_AH_SLEEPABLE); 2283 if (ret) 2284 return ret; 2285 2286 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn); 2287 2288 return 0; 2289 } 2290 2291 int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 2292 { 2293 struct erdma_ah *ah = to_eah(ibah); 2294 2295 memset(ah_attr, 0, sizeof(*ah_attr)); 2296 erdma_av_to_attr(&ah->av, ah_attr); 2297 2298 return 0; 2299 } 2300