1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_query_device - get device attributes 7 * @ibdev: device pointer from stack 8 * @props: returning device attributes 9 * @udata: user data 10 */ 11 static int irdma_query_device(struct ib_device *ibdev, 12 struct ib_device_attr *props, 13 struct ib_udata *udata) 14 { 15 struct irdma_device *iwdev = to_iwdev(ibdev); 16 struct irdma_pci_f *rf = iwdev->rf; 17 struct pci_dev *pcidev = iwdev->rf->pcidev; 18 struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; 19 int err; 20 21 err = ib_is_udata_in_empty(udata); 22 if (err) 23 return err; 24 25 memset(props, 0, sizeof(*props)); 26 addrconf_addr_eui48((u8 *)&props->sys_image_guid, 27 iwdev->netdev->dev_addr); 28 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | 29 irdma_fw_minor_ver(&rf->sc_dev); 30 props->device_cap_flags = IB_DEVICE_MEM_WINDOW | 31 IB_DEVICE_MEM_MGT_EXTENSIONS; 32 if (hw_attrs->uk_attrs.hw_rev < IRDMA_GEN_3) 33 props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 34 props->vendor_id = pcidev->vendor; 35 props->vendor_part_id = pcidev->device; 36 37 props->hw_ver = rf->pcidev->revision; 38 props->page_size_cap = hw_attrs->page_size_cap; 39 props->max_mr_size = hw_attrs->max_mr_size; 40 props->max_qp = rf->max_qp - rf->used_qps; 41 props->max_qp_wr = hw_attrs->max_qp_wr; 42 props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 43 props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 44 props->max_cq = rf->max_cq - rf->used_cqs; 45 props->max_cqe = rf->max_cqe - 1; 46 props->max_mr = rf->max_mr - rf->used_mrs; 47 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) 48 props->max_mw = props->max_mr; 49 props->max_pd = rf->max_pd - rf->used_pds; 50 props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; 51 props->max_qp_rd_atom = hw_attrs->max_hw_ird; 52 props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; 53 if (rdma_protocol_roce(ibdev, 1)) { 54 props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN; 55 props->max_pkeys = IRDMA_PKEY_TBL_SZ; 56 } 57 58 props->max_ah = rf->max_ah; 59 props->max_mcast_grp = rf->max_mcg; 60 props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; 61 props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; 62 props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; 63 props->max_srq = rf->max_srq - rf->used_srqs; 64 props->max_srq_wr = IRDMA_MAX_SRQ_WRS; 65 props->max_srq_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 66 if (hw_attrs->uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 67 props->atomic_cap = IB_ATOMIC_HCA; 68 else 69 props->atomic_cap = IB_ATOMIC_NONE; 70 props->masked_atomic_cap = props->atomic_cap; 71 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) { 72 #define HCA_CORE_CLOCK_KHZ 1000000UL 73 props->timestamp_mask = GENMASK(31, 0); 74 props->hca_core_clock = HCA_CORE_CLOCK_KHZ; 75 } 76 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) 77 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; 78 79 return ib_respond_empty_udata(udata); 80 } 81 82 /** 83 * irdma_query_port - get port attributes 84 * @ibdev: device pointer from stack 85 * @port: port number for query 86 * @props: returning device attributes 87 */ 88 static int irdma_query_port(struct ib_device *ibdev, u32 port, 89 struct ib_port_attr *props) 90 { 91 struct irdma_device *iwdev = to_iwdev(ibdev); 92 struct net_device *netdev = iwdev->netdev; 93 94 /* no need to zero out pros here. done by caller */ 95 96 props->max_mtu = IB_MTU_4096; 97 props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); 98 props->lid = 1; 99 props->lmc = 0; 100 props->sm_lid = 0; 101 props->sm_sl = 0; 102 if (netif_carrier_ok(netdev) && netif_running(netdev)) { 103 props->state = IB_PORT_ACTIVE; 104 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 105 } else { 106 props->state = IB_PORT_DOWN; 107 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 108 } 109 110 ib_get_eth_speed(ibdev, port, &props->active_speed, 111 &props->active_width); 112 113 if (rdma_protocol_roce(ibdev, 1)) { 114 props->gid_tbl_len = 32; 115 props->ip_gids = true; 116 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 117 } else { 118 props->gid_tbl_len = 1; 119 } 120 props->qkey_viol_cntr = 0; 121 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 122 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 123 124 return 0; 125 } 126 127 /** 128 * irdma_disassociate_ucontext - Disassociate user context 129 * @context: ib user context 130 */ 131 static void irdma_disassociate_ucontext(struct ib_ucontext *context) 132 { 133 } 134 135 static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, 136 struct vm_area_struct *vma) 137 { 138 u64 pfn; 139 140 if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) 141 return -EINVAL; 142 143 vma->vm_private_data = ucontext; 144 pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + 145 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 146 147 return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, 148 pgprot_noncached(vma->vm_page_prot), NULL); 149 } 150 151 static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 152 { 153 struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); 154 155 kfree(entry); 156 } 157 158 static struct rdma_user_mmap_entry* 159 irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, 160 enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) 161 { 162 struct irdma_user_mmap_entry *entry = kzalloc_obj(*entry); 163 int ret; 164 165 if (!entry) 166 return NULL; 167 168 entry->bar_offset = bar_offset; 169 entry->mmap_flag = mmap_flag; 170 171 ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, 172 &entry->rdma_entry, PAGE_SIZE); 173 if (ret) { 174 kfree(entry); 175 return NULL; 176 } 177 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 178 179 return &entry->rdma_entry; 180 } 181 182 /** 183 * irdma_mmap - user memory map 184 * @context: context created during alloc 185 * @vma: kernel info for user memory map 186 */ 187 static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 188 { 189 struct rdma_user_mmap_entry *rdma_entry; 190 struct irdma_user_mmap_entry *entry; 191 struct irdma_ucontext *ucontext; 192 u64 pfn; 193 int ret; 194 195 ucontext = to_ucontext(context); 196 197 /* Legacy support for libi40iw with hard-coded mmap key */ 198 if (ucontext->legacy_mode) 199 return irdma_mmap_legacy(ucontext, vma); 200 201 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 202 if (!rdma_entry) { 203 ibdev_dbg(&ucontext->iwdev->ibdev, 204 "VERBS: pgoff[0x%lx] does not have valid entry\n", 205 vma->vm_pgoff); 206 return -EINVAL; 207 } 208 209 entry = to_irdma_mmap_entry(rdma_entry); 210 ibdev_dbg(&ucontext->iwdev->ibdev, 211 "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n", 212 entry->bar_offset, entry->mmap_flag); 213 214 pfn = (entry->bar_offset + 215 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 216 217 switch (entry->mmap_flag) { 218 case IRDMA_MMAP_IO_NC: 219 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 220 pgprot_noncached(vma->vm_page_prot), 221 rdma_entry); 222 break; 223 case IRDMA_MMAP_IO_WC: 224 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 225 pgprot_writecombine(vma->vm_page_prot), 226 rdma_entry); 227 break; 228 default: 229 ret = -EINVAL; 230 } 231 232 if (ret) 233 ibdev_dbg(&ucontext->iwdev->ibdev, 234 "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n", 235 entry->bar_offset, entry->mmap_flag, ret); 236 rdma_user_mmap_entry_put(rdma_entry); 237 238 return ret; 239 } 240 241 /** 242 * irdma_alloc_push_page - allocate a push page for qp 243 * @iwqp: qp pointer 244 */ 245 static void irdma_alloc_push_page(struct irdma_qp *iwqp) 246 { 247 struct irdma_cqp_request *cqp_request; 248 struct cqp_cmds_info *cqp_info; 249 struct irdma_device *iwdev = iwqp->iwdev; 250 struct irdma_sc_qp *qp = &iwqp->sc_qp; 251 int status; 252 253 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 254 if (!cqp_request) 255 return; 256 257 cqp_info = &cqp_request->info; 258 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 259 cqp_info->post_sq = 1; 260 cqp_info->in.u.manage_push_page.info.push_idx = 0; 261 cqp_info->in.u.manage_push_page.info.qs_handle = 262 qp->vsi->qos[qp->user_pri].qs_handle; 263 cqp_info->in.u.manage_push_page.info.free_page = 0; 264 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 265 cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; 266 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 267 268 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 269 if (!status && cqp_request->compl_info.op_ret_val < 270 iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { 271 qp->push_idx = cqp_request->compl_info.op_ret_val; 272 qp->push_offset = 0; 273 } 274 275 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 276 } 277 278 /** 279 * irdma_alloc_ucontext - Allocate the user context data structure 280 * @uctx: uverbs context pointer 281 * @udata: user data 282 * 283 * This keeps track of all objects associated with a particular 284 * user-mode client. 285 */ 286 static int irdma_alloc_ucontext(struct ib_ucontext *uctx, 287 struct ib_udata *udata) 288 { 289 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) 290 struct ib_device *ibdev = uctx->device; 291 struct irdma_device *iwdev = to_iwdev(ibdev); 292 struct irdma_alloc_ucontext_req req = {}; 293 struct irdma_alloc_ucontext_resp uresp = {}; 294 struct irdma_ucontext *ucontext = to_ucontext(uctx); 295 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 296 int ret; 297 298 if (udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) 299 return -EINVAL; 300 301 ret = ib_copy_validate_udata_in_cm(udata, req, rsvd8, 302 IRDMA_ALLOC_UCTX_USE_RAW_ATTR | 303 IRDMA_SUPPORT_WQE_FORMAT_V2); 304 if (ret) 305 return ret; 306 307 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 308 goto ver_error; 309 310 ucontext->iwdev = iwdev; 311 ucontext->abi_ver = req.userspace_ver; 312 313 if (!(req.comp_mask & IRDMA_SUPPORT_WQE_FORMAT_V2) && 314 uk_attrs->hw_rev >= IRDMA_GEN_3) 315 return -EOPNOTSUPP; 316 317 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR) 318 ucontext->use_raw_attrs = true; 319 320 /* GEN_1 legacy support with libi40iw */ 321 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { 322 if (uk_attrs->hw_rev != IRDMA_GEN_1) 323 return -EOPNOTSUPP; 324 325 ucontext->legacy_mode = true; 326 uresp.max_qps = iwdev->rf->max_qp; 327 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 328 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 329 uresp.kernel_ver = req.userspace_ver; 330 ret = ib_respond_udata(udata, uresp); 331 if (ret) 332 return ret; 333 } else { 334 u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 335 336 ucontext->db_mmap_entry = 337 irdma_user_mmap_entry_insert(ucontext, bar_off, 338 IRDMA_MMAP_IO_NC, 339 &uresp.db_mmap_key); 340 if (!ucontext->db_mmap_entry) 341 return -ENOMEM; 342 343 uresp.kernel_ver = IRDMA_ABI_VER; 344 uresp.feature_flags = uk_attrs->feature_flags; 345 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 346 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 347 uresp.max_hw_inline = uk_attrs->max_hw_inline; 348 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 349 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 350 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 351 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 352 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 353 uresp.hw_rev = uk_attrs->hw_rev; 354 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; 355 uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size; 356 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE; 357 uresp.max_hw_srq_quanta = uk_attrs->max_hw_srq_quanta; 358 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA; 359 ret = ib_respond_udata(udata, uresp); 360 if (ret) { 361 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 362 return ret; 363 } 364 } 365 366 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 367 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 368 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 369 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 370 INIT_LIST_HEAD(&ucontext->srq_reg_mem_list); 371 spin_lock_init(&ucontext->srq_reg_mem_list_lock); 372 373 return 0; 374 375 ver_error: 376 ibdev_err(&iwdev->ibdev, 377 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 378 req.userspace_ver, IRDMA_ABI_VER); 379 return -EINVAL; 380 } 381 382 /** 383 * irdma_dealloc_ucontext - deallocate the user context data structure 384 * @context: user context created during alloc 385 */ 386 static void irdma_dealloc_ucontext(struct ib_ucontext *context) 387 { 388 struct irdma_ucontext *ucontext = to_ucontext(context); 389 390 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 391 } 392 393 /** 394 * irdma_alloc_pd - allocate protection domain 395 * @pd: PD pointer 396 * @udata: user data 397 */ 398 static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 399 { 400 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) 401 struct irdma_pd *iwpd = to_iwpd(pd); 402 struct irdma_device *iwdev = to_iwdev(pd->device); 403 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 404 struct irdma_pci_f *rf = iwdev->rf; 405 struct irdma_alloc_pd_resp uresp = {}; 406 struct irdma_sc_pd *sc_pd; 407 u32 pd_id = 0; 408 int err; 409 410 if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) 411 return -EINVAL; 412 413 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 414 &rf->next_pd); 415 if (err) 416 return err; 417 418 sc_pd = &iwpd->sc_pd; 419 if (udata) { 420 struct irdma_ucontext *ucontext = 421 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 422 ibucontext); 423 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 424 uresp.pd_id = pd_id; 425 err = ib_respond_udata(udata, uresp); 426 if (err) 427 goto error; 428 } else { 429 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 430 } 431 432 return 0; 433 error: 434 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 435 436 return err; 437 } 438 439 /** 440 * irdma_dealloc_pd - deallocate pd 441 * @ibpd: ptr of pd to be deallocated 442 * @udata: user data 443 */ 444 static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 445 { 446 struct irdma_pd *iwpd = to_iwpd(ibpd); 447 struct irdma_device *iwdev = to_iwdev(ibpd->device); 448 449 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 450 451 return 0; 452 } 453 454 /** 455 * irdma_get_pbl - Retrieve pbl from a list given a virtual 456 * address 457 * @va: user virtual address 458 * @pbl_list: pbl list to search in (QP's or CQ's) 459 */ 460 static struct irdma_pbl *irdma_get_pbl(unsigned long va, 461 struct list_head *pbl_list) 462 { 463 struct irdma_pbl *iwpbl; 464 465 list_for_each_entry (iwpbl, pbl_list, list) { 466 if (iwpbl->user_base == va) { 467 list_del(&iwpbl->list); 468 iwpbl->on_list = false; 469 return iwpbl; 470 } 471 } 472 473 return NULL; 474 } 475 476 /** 477 * irdma_clean_cqes - clean cq entries for qp 478 * @iwqp: qp ptr (user or kernel) 479 * @iwcq: cq ptr 480 */ 481 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) 482 { 483 struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; 484 unsigned long flags; 485 486 spin_lock_irqsave(&iwcq->lock, flags); 487 irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); 488 spin_unlock_irqrestore(&iwcq->lock, flags); 489 } 490 491 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) 492 { 493 if (iwqp->push_db_mmap_entry) { 494 rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); 495 iwqp->push_db_mmap_entry = NULL; 496 } 497 if (iwqp->push_wqe_mmap_entry) { 498 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 499 iwqp->push_wqe_mmap_entry = NULL; 500 } 501 } 502 503 static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, 504 struct irdma_qp *iwqp, 505 u64 *push_wqe_mmap_key, 506 u64 *push_db_mmap_key) 507 { 508 struct irdma_device *iwdev = ucontext->iwdev; 509 u64 rsvd, bar_off; 510 511 rsvd = IRDMA_PF_BAR_RSVD; 512 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 513 /* skip over db page */ 514 bar_off += IRDMA_HW_PAGE_SIZE; 515 /* push wqe page */ 516 bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; 517 iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 518 bar_off, IRDMA_MMAP_IO_WC, 519 push_wqe_mmap_key); 520 if (!iwqp->push_wqe_mmap_entry) 521 return -ENOMEM; 522 523 /* push doorbell page */ 524 bar_off += IRDMA_HW_PAGE_SIZE; 525 iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 526 bar_off, IRDMA_MMAP_IO_NC, 527 push_db_mmap_key); 528 if (!iwqp->push_db_mmap_entry) { 529 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 530 return -ENOMEM; 531 } 532 533 return 0; 534 } 535 536 /** 537 * irdma_destroy_qp - destroy qp 538 * @ibqp: qp's ib pointer also to get to device's qp address 539 * @udata: user data 540 */ 541 static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 542 { 543 struct irdma_qp *iwqp = to_iwqp(ibqp); 544 struct irdma_device *iwdev = iwqp->iwdev; 545 546 iwqp->sc_qp.qp_uk.destroy_pending = true; 547 548 if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE) 549 irdma_modify_qp_to_err(&iwqp->sc_qp); 550 551 if (!iwqp->user_mode) 552 cancel_delayed_work_sync(&iwqp->dwork_flush); 553 554 if (!iwqp->user_mode) { 555 if (iwqp->iwscq) { 556 irdma_clean_cqes(iwqp, iwqp->iwscq); 557 if (iwqp->iwrcq != iwqp->iwscq) 558 irdma_clean_cqes(iwqp, iwqp->iwrcq); 559 } 560 } 561 562 irdma_qp_rem_ref(&iwqp->ibqp); 563 if (!iwdev->rf->reset) 564 wait_for_completion(&iwqp->free_qp); 565 irdma_free_lsmm_rsrc(iwqp); 566 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); 567 568 irdma_remove_push_mmap_entries(iwqp); 569 570 if (iwqp->sc_qp.qp_uk.qp_id == 1) 571 iwdev->rf->hwqp1_rsvd = false; 572 irdma_free_qp_rsrc(iwqp); 573 574 return 0; 575 } 576 577 /** 578 * irdma_setup_virt_qp - setup for allocation of virtual qp 579 * @iwdev: irdma device 580 * @iwqp: qp ptr 581 * @init_info: initialize info to return 582 */ 583 static void irdma_setup_virt_qp(struct irdma_device *iwdev, 584 struct irdma_qp *iwqp, 585 struct irdma_qp_init_info *init_info) 586 { 587 struct irdma_pbl *iwpbl = iwqp->iwpbl; 588 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 589 590 iwqp->page = qpmr->sq_page; 591 init_info->shadow_area_pa = qpmr->shadow; 592 if (iwpbl->pbl_allocated) { 593 init_info->virtual_map = true; 594 init_info->sq_pa = qpmr->sq_pbl.idx; 595 /* Need to use contiguous buffer for RQ of QP 596 * in case it is associated with SRQ. 597 */ 598 init_info->rq_pa = init_info->qp_uk_init_info.srq_uk ? 599 qpmr->rq_pa : qpmr->rq_pbl.idx; 600 } else { 601 init_info->sq_pa = qpmr->sq_pbl.addr; 602 init_info->rq_pa = qpmr->rq_pbl.addr; 603 } 604 } 605 606 /** 607 * irdma_setup_umode_qp - setup sq and rq size in user mode qp 608 * @udata: udata 609 * @iwdev: iwarp device 610 * @iwqp: qp ptr (user or kernel) 611 * @info: initialize info to return 612 * @init_attr: Initial QP create attributes 613 */ 614 static int irdma_setup_umode_qp(struct ib_udata *udata, 615 struct irdma_device *iwdev, 616 struct irdma_qp *iwqp, 617 struct irdma_qp_init_info *info, 618 struct ib_qp_init_attr *init_attr) 619 { 620 struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, 621 struct irdma_ucontext, ibucontext); 622 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 623 struct irdma_create_qp_req req; 624 unsigned long flags; 625 int ret; 626 627 ret = ib_copy_from_udata(&req, udata, 628 min(sizeof(req), udata->inlen)); 629 if (ret) { 630 ibdev_dbg(&iwdev->ibdev, "VERBS: ib_copy_from_data fail\n"); 631 return ret; 632 } 633 634 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; 635 iwqp->user_mode = 1; 636 if (req.user_wqe_bufs) { 637 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 638 iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, 639 &ucontext->qp_reg_mem_list); 640 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 641 642 if (!iwqp->iwpbl) { 643 ret = -ENODATA; 644 ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); 645 return ret; 646 } 647 } 648 649 if (!ucontext->use_raw_attrs) { 650 /** 651 * Maintain backward compat with older ABI which passes sq and 652 * rq depth in quanta in cap.max_send_wr and cap.max_recv_wr. 653 * There is no way to compute the correct value of 654 * iwqp->max_send_wr/max_recv_wr in the kernel. 655 */ 656 iwqp->max_send_wr = init_attr->cap.max_send_wr; 657 iwqp->max_recv_wr = init_attr->cap.max_recv_wr; 658 ukinfo->sq_size = init_attr->cap.max_send_wr; 659 ukinfo->rq_size = init_attr->cap.max_recv_wr; 660 irdma_uk_calc_shift_wq(ukinfo, &ukinfo->sq_shift, 661 &ukinfo->rq_shift); 662 } else { 663 ret = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth, 664 &ukinfo->sq_shift); 665 if (ret) 666 return ret; 667 668 ret = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth, 669 &ukinfo->rq_shift); 670 if (ret) 671 return ret; 672 673 iwqp->max_send_wr = 674 (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift; 675 iwqp->max_recv_wr = 676 (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift; 677 ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; 678 ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; 679 } 680 681 irdma_setup_virt_qp(iwdev, iwqp, info); 682 683 return 0; 684 } 685 686 /** 687 * irdma_setup_kmode_qp - setup initialization for kernel mode qp 688 * @iwdev: iwarp device 689 * @iwqp: qp ptr (user or kernel) 690 * @info: initialize info to return 691 * @init_attr: Initial QP create attributes 692 */ 693 static int irdma_setup_kmode_qp(struct irdma_device *iwdev, 694 struct irdma_qp *iwqp, 695 struct irdma_qp_init_info *info, 696 struct ib_qp_init_attr *init_attr) 697 { 698 struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; 699 u32 size; 700 int status; 701 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 702 703 status = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth, 704 &ukinfo->sq_shift); 705 if (status) 706 return status; 707 708 status = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth, 709 &ukinfo->rq_shift); 710 if (status) 711 return status; 712 713 iwqp->kqp.sq_wrid_mem = 714 kzalloc_objs(*iwqp->kqp.sq_wrid_mem, ukinfo->sq_depth); 715 if (!iwqp->kqp.sq_wrid_mem) 716 return -ENOMEM; 717 718 iwqp->kqp.rq_wrid_mem = 719 kzalloc_objs(*iwqp->kqp.rq_wrid_mem, ukinfo->rq_depth); 720 721 if (!iwqp->kqp.rq_wrid_mem) { 722 kfree(iwqp->kqp.sq_wrid_mem); 723 iwqp->kqp.sq_wrid_mem = NULL; 724 return -ENOMEM; 725 } 726 727 ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; 728 ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; 729 730 size = (ukinfo->sq_depth + ukinfo->rq_depth) * IRDMA_QP_WQE_MIN_SIZE; 731 size += (IRDMA_SHADOW_AREA_SIZE << 3); 732 733 mem->size = ALIGN(size, 256); 734 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 735 &mem->pa, GFP_KERNEL); 736 if (!mem->va) { 737 kfree(iwqp->kqp.sq_wrid_mem); 738 iwqp->kqp.sq_wrid_mem = NULL; 739 kfree(iwqp->kqp.rq_wrid_mem); 740 iwqp->kqp.rq_wrid_mem = NULL; 741 return -ENOMEM; 742 } 743 744 ukinfo->sq = mem->va; 745 info->sq_pa = mem->pa; 746 ukinfo->rq = &ukinfo->sq[ukinfo->sq_depth]; 747 info->rq_pa = info->sq_pa + (ukinfo->sq_depth * IRDMA_QP_WQE_MIN_SIZE); 748 ukinfo->shadow_area = ukinfo->rq[ukinfo->rq_depth].elem; 749 info->shadow_area_pa = 750 info->rq_pa + (ukinfo->rq_depth * IRDMA_QP_WQE_MIN_SIZE); 751 ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; 752 ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; 753 ukinfo->qp_id = info->qp_uk_init_info.qp_id; 754 755 iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift; 756 iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift; 757 init_attr->cap.max_send_wr = iwqp->max_send_wr; 758 init_attr->cap.max_recv_wr = iwqp->max_recv_wr; 759 760 return 0; 761 } 762 763 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) 764 { 765 struct irdma_pci_f *rf = iwqp->iwdev->rf; 766 struct irdma_cqp_request *cqp_request; 767 struct cqp_cmds_info *cqp_info; 768 struct irdma_create_qp_info *qp_info; 769 int status; 770 771 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 772 if (!cqp_request) 773 return -ENOMEM; 774 775 cqp_info = &cqp_request->info; 776 qp_info = &cqp_request->info.in.u.qp_create.info; 777 qp_info->mac_valid = true; 778 qp_info->cq_num_valid = true; 779 qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; 780 781 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 782 cqp_info->post_sq = 1; 783 cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; 784 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 785 status = irdma_handle_cqp_op(rf, cqp_request); 786 irdma_put_cqp_request(&rf->cqp, cqp_request); 787 788 return status; 789 } 790 791 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 792 struct irdma_qp_host_ctx_info *ctx_info) 793 { 794 struct irdma_device *iwdev = iwqp->iwdev; 795 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 796 struct irdma_roce_offload_info *roce_info; 797 struct irdma_udp_offload_info *udp_info; 798 799 udp_info = &iwqp->udp_info; 800 udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); 801 udp_info->cwnd = iwdev->roce_cwnd; 802 udp_info->rexmit_thresh = 2; 803 udp_info->rnr_nak_thresh = 2; 804 udp_info->src_port = 0xc000; 805 udp_info->dst_port = ROCE_V2_UDP_DPORT; 806 roce_info = &iwqp->roce_info; 807 ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr); 808 809 if (iwqp->ibqp.qp_type == IB_QPT_GSI && iwqp->ibqp.qp_num != 1) 810 roce_info->is_qp1 = true; 811 roce_info->rd_en = true; 812 roce_info->wr_rdresp_en = true; 813 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) 814 roce_info->bind_en = true; 815 roce_info->dcqcn_en = false; 816 roce_info->rtomin = 5; 817 818 roce_info->ack_credits = iwdev->roce_ackcreds; 819 roce_info->ird_size = dev->hw_attrs.max_hw_ird; 820 roce_info->ord_size = dev->hw_attrs.max_hw_ord; 821 822 if (!iwqp->user_mode) { 823 roce_info->priv_mode_en = true; 824 roce_info->fast_reg_en = true; 825 roce_info->udprivcq_en = true; 826 } 827 roce_info->roce_tver = 0; 828 829 ctx_info->roce_info = &iwqp->roce_info; 830 ctx_info->udp_info = &iwqp->udp_info; 831 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 832 } 833 834 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 835 struct irdma_qp_host_ctx_info *ctx_info) 836 { 837 struct irdma_device *iwdev = iwqp->iwdev; 838 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 839 struct irdma_iwarp_offload_info *iwarp_info; 840 841 iwarp_info = &iwqp->iwarp_info; 842 ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr); 843 iwarp_info->rd_en = true; 844 iwarp_info->wr_rdresp_en = true; 845 iwarp_info->ecn_en = true; 846 iwarp_info->rtomin = 5; 847 848 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 849 iwarp_info->ib_rd_en = true; 850 if (!iwqp->user_mode) { 851 iwarp_info->priv_mode_en = true; 852 iwarp_info->fast_reg_en = true; 853 } 854 iwarp_info->ddp_ver = 1; 855 iwarp_info->rdmap_ver = 1; 856 857 ctx_info->iwarp_info = &iwqp->iwarp_info; 858 ctx_info->iwarp_info_valid = true; 859 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 860 ctx_info->iwarp_info_valid = false; 861 } 862 863 static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, 864 struct irdma_device *iwdev) 865 { 866 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 867 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 868 869 if (init_attr->create_flags) 870 return -EOPNOTSUPP; 871 872 if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || 873 init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || 874 init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags || 875 init_attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta || 876 init_attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta) 877 return -EINVAL; 878 879 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 880 if (init_attr->qp_type != IB_QPT_RC && 881 init_attr->qp_type != IB_QPT_UD && 882 init_attr->qp_type != IB_QPT_GSI) 883 return -EOPNOTSUPP; 884 } else { 885 if (init_attr->qp_type != IB_QPT_RC) 886 return -EOPNOTSUPP; 887 } 888 889 return 0; 890 } 891 892 static void irdma_flush_worker(struct work_struct *work) 893 { 894 struct delayed_work *dwork = to_delayed_work(work); 895 struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush); 896 897 irdma_generate_flush_completions(iwqp); 898 } 899 900 static int irdma_setup_gsi_qp_rsrc(struct irdma_qp *iwqp, u32 *qp_num) 901 { 902 struct irdma_device *iwdev = iwqp->iwdev; 903 struct irdma_pci_f *rf = iwdev->rf; 904 unsigned long flags; 905 int ret; 906 907 if (rf->rdma_ver <= IRDMA_GEN_2) { 908 *qp_num = 1; 909 return 0; 910 } 911 912 spin_lock_irqsave(&rf->rsrc_lock, flags); 913 if (!rf->hwqp1_rsvd) { 914 *qp_num = 1; 915 rf->hwqp1_rsvd = true; 916 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 917 } else { 918 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 919 ret = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 920 qp_num, &rf->next_qp); 921 if (ret) 922 return ret; 923 } 924 925 ret = irdma_vchnl_req_add_vport(&rf->sc_dev, iwdev->vport_id, *qp_num, 926 (&iwdev->vsi)->qos); 927 if (ret) { 928 if (*qp_num != 1) { 929 irdma_free_rsrc(rf, rf->allocated_qps, *qp_num); 930 } else { 931 spin_lock_irqsave(&rf->rsrc_lock, flags); 932 rf->hwqp1_rsvd = false; 933 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 934 } 935 return ret; 936 } 937 938 return 0; 939 } 940 941 /** 942 * irdma_create_qp - create qp 943 * @ibqp: ptr of qp 944 * @init_attr: attributes for qp 945 * @udata: user data for create qp 946 */ 947 static int irdma_create_qp(struct ib_qp *ibqp, 948 struct ib_qp_init_attr *init_attr, 949 struct ib_udata *udata) 950 { 951 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx) 952 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd) 953 struct ib_pd *ibpd = ibqp->pd; 954 struct irdma_pd *iwpd = to_iwpd(ibpd); 955 struct irdma_device *iwdev = to_iwdev(ibpd->device); 956 struct irdma_pci_f *rf = iwdev->rf; 957 struct irdma_qp *iwqp = to_iwqp(ibqp); 958 struct irdma_create_qp_resp uresp = {}; 959 u32 qp_num = 0; 960 int err_code; 961 struct irdma_sc_qp *qp; 962 struct irdma_sc_dev *dev = &rf->sc_dev; 963 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 964 struct irdma_qp_init_info init_info = {}; 965 struct irdma_qp_host_ctx_info *ctx_info; 966 struct irdma_srq *iwsrq; 967 bool srq_valid = false; 968 u32 srq_id = 0; 969 970 if (init_attr->srq) { 971 iwsrq = to_iwsrq(init_attr->srq); 972 srq_valid = true; 973 srq_id = iwsrq->srq_num; 974 init_attr->cap.max_recv_sge = uk_attrs->max_hw_wq_frags; 975 init_attr->cap.max_recv_wr = 4; 976 init_info.qp_uk_init_info.srq_uk = &iwsrq->sc_srq.srq_uk; 977 } 978 979 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 980 if (err_code) 981 return err_code; 982 983 if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN || 984 udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN)) 985 return -EINVAL; 986 987 init_info.vsi = &iwdev->vsi; 988 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 989 init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr; 990 init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr; 991 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 992 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 993 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 994 995 qp = &iwqp->sc_qp; 996 qp->qp_uk.back_qp = iwqp; 997 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 998 999 iwqp->iwdev = iwdev; 1000 iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, 1001 256); 1002 iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, 1003 iwqp->q2_ctx_mem.size, 1004 &iwqp->q2_ctx_mem.pa, 1005 GFP_KERNEL); 1006 if (!iwqp->q2_ctx_mem.va) 1007 return -ENOMEM; 1008 1009 init_info.q2 = iwqp->q2_ctx_mem.va; 1010 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 1011 init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE); 1012 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 1013 1014 if (init_attr->qp_type == IB_QPT_GSI) { 1015 err_code = irdma_setup_gsi_qp_rsrc(iwqp, &qp_num); 1016 if (err_code) 1017 goto error; 1018 iwqp->ibqp.qp_num = 1; 1019 } else { 1020 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 1021 &qp_num, &rf->next_qp); 1022 if (err_code) 1023 goto error; 1024 iwqp->ibqp.qp_num = qp_num; 1025 } 1026 1027 iwqp->iwpd = iwpd; 1028 qp = &iwqp->sc_qp; 1029 iwqp->iwscq = to_iwcq(init_attr->send_cq); 1030 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 1031 iwqp->host_ctx.va = init_info.host_ctx; 1032 iwqp->host_ctx.pa = init_info.host_ctx_pa; 1033 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 1034 1035 init_info.pd = &iwpd->sc_pd; 1036 init_info.qp_uk_init_info.qp_id = qp_num; 1037 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 1038 init_info.qp_uk_init_info.first_sq_wq = 1; 1039 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 1040 init_waitqueue_head(&iwqp->waitq); 1041 init_waitqueue_head(&iwqp->mod_qp_waitq); 1042 1043 if (udata) { 1044 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 1045 err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, 1046 init_attr); 1047 } else { 1048 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); 1049 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 1050 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 1051 } 1052 1053 if (err_code) { 1054 ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n"); 1055 goto error; 1056 } 1057 1058 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 1059 if (init_attr->qp_type == IB_QPT_RC) { 1060 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 1061 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 1062 IRDMA_WRITE_WITH_IMM | 1063 IRDMA_ROCE; 1064 } else { 1065 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 1066 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 1067 IRDMA_ROCE; 1068 } 1069 } else { 1070 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 1071 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 1072 } 1073 1074 if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) 1075 init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE; 1076 1077 err_code = irdma_sc_qp_init(qp, &init_info); 1078 if (err_code) { 1079 ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n"); 1080 goto error; 1081 } 1082 1083 ctx_info = &iwqp->ctx_info; 1084 ctx_info->srq_valid = srq_valid; 1085 ctx_info->srq_id = srq_id; 1086 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1087 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1088 1089 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 1090 if (dev->ws_add(&iwdev->vsi, 0)) { 1091 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 1092 err_code = -EINVAL; 1093 goto error; 1094 } 1095 irdma_qp_add_qos(&iwqp->sc_qp); 1096 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 1097 } else { 1098 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 1099 } 1100 1101 err_code = irdma_cqp_create_qp_cmd(iwqp); 1102 if (err_code) 1103 goto error; 1104 1105 refcount_set(&iwqp->refcnt, 1); 1106 spin_lock_init(&iwqp->lock); 1107 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 1108 iwqp->sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1109 rf->qp_table[qp_num] = iwqp; 1110 init_completion(&iwqp->free_qp); 1111 1112 if (udata) { 1113 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 1114 if (udata->outlen < sizeof(uresp)) { 1115 uresp.lsmm = 1; 1116 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 1117 } else { 1118 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 1119 uresp.lsmm = 1; 1120 } 1121 uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size; 1122 uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size; 1123 uresp.qp_id = qp_num; 1124 uresp.qp_caps = qp->qp_uk.qp_caps; 1125 1126 err_code = ib_respond_udata(udata, uresp); 1127 if (err_code) { 1128 irdma_destroy_qp(&iwqp->ibqp, udata); 1129 return err_code; 1130 } 1131 } 1132 1133 return 0; 1134 1135 error: 1136 irdma_free_qp_rsrc(iwqp); 1137 return err_code; 1138 } 1139 1140 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) 1141 { 1142 int acc_flags = 0; 1143 1144 if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { 1145 if (iwqp->roce_info.wr_rdresp_en) { 1146 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1147 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1148 } 1149 if (iwqp->roce_info.rd_en) 1150 acc_flags |= IB_ACCESS_REMOTE_READ; 1151 if (iwqp->roce_info.bind_en) 1152 acc_flags |= IB_ACCESS_MW_BIND; 1153 if (iwqp->ctx_info.remote_atomics_en) 1154 acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 1155 } else { 1156 if (iwqp->iwarp_info.wr_rdresp_en) { 1157 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1158 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1159 } 1160 if (iwqp->iwarp_info.rd_en) 1161 acc_flags |= IB_ACCESS_REMOTE_READ; 1162 if (iwqp->ctx_info.remote_atomics_en) 1163 acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 1164 } 1165 return acc_flags; 1166 } 1167 1168 /** 1169 * irdma_query_qp - query qp attributes 1170 * @ibqp: qp pointer 1171 * @attr: attributes pointer 1172 * @attr_mask: Not used 1173 * @init_attr: qp attributes to return 1174 */ 1175 static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1176 int attr_mask, struct ib_qp_init_attr *init_attr) 1177 { 1178 struct irdma_qp *iwqp = to_iwqp(ibqp); 1179 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1180 1181 memset(attr, 0, sizeof(*attr)); 1182 memset(init_attr, 0, sizeof(*init_attr)); 1183 1184 attr->qp_state = iwqp->ibqp_state; 1185 attr->cur_qp_state = iwqp->ibqp_state; 1186 attr->cap.max_send_wr = iwqp->max_send_wr; 1187 attr->cap.max_recv_wr = iwqp->max_recv_wr; 1188 attr->cap.max_inline_data = qp->qp_uk.max_inline_data; 1189 attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; 1190 attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; 1191 attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); 1192 attr->port_num = 1; 1193 if (rdma_protocol_roce(ibqp->device, 1)) { 1194 attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); 1195 attr->qkey = iwqp->roce_info.qkey; 1196 attr->rq_psn = iwqp->udp_info.epsn; 1197 attr->sq_psn = iwqp->udp_info.psn_nxt; 1198 attr->dest_qp_num = iwqp->roce_info.dest_qp; 1199 attr->pkey_index = iwqp->roce_info.p_key; 1200 attr->retry_cnt = iwqp->udp_info.rexmit_thresh; 1201 attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; 1202 attr->min_rnr_timer = iwqp->udp_info.min_rnr_timer; 1203 attr->max_rd_atomic = iwqp->roce_info.ord_size; 1204 attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; 1205 } 1206 1207 init_attr->event_handler = iwqp->ibqp.event_handler; 1208 init_attr->qp_context = iwqp->ibqp.qp_context; 1209 init_attr->send_cq = iwqp->ibqp.send_cq; 1210 init_attr->recv_cq = iwqp->ibqp.recv_cq; 1211 init_attr->srq = iwqp->ibqp.srq; 1212 init_attr->cap = attr->cap; 1213 1214 return 0; 1215 } 1216 1217 /** 1218 * irdma_query_pkey - Query partition key 1219 * @ibdev: device pointer from stack 1220 * @port: port number 1221 * @index: index of pkey 1222 * @pkey: pointer to store the pkey 1223 */ 1224 static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 1225 u16 *pkey) 1226 { 1227 if (index >= IRDMA_PKEY_TBL_SZ) 1228 return -EINVAL; 1229 1230 *pkey = IRDMA_DEFAULT_PKEY; 1231 return 0; 1232 } 1233 1234 static u8 irdma_roce_get_vlan_prio(const struct ib_gid_attr *attr, u8 prio) 1235 { 1236 struct net_device *ndev; 1237 1238 rcu_read_lock(); 1239 ndev = rcu_dereference(attr->ndev); 1240 if (!ndev) 1241 goto exit; 1242 if (is_vlan_dev(ndev)) { 1243 u16 vlan_qos = vlan_dev_get_egress_qos_mask(ndev, prio); 1244 1245 prio = (vlan_qos & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1246 } 1247 exit: 1248 rcu_read_unlock(); 1249 return prio; 1250 } 1251 1252 static int irdma_wait_for_suspend(struct irdma_qp *iwqp) 1253 { 1254 if (!wait_event_timeout(iwqp->iwdev->suspend_wq, 1255 !iwqp->suspend_pending, 1256 msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) { 1257 iwqp->suspend_pending = false; 1258 ibdev_warn(&iwqp->iwdev->ibdev, 1259 "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n", 1260 iwqp->ibqp.qp_num, iwqp->last_aeq); 1261 return -EBUSY; 1262 } 1263 1264 return 0; 1265 } 1266 1267 /** 1268 * irdma_modify_qp_roce - modify qp request 1269 * @ibqp: qp's pointer for modify 1270 * @attr: access attributes 1271 * @attr_mask: state mask 1272 * @udata: user data 1273 */ 1274 int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1275 int attr_mask, struct ib_udata *udata) 1276 { 1277 #define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) 1278 #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) 1279 struct irdma_pd *iwpd = to_iwpd(ibqp->pd); 1280 struct irdma_qp *iwqp = to_iwqp(ibqp); 1281 struct irdma_device *iwdev = iwqp->iwdev; 1282 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1283 struct irdma_qp_host_ctx_info *ctx_info; 1284 struct irdma_roce_offload_info *roce_info; 1285 struct irdma_udp_offload_info *udp_info; 1286 struct irdma_modify_qp_info info = {}; 1287 struct irdma_modify_qp_resp uresp = {}; 1288 struct irdma_modify_qp_req ureq = {}; 1289 unsigned long flags; 1290 u8 issue_modify_qp = 0; 1291 int ret = 0; 1292 1293 ctx_info = &iwqp->ctx_info; 1294 roce_info = &iwqp->roce_info; 1295 udp_info = &iwqp->udp_info; 1296 1297 if (udata) { 1298 /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ 1299 if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || 1300 (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) 1301 return -EINVAL; 1302 } 1303 1304 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1305 return -EOPNOTSUPP; 1306 1307 if (attr_mask & IB_QP_DEST_QPN) 1308 roce_info->dest_qp = attr->dest_qp_num; 1309 1310 if (attr_mask & IB_QP_PKEY_INDEX) { 1311 ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, 1312 &roce_info->p_key); 1313 if (ret) 1314 return ret; 1315 } 1316 1317 if (attr_mask & IB_QP_QKEY) 1318 roce_info->qkey = attr->qkey; 1319 1320 if (attr_mask & IB_QP_PATH_MTU) 1321 udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); 1322 1323 if (attr_mask & IB_QP_SQ_PSN) { 1324 udp_info->psn_nxt = attr->sq_psn; 1325 udp_info->lsn = 0xffff; 1326 udp_info->psn_una = attr->sq_psn; 1327 udp_info->psn_max = attr->sq_psn; 1328 } 1329 1330 if (attr_mask & IB_QP_RQ_PSN) 1331 udp_info->epsn = attr->rq_psn; 1332 1333 if (attr_mask & IB_QP_RNR_RETRY) 1334 udp_info->rnr_nak_thresh = attr->rnr_retry; 1335 1336 if (attr_mask & IB_QP_MIN_RNR_TIMER && 1337 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) 1338 udp_info->min_rnr_timer = attr->min_rnr_timer; 1339 1340 if (attr_mask & IB_QP_RETRY_CNT) 1341 udp_info->rexmit_thresh = attr->retry_cnt; 1342 1343 ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; 1344 1345 if (attr_mask & IB_QP_AV) { 1346 struct irdma_av *av = &iwqp->roce_ah.av; 1347 const struct ib_gid_attr *sgid_attr = 1348 attr->ah_attr.grh.sgid_attr; 1349 u16 vlan_id = VLAN_N_VID; 1350 u32 local_ip[4]; 1351 1352 memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); 1353 if (attr->ah_attr.ah_flags & IB_AH_GRH) { 1354 udp_info->ttl = attr->ah_attr.grh.hop_limit; 1355 udp_info->flow_label = attr->ah_attr.grh.flow_label; 1356 udp_info->tos = attr->ah_attr.grh.traffic_class; 1357 udp_info->src_port = 1358 rdma_get_udp_sport(udp_info->flow_label, 1359 ibqp->qp_num, 1360 roce_info->dest_qp); 1361 irdma_qp_rem_qos(&iwqp->sc_qp); 1362 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); 1363 if (iwqp->sc_qp.vsi->dscp_mode) 1364 ctx_info->user_pri = 1365 iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(udp_info->tos)]; 1366 else 1367 ctx_info->user_pri = rt_tos2priority(udp_info->tos); 1368 } 1369 ret = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, 1370 ctx_info->roce_info->mac_addr); 1371 if (ret) 1372 return ret; 1373 ctx_info->user_pri = irdma_roce_get_vlan_prio(sgid_attr, 1374 ctx_info->user_pri); 1375 if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) 1376 return -ENOMEM; 1377 iwqp->sc_qp.user_pri = ctx_info->user_pri; 1378 irdma_qp_add_qos(&iwqp->sc_qp); 1379 1380 if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 1381 vlan_id = 0; 1382 if (vlan_id < VLAN_N_VID) { 1383 udp_info->insert_vlan_tag = true; 1384 udp_info->vlan_tag = vlan_id | 1385 ctx_info->user_pri << VLAN_PRIO_SHIFT; 1386 } else { 1387 udp_info->insert_vlan_tag = false; 1388 } 1389 1390 av->attrs = attr->ah_attr; 1391 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid); 1392 rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); 1393 av->net_type = rdma_gid_attr_network_type(sgid_attr); 1394 if (av->net_type == RDMA_NETWORK_IPV6) { 1395 __be32 *daddr = 1396 av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1397 __be32 *saddr = 1398 av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1399 1400 irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); 1401 irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); 1402 1403 udp_info->ipv4 = false; 1404 irdma_copy_ip_ntohl(local_ip, daddr); 1405 1406 } else if (av->net_type == RDMA_NETWORK_IPV4) { 1407 __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; 1408 __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; 1409 1410 local_ip[0] = ntohl(daddr); 1411 1412 udp_info->ipv4 = true; 1413 udp_info->dest_ip_addr[0] = 0; 1414 udp_info->dest_ip_addr[1] = 0; 1415 udp_info->dest_ip_addr[2] = 0; 1416 udp_info->dest_ip_addr[3] = local_ip[0]; 1417 1418 udp_info->local_ipaddr[0] = 0; 1419 udp_info->local_ipaddr[1] = 0; 1420 udp_info->local_ipaddr[2] = 0; 1421 udp_info->local_ipaddr[3] = ntohl(saddr); 1422 } 1423 udp_info->arp_idx = 1424 irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4, 1425 attr->ah_attr.roce.dmac); 1426 } 1427 1428 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1429 if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { 1430 ibdev_err(&iwdev->ibdev, 1431 "rd_atomic = %d, above max_hw_ord=%d\n", 1432 attr->max_rd_atomic, 1433 dev->hw_attrs.max_hw_ord); 1434 return -EINVAL; 1435 } 1436 if (attr->max_rd_atomic) 1437 roce_info->ord_size = attr->max_rd_atomic; 1438 info.ord_valid = true; 1439 } 1440 1441 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1442 if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { 1443 ibdev_err(&iwdev->ibdev, 1444 "rd_atomic = %d, above max_hw_ird=%d\n", 1445 attr->max_dest_rd_atomic, 1446 dev->hw_attrs.max_hw_ird); 1447 return -EINVAL; 1448 } 1449 if (attr->max_dest_rd_atomic) 1450 roce_info->ird_size = attr->max_dest_rd_atomic; 1451 } 1452 1453 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1454 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1455 roce_info->wr_rdresp_en = true; 1456 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1457 roce_info->wr_rdresp_en = true; 1458 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1459 roce_info->rd_en = true; 1460 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 1461 if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) 1462 ctx_info->remote_atomics_en = true; 1463 } 1464 1465 ibdev_dbg(&iwdev->ibdev, 1466 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", 1467 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1468 iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); 1469 1470 spin_lock_irqsave(&iwqp->lock, flags); 1471 if (attr_mask & IB_QP_STATE) { 1472 if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, 1473 iwqp->ibqp.qp_type, attr_mask)) { 1474 ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", 1475 iwqp->ibqp.qp_num, iwqp->ibqp_state, 1476 attr->qp_state); 1477 ret = -EINVAL; 1478 goto exit; 1479 } 1480 info.curr_iwarp_state = iwqp->iwarp_state; 1481 1482 switch (attr->qp_state) { 1483 case IB_QPS_INIT: 1484 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1485 ret = -EINVAL; 1486 goto exit; 1487 } 1488 1489 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1490 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1491 issue_modify_qp = 1; 1492 } 1493 break; 1494 case IB_QPS_RTR: 1495 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1496 ret = -EINVAL; 1497 goto exit; 1498 } 1499 info.arp_cache_idx_valid = true; 1500 info.cq_num_valid = true; 1501 info.next_iwarp_state = IRDMA_QP_STATE_RTR; 1502 issue_modify_qp = 1; 1503 break; 1504 case IB_QPS_RTS: 1505 if (iwqp->ibqp_state < IB_QPS_RTR || 1506 iwqp->ibqp_state == IB_QPS_ERR) { 1507 ret = -EINVAL; 1508 goto exit; 1509 } 1510 1511 info.arp_cache_idx_valid = true; 1512 info.cq_num_valid = true; 1513 info.ord_valid = true; 1514 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1515 issue_modify_qp = 1; 1516 if (iwdev->push_mode && udata && 1517 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1518 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1519 spin_unlock_irqrestore(&iwqp->lock, flags); 1520 irdma_alloc_push_page(iwqp); 1521 spin_lock_irqsave(&iwqp->lock, flags); 1522 } 1523 break; 1524 case IB_QPS_SQD: 1525 if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) 1526 goto exit; 1527 1528 if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { 1529 ret = -EINVAL; 1530 goto exit; 1531 } 1532 1533 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1534 issue_modify_qp = 1; 1535 iwqp->suspend_pending = true; 1536 break; 1537 case IB_QPS_SQE: 1538 case IB_QPS_ERR: 1539 case IB_QPS_RESET: 1540 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1541 iwqp->ibqp_state = attr->qp_state; 1542 spin_unlock_irqrestore(&iwqp->lock, flags); 1543 if (udata && udata->inlen) { 1544 if (ib_copy_from_udata(&ureq, udata, 1545 min(sizeof(ureq), udata->inlen))) 1546 return -EINVAL; 1547 1548 irdma_flush_wqes(iwqp, 1549 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1550 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1551 IRDMA_REFLUSH); 1552 } 1553 return 0; 1554 } 1555 1556 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1557 issue_modify_qp = 1; 1558 break; 1559 default: 1560 ret = -EINVAL; 1561 goto exit; 1562 } 1563 1564 iwqp->ibqp_state = attr->qp_state; 1565 } 1566 1567 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1568 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1569 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1570 spin_unlock_irqrestore(&iwqp->lock, flags); 1571 1572 if (attr_mask & IB_QP_STATE) { 1573 if (issue_modify_qp) { 1574 ctx_info->rem_endpoint_idx = udp_info->arp_idx; 1575 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1576 return -EINVAL; 1577 if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) { 1578 ret = irdma_wait_for_suspend(iwqp); 1579 if (ret) 1580 return ret; 1581 } 1582 spin_lock_irqsave(&iwqp->lock, flags); 1583 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1584 iwqp->iwarp_state = info.next_iwarp_state; 1585 iwqp->ibqp_state = attr->qp_state; 1586 } 1587 if (iwqp->ibqp_state > IB_QPS_RTS && 1588 !iwqp->flush_issued) { 1589 spin_unlock_irqrestore(&iwqp->lock, flags); 1590 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | 1591 IRDMA_FLUSH_RQ | 1592 IRDMA_FLUSH_WAIT); 1593 iwqp->flush_issued = 1; 1594 } else { 1595 spin_unlock_irqrestore(&iwqp->lock, flags); 1596 } 1597 } else { 1598 iwqp->ibqp_state = attr->qp_state; 1599 } 1600 if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1601 struct irdma_ucontext *ucontext; 1602 1603 ucontext = rdma_udata_to_drv_context(udata, 1604 struct irdma_ucontext, ibucontext); 1605 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1606 !iwqp->push_wqe_mmap_entry && 1607 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1608 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1609 uresp.push_valid = 1; 1610 uresp.push_offset = iwqp->sc_qp.push_offset; 1611 } 1612 ret = ib_respond_udata(udata, uresp); 1613 if (ret) { 1614 irdma_remove_push_mmap_entries(iwqp); 1615 return ret; 1616 } 1617 } 1618 } 1619 1620 return 0; 1621 exit: 1622 spin_unlock_irqrestore(&iwqp->lock, flags); 1623 1624 return ret; 1625 } 1626 1627 /** 1628 * irdma_modify_qp - modify qp request 1629 * @ibqp: qp's pointer for modify 1630 * @attr: access attributes 1631 * @attr_mask: state mask 1632 * @udata: user data 1633 */ 1634 int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1635 struct ib_udata *udata) 1636 { 1637 #define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) 1638 #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) 1639 struct irdma_qp *iwqp = to_iwqp(ibqp); 1640 struct irdma_device *iwdev = iwqp->iwdev; 1641 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1642 struct irdma_qp_host_ctx_info *ctx_info; 1643 struct irdma_tcp_offload_info *tcp_info; 1644 struct irdma_iwarp_offload_info *offload_info; 1645 struct irdma_modify_qp_info info = {}; 1646 struct irdma_modify_qp_resp uresp = {}; 1647 struct irdma_modify_qp_req ureq = {}; 1648 u8 issue_modify_qp = 0; 1649 u8 dont_wait = 0; 1650 int err; 1651 unsigned long flags; 1652 1653 if (udata) { 1654 /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ 1655 if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || 1656 (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) 1657 return -EINVAL; 1658 } 1659 1660 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1661 return -EOPNOTSUPP; 1662 1663 ctx_info = &iwqp->ctx_info; 1664 offload_info = &iwqp->iwarp_info; 1665 tcp_info = &iwqp->tcp_info; 1666 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1667 ibdev_dbg(&iwdev->ibdev, 1668 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", 1669 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1670 iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, 1671 iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); 1672 1673 spin_lock_irqsave(&iwqp->lock, flags); 1674 if (attr_mask & IB_QP_STATE) { 1675 info.curr_iwarp_state = iwqp->iwarp_state; 1676 switch (attr->qp_state) { 1677 case IB_QPS_INIT: 1678 case IB_QPS_RTR: 1679 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1680 err = -EINVAL; 1681 goto exit; 1682 } 1683 1684 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1685 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1686 issue_modify_qp = 1; 1687 } 1688 if (iwdev->push_mode && udata && 1689 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1690 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1691 spin_unlock_irqrestore(&iwqp->lock, flags); 1692 irdma_alloc_push_page(iwqp); 1693 spin_lock_irqsave(&iwqp->lock, flags); 1694 } 1695 break; 1696 case IB_QPS_RTS: 1697 if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || 1698 !iwqp->cm_id) { 1699 err = -EINVAL; 1700 goto exit; 1701 } 1702 1703 issue_modify_qp = 1; 1704 iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 1705 iwqp->hte_added = 1; 1706 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1707 info.tcp_ctx_valid = true; 1708 info.ord_valid = true; 1709 info.arp_cache_idx_valid = true; 1710 info.cq_num_valid = true; 1711 break; 1712 case IB_QPS_SQD: 1713 if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { 1714 err = 0; 1715 goto exit; 1716 } 1717 1718 if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || 1719 iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { 1720 err = 0; 1721 goto exit; 1722 } 1723 1724 if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { 1725 err = -EINVAL; 1726 goto exit; 1727 } 1728 1729 info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; 1730 issue_modify_qp = 1; 1731 break; 1732 case IB_QPS_SQE: 1733 if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { 1734 err = -EINVAL; 1735 goto exit; 1736 } 1737 1738 info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; 1739 issue_modify_qp = 1; 1740 break; 1741 case IB_QPS_ERR: 1742 case IB_QPS_RESET: 1743 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1744 iwqp->ibqp_state = attr->qp_state; 1745 spin_unlock_irqrestore(&iwqp->lock, flags); 1746 if (udata && udata->inlen) { 1747 if (ib_copy_from_udata(&ureq, udata, 1748 min(sizeof(ureq), udata->inlen))) 1749 return -EINVAL; 1750 1751 irdma_flush_wqes(iwqp, 1752 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1753 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1754 IRDMA_REFLUSH); 1755 } 1756 return 0; 1757 } 1758 1759 if (iwqp->sc_qp.term_flags) { 1760 spin_unlock_irqrestore(&iwqp->lock, flags); 1761 irdma_terminate_del_timer(&iwqp->sc_qp); 1762 spin_lock_irqsave(&iwqp->lock, flags); 1763 } 1764 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1765 if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && 1766 iwdev->iw_status && 1767 iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) 1768 info.reset_tcp_conn = true; 1769 else 1770 dont_wait = 1; 1771 1772 issue_modify_qp = 1; 1773 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1774 break; 1775 default: 1776 err = -EINVAL; 1777 goto exit; 1778 } 1779 1780 iwqp->ibqp_state = attr->qp_state; 1781 } 1782 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1783 ctx_info->iwarp_info_valid = true; 1784 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1785 offload_info->wr_rdresp_en = true; 1786 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1787 offload_info->wr_rdresp_en = true; 1788 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1789 offload_info->rd_en = true; 1790 } 1791 1792 if (ctx_info->iwarp_info_valid) { 1793 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1794 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1795 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1796 } 1797 spin_unlock_irqrestore(&iwqp->lock, flags); 1798 1799 if (attr_mask & IB_QP_STATE) { 1800 if (issue_modify_qp) { 1801 ctx_info->rem_endpoint_idx = tcp_info->arp_idx; 1802 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1803 return -EINVAL; 1804 } 1805 1806 spin_lock_irqsave(&iwqp->lock, flags); 1807 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1808 iwqp->iwarp_state = info.next_iwarp_state; 1809 iwqp->ibqp_state = attr->qp_state; 1810 } 1811 spin_unlock_irqrestore(&iwqp->lock, flags); 1812 } 1813 1814 if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { 1815 if (dont_wait) { 1816 if (iwqp->hw_tcp_state) { 1817 spin_lock_irqsave(&iwqp->lock, flags); 1818 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1819 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1820 spin_unlock_irqrestore(&iwqp->lock, flags); 1821 } 1822 irdma_cm_disconn(iwqp); 1823 } else { 1824 int close_timer_started; 1825 1826 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 1827 1828 if (iwqp->cm_node) { 1829 refcount_inc(&iwqp->cm_node->refcnt); 1830 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1831 close_timer_started = atomic_inc_return(&iwqp->close_timer_started); 1832 if (iwqp->cm_id && close_timer_started == 1) 1833 irdma_schedule_cm_timer(iwqp->cm_node, 1834 (struct irdma_puda_buf *)iwqp, 1835 IRDMA_TIMER_TYPE_CLOSE, 1, 0); 1836 1837 irdma_rem_ref_cm_node(iwqp->cm_node); 1838 } else { 1839 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1840 } 1841 } 1842 } 1843 if (attr_mask & IB_QP_STATE && udata && udata->outlen && 1844 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1845 struct irdma_ucontext *ucontext; 1846 1847 ucontext = rdma_udata_to_drv_context(udata, 1848 struct irdma_ucontext, ibucontext); 1849 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1850 !iwqp->push_wqe_mmap_entry && 1851 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1852 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1853 uresp.push_valid = 1; 1854 uresp.push_offset = iwqp->sc_qp.push_offset; 1855 } 1856 1857 err = ib_respond_udata(udata, uresp); 1858 if (err) { 1859 irdma_remove_push_mmap_entries(iwqp); 1860 return err; 1861 } 1862 } 1863 1864 return 0; 1865 exit: 1866 spin_unlock_irqrestore(&iwqp->lock, flags); 1867 1868 return err; 1869 } 1870 1871 /** 1872 * irdma_srq_free_rsrc - free up resources for srq 1873 * @rf: RDMA PCI function 1874 * @iwsrq: srq ptr 1875 */ 1876 static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq) 1877 { 1878 struct irdma_sc_srq *srq = &iwsrq->sc_srq; 1879 1880 if (!iwsrq->user_mode) { 1881 dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size, 1882 iwsrq->kmem.va, iwsrq->kmem.pa); 1883 iwsrq->kmem.va = NULL; 1884 } 1885 1886 irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id); 1887 } 1888 1889 /** 1890 * irdma_cq_free_rsrc - free up resources for cq 1891 * @rf: RDMA PCI function 1892 * @iwcq: cq ptr 1893 */ 1894 static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) 1895 { 1896 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1897 1898 if (!iwcq->user_mode) { 1899 dma_free_coherent(rf->sc_dev.hw->device, iwcq->kmem.size, 1900 iwcq->kmem.va, iwcq->kmem.pa); 1901 iwcq->kmem.va = NULL; 1902 dma_free_coherent(rf->sc_dev.hw->device, 1903 iwcq->kmem_shadow.size, 1904 iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); 1905 iwcq->kmem_shadow.va = NULL; 1906 } 1907 1908 irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); 1909 } 1910 1911 /** 1912 * irdma_free_cqbuf - worker to free a cq buffer 1913 * @work: provides access to the cq buffer to free 1914 */ 1915 static void irdma_free_cqbuf(struct work_struct *work) 1916 { 1917 struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); 1918 1919 dma_free_coherent(cq_buf->hw->device, cq_buf->kmem_buf.size, 1920 cq_buf->kmem_buf.va, cq_buf->kmem_buf.pa); 1921 cq_buf->kmem_buf.va = NULL; 1922 kfree(cq_buf); 1923 } 1924 1925 /** 1926 * irdma_process_resize_list - remove resized cq buffers from the resize_list 1927 * @iwcq: cq which owns the resize_list 1928 * @iwdev: irdma device 1929 * @lcqe_buf: the buffer where the last cqe is received 1930 */ 1931 static int irdma_process_resize_list(struct irdma_cq *iwcq, 1932 struct irdma_device *iwdev, 1933 struct irdma_cq_buf *lcqe_buf) 1934 { 1935 struct list_head *tmp_node, *list_node; 1936 struct irdma_cq_buf *cq_buf; 1937 int cnt = 0; 1938 1939 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 1940 cq_buf = list_entry(list_node, struct irdma_cq_buf, list); 1941 if (cq_buf == lcqe_buf) 1942 return cnt; 1943 1944 list_del(&cq_buf->list); 1945 queue_work(iwdev->cleanup_wq, &cq_buf->work); 1946 cnt++; 1947 } 1948 1949 return cnt; 1950 } 1951 1952 /** 1953 * irdma_destroy_srq - destroy srq 1954 * @ibsrq: srq pointer 1955 * @udata: user data 1956 */ 1957 static int irdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1958 { 1959 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 1960 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 1961 struct irdma_sc_srq *srq = &iwsrq->sc_srq; 1962 1963 irdma_srq_wq_destroy(iwdev->rf, srq); 1964 irdma_srq_free_rsrc(iwdev->rf, iwsrq); 1965 return 0; 1966 } 1967 1968 /** 1969 * irdma_destroy_cq - destroy cq 1970 * @ib_cq: cq pointer 1971 * @udata: user data 1972 */ 1973 static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1974 { 1975 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1976 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1977 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1978 struct irdma_sc_dev *dev = cq->dev; 1979 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1980 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1981 unsigned long flags; 1982 1983 spin_lock_irqsave(&iwcq->lock, flags); 1984 if (!list_empty(&iwcq->cmpl_generated)) 1985 irdma_remove_cmpls_list(iwcq); 1986 if (!list_empty(&iwcq->resize_list)) 1987 irdma_process_resize_list(iwcq, iwdev, NULL); 1988 spin_unlock_irqrestore(&iwcq->lock, flags); 1989 1990 irdma_cq_rem_ref(ib_cq); 1991 wait_for_completion(&iwcq->free_cq); 1992 1993 irdma_cq_wq_destroy(iwdev->rf, cq); 1994 1995 spin_lock_irqsave(&iwceq->ce_lock, flags); 1996 irdma_sc_cleanup_ceqes(cq, ceq); 1997 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1998 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1999 2000 return 0; 2001 } 2002 2003 /** 2004 * irdma_resize_cq - resize cq 2005 * @ibcq: cq to be resized 2006 * @entries: desired cq size 2007 * @udata: user data 2008 */ 2009 static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, 2010 struct ib_udata *udata) 2011 { 2012 #define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer) 2013 struct irdma_cq *iwcq = to_iwcq(ibcq); 2014 struct irdma_sc_dev *dev = iwcq->sc_cq.dev; 2015 struct irdma_cqp_request *cqp_request; 2016 struct cqp_cmds_info *cqp_info; 2017 struct irdma_modify_cq_info *m_info; 2018 struct irdma_modify_cq_info info = {}; 2019 struct irdma_dma_mem kmem_buf; 2020 struct irdma_cq_mr *cqmr_buf; 2021 struct irdma_pbl *iwpbl_buf; 2022 struct irdma_device *iwdev; 2023 struct irdma_pci_f *rf; 2024 struct irdma_cq_buf *cq_buf = NULL; 2025 unsigned long flags; 2026 u8 cqe_size; 2027 int ret; 2028 2029 iwdev = to_iwdev(ibcq->device); 2030 rf = iwdev->rf; 2031 2032 if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2033 IRDMA_FEATURE_CQ_RESIZE)) 2034 return -EOPNOTSUPP; 2035 2036 if (udata && udata->inlen < IRDMA_RESIZE_CQ_MIN_REQ_LEN) 2037 return -EINVAL; 2038 2039 if (entries > rf->max_cqe) 2040 return -EINVAL; 2041 2042 if (!iwcq->user_mode) { 2043 entries += 2; 2044 2045 if (!iwcq->sc_cq.cq_uk.avoid_mem_cflct && 2046 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2047 entries *= 2; 2048 2049 if (entries & 1) 2050 entries += 1; /* cq size must be an even number */ 2051 2052 cqe_size = iwcq->sc_cq.cq_uk.avoid_mem_cflct ? 64 : 32; 2053 if (entries * cqe_size == IRDMA_HW_PAGE_SIZE) 2054 entries += 2; 2055 } 2056 2057 info.cq_size = max(entries, 4); 2058 2059 if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) 2060 return 0; 2061 2062 if (udata) { 2063 struct irdma_resize_cq_req req = {}; 2064 struct irdma_ucontext *ucontext = 2065 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2066 ibucontext); 2067 2068 if (ib_copy_from_udata(&req, udata, 2069 min(sizeof(req), udata->inlen))) 2070 return -EINVAL; 2071 2072 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2073 iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, 2074 &ucontext->cq_reg_mem_list); 2075 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2076 2077 if (!iwpbl_buf) 2078 return -ENOMEM; 2079 2080 cqmr_buf = &iwpbl_buf->cq_mr; 2081 if (iwpbl_buf->pbl_allocated) { 2082 info.virtual_map = true; 2083 info.pbl_chunk_size = 1; 2084 info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; 2085 } else { 2086 info.cq_pa = cqmr_buf->cq_pbl.addr; 2087 } 2088 } else { 2089 /* Kmode CQ resize */ 2090 int rsize; 2091 2092 rsize = info.cq_size * sizeof(struct irdma_cqe); 2093 kmem_buf.size = ALIGN(round_up(rsize, 256), 256); 2094 kmem_buf.va = dma_alloc_coherent(dev->hw->device, 2095 kmem_buf.size, &kmem_buf.pa, 2096 GFP_KERNEL); 2097 if (!kmem_buf.va) 2098 return -ENOMEM; 2099 2100 info.cq_base = kmem_buf.va; 2101 info.cq_pa = kmem_buf.pa; 2102 cq_buf = kzalloc_obj(*cq_buf); 2103 if (!cq_buf) { 2104 ret = -ENOMEM; 2105 goto error; 2106 } 2107 } 2108 2109 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2110 if (!cqp_request) { 2111 ret = -ENOMEM; 2112 goto error; 2113 } 2114 2115 info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; 2116 info.cq_resize = true; 2117 2118 cqp_info = &cqp_request->info; 2119 m_info = &cqp_info->in.u.cq_modify.info; 2120 memcpy(m_info, &info, sizeof(*m_info)); 2121 2122 cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; 2123 cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; 2124 cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; 2125 cqp_info->post_sq = 1; 2126 ret = irdma_handle_cqp_op(rf, cqp_request); 2127 irdma_put_cqp_request(&rf->cqp, cqp_request); 2128 if (ret) 2129 goto error; 2130 2131 spin_lock_irqsave(&iwcq->lock, flags); 2132 if (cq_buf) { 2133 cq_buf->kmem_buf = iwcq->kmem; 2134 cq_buf->hw = dev->hw; 2135 memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); 2136 INIT_WORK(&cq_buf->work, irdma_free_cqbuf); 2137 list_add_tail(&cq_buf->list, &iwcq->resize_list); 2138 iwcq->kmem = kmem_buf; 2139 } 2140 2141 irdma_sc_cq_resize(&iwcq->sc_cq, &info); 2142 ibcq->cqe = info.cq_size - 1; 2143 spin_unlock_irqrestore(&iwcq->lock, flags); 2144 2145 return 0; 2146 error: 2147 if (!udata) { 2148 dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, 2149 kmem_buf.pa); 2150 kmem_buf.va = NULL; 2151 } 2152 kfree(cq_buf); 2153 2154 return ret; 2155 } 2156 2157 /** 2158 * irdma_srq_event - event notification for srq limit 2159 * @srq: shared srq struct 2160 */ 2161 void irdma_srq_event(struct irdma_sc_srq *srq) 2162 { 2163 struct irdma_srq *iwsrq = container_of(srq, struct irdma_srq, sc_srq); 2164 struct ib_srq *ibsrq = &iwsrq->ibsrq; 2165 struct ib_event event; 2166 2167 srq->srq_limit = 0; 2168 2169 if (!ibsrq->event_handler) 2170 return; 2171 2172 event.device = ibsrq->device; 2173 event.element.port_num = 1; 2174 event.element.srq = ibsrq; 2175 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 2176 ibsrq->event_handler(&event, ibsrq->srq_context); 2177 } 2178 2179 /** 2180 * irdma_modify_srq - modify srq request 2181 * @ibsrq: srq's pointer for modify 2182 * @attr: access attributes 2183 * @attr_mask: state mask 2184 * @udata: user data 2185 */ 2186 static int irdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 2187 enum ib_srq_attr_mask attr_mask, 2188 struct ib_udata *udata) 2189 { 2190 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 2191 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2192 struct irdma_cqp_request *cqp_request; 2193 struct irdma_pci_f *rf = iwdev->rf; 2194 struct irdma_modify_srq_info *info; 2195 struct cqp_cmds_info *cqp_info; 2196 int status; 2197 2198 if (attr_mask & IB_SRQ_MAX_WR) 2199 return -EINVAL; 2200 2201 if (!(attr_mask & IB_SRQ_LIMIT)) 2202 return 0; 2203 2204 if (attr->srq_limit > iwsrq->sc_srq.srq_uk.srq_size) 2205 return -EINVAL; 2206 2207 /* Execute this cqp op synchronously, so we can update srq_limit 2208 * upon successful completion. 2209 */ 2210 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2211 if (!cqp_request) 2212 return -ENOMEM; 2213 2214 cqp_info = &cqp_request->info; 2215 info = &cqp_info->in.u.srq_modify.info; 2216 info->srq_limit = attr->srq_limit; 2217 if (info->srq_limit > 0xFFF) 2218 info->srq_limit = 0xFFF; 2219 info->arm_limit_event = 1; 2220 2221 cqp_info->cqp_cmd = IRDMA_OP_SRQ_MODIFY; 2222 cqp_info->post_sq = 1; 2223 cqp_info->in.u.srq_modify.srq = &iwsrq->sc_srq; 2224 cqp_info->in.u.srq_modify.scratch = (uintptr_t)cqp_request; 2225 status = irdma_handle_cqp_op(rf, cqp_request); 2226 irdma_put_cqp_request(&rf->cqp, cqp_request); 2227 if (status) 2228 return status; 2229 2230 iwsrq->sc_srq.srq_limit = info->srq_limit; 2231 2232 return 0; 2233 } 2234 2235 static int irdma_setup_umode_srq(struct irdma_device *iwdev, 2236 struct irdma_srq *iwsrq, 2237 struct irdma_srq_init_info *info, 2238 struct ib_udata *udata) 2239 { 2240 #define IRDMA_CREATE_SRQ_MIN_REQ_LEN \ 2241 offsetofend(struct irdma_create_srq_req, user_shadow_area) 2242 struct irdma_create_srq_req req = {}; 2243 struct irdma_ucontext *ucontext; 2244 struct irdma_srq_mr *srqmr; 2245 struct irdma_pbl *iwpbl; 2246 unsigned long flags; 2247 2248 iwsrq->user_mode = true; 2249 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2250 ibucontext); 2251 2252 if (udata->inlen < IRDMA_CREATE_SRQ_MIN_REQ_LEN) 2253 return -EINVAL; 2254 2255 if (ib_copy_from_udata(&req, udata, 2256 min(sizeof(req), udata->inlen))) 2257 return -EFAULT; 2258 2259 spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); 2260 iwpbl = irdma_get_pbl((unsigned long)req.user_srq_buf, 2261 &ucontext->srq_reg_mem_list); 2262 spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); 2263 if (!iwpbl) 2264 return -EPROTO; 2265 2266 iwsrq->iwpbl = iwpbl; 2267 srqmr = &iwpbl->srq_mr; 2268 2269 if (iwpbl->pbl_allocated) { 2270 info->virtual_map = true; 2271 info->pbl_chunk_size = 1; 2272 info->first_pm_pbl_idx = srqmr->srq_pbl.idx; 2273 info->leaf_pbl_size = 1; 2274 } else { 2275 info->srq_pa = srqmr->srq_pbl.addr; 2276 } 2277 info->shadow_area_pa = srqmr->shadow; 2278 2279 return 0; 2280 } 2281 2282 static int irdma_setup_kmode_srq(struct irdma_device *iwdev, 2283 struct irdma_srq *iwsrq, 2284 struct irdma_srq_init_info *info, u32 depth, 2285 u8 shift) 2286 { 2287 struct irdma_srq_uk_init_info *ukinfo = &info->srq_uk_init_info; 2288 struct irdma_dma_mem *mem = &iwsrq->kmem; 2289 u32 size, ring_size; 2290 2291 ring_size = depth * IRDMA_QP_WQE_MIN_SIZE; 2292 size = ring_size + (IRDMA_SHADOW_AREA_SIZE << 3); 2293 2294 mem->size = ALIGN(size, 256); 2295 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 2296 &mem->pa, GFP_KERNEL); 2297 if (!mem->va) 2298 return -ENOMEM; 2299 2300 ukinfo->srq = mem->va; 2301 ukinfo->srq_size = depth >> shift; 2302 ukinfo->shadow_area = mem->va + ring_size; 2303 2304 info->srq_pa = mem->pa; 2305 info->shadow_area_pa = info->srq_pa + ring_size; 2306 2307 return 0; 2308 } 2309 2310 /** 2311 * irdma_create_srq - create srq 2312 * @ibsrq: ib's srq pointer 2313 * @initattrs: attributes for srq 2314 * @udata: user data for create srq 2315 */ 2316 static int irdma_create_srq(struct ib_srq *ibsrq, 2317 struct ib_srq_init_attr *initattrs, 2318 struct ib_udata *udata) 2319 { 2320 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 2321 struct ib_srq_attr *attr = &initattrs->attr; 2322 struct irdma_pd *iwpd = to_iwpd(ibsrq->pd); 2323 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2324 struct irdma_srq_uk_init_info *ukinfo; 2325 struct irdma_cqp_request *cqp_request; 2326 struct irdma_srq_init_info info = {}; 2327 struct irdma_pci_f *rf = iwdev->rf; 2328 struct irdma_uk_attrs *uk_attrs; 2329 struct cqp_cmds_info *cqp_info; 2330 int err_code = 0; 2331 u32 depth; 2332 u8 shift; 2333 2334 uk_attrs = &rf->sc_dev.hw_attrs.uk_attrs; 2335 ukinfo = &info.srq_uk_init_info; 2336 2337 if (initattrs->srq_type != IB_SRQT_BASIC) 2338 return -EOPNOTSUPP; 2339 2340 if (!(uk_attrs->feature_flags & IRDMA_FEATURE_SRQ) || 2341 attr->max_sge > uk_attrs->max_hw_wq_frags) 2342 return -EINVAL; 2343 2344 refcount_set(&iwsrq->refcnt, 1); 2345 spin_lock_init(&iwsrq->lock); 2346 err_code = irdma_alloc_rsrc(rf, rf->allocated_srqs, rf->max_srq, 2347 &iwsrq->srq_num, &rf->next_srq); 2348 if (err_code) 2349 return err_code; 2350 2351 ukinfo->max_srq_frag_cnt = attr->max_sge; 2352 ukinfo->uk_attrs = uk_attrs; 2353 ukinfo->srq_id = iwsrq->srq_num; 2354 2355 irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_srq_frag_cnt, 0, 2356 &shift); 2357 2358 err_code = irdma_get_srqdepth(ukinfo->uk_attrs, attr->max_wr, 2359 shift, &depth); 2360 if (err_code) 2361 return err_code; 2362 2363 /* Actual SRQ size in WRs for ring and HW */ 2364 ukinfo->srq_size = depth >> shift; 2365 2366 /* Max postable WRs to SRQ */ 2367 iwsrq->max_wr = (depth - IRDMA_RQ_RSVD) >> shift; 2368 attr->max_wr = iwsrq->max_wr; 2369 2370 if (udata) 2371 err_code = irdma_setup_umode_srq(iwdev, iwsrq, &info, udata); 2372 else 2373 err_code = irdma_setup_kmode_srq(iwdev, iwsrq, &info, depth, 2374 shift); 2375 2376 if (err_code) 2377 goto free_rsrc; 2378 2379 info.vsi = &iwdev->vsi; 2380 info.pd = &iwpd->sc_pd; 2381 2382 iwsrq->sc_srq.srq_uk.lock = &iwsrq->lock; 2383 err_code = irdma_sc_srq_init(&iwsrq->sc_srq, &info); 2384 if (err_code) 2385 goto free_dmem; 2386 2387 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2388 if (!cqp_request) { 2389 err_code = -ENOMEM; 2390 goto free_dmem; 2391 } 2392 2393 cqp_info = &cqp_request->info; 2394 cqp_info->cqp_cmd = IRDMA_OP_SRQ_CREATE; 2395 cqp_info->post_sq = 1; 2396 cqp_info->in.u.srq_create.srq = &iwsrq->sc_srq; 2397 cqp_info->in.u.srq_create.scratch = (uintptr_t)cqp_request; 2398 err_code = irdma_handle_cqp_op(rf, cqp_request); 2399 irdma_put_cqp_request(&rf->cqp, cqp_request); 2400 if (err_code) 2401 goto free_dmem; 2402 2403 if (udata) { 2404 struct irdma_create_srq_resp resp = {}; 2405 2406 resp.srq_id = iwsrq->srq_num; 2407 resp.srq_size = ukinfo->srq_size; 2408 err_code = ib_respond_udata(udata, resp); 2409 if (err_code) 2410 goto srq_destroy; 2411 } 2412 2413 return 0; 2414 2415 srq_destroy: 2416 irdma_srq_wq_destroy(rf, &iwsrq->sc_srq); 2417 2418 free_dmem: 2419 if (!iwsrq->user_mode) 2420 dma_free_coherent(rf->hw.device, iwsrq->kmem.size, 2421 iwsrq->kmem.va, iwsrq->kmem.pa); 2422 free_rsrc: 2423 irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num); 2424 return err_code; 2425 } 2426 2427 /** 2428 * irdma_query_srq - get SRQ attributes 2429 * @ibsrq: the SRQ to query 2430 * @attr: the attributes of the SRQ 2431 */ 2432 static int irdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 2433 { 2434 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2435 2436 attr->max_wr = iwsrq->max_wr; 2437 attr->max_sge = iwsrq->sc_srq.srq_uk.max_srq_frag_cnt; 2438 attr->srq_limit = iwsrq->sc_srq.srq_limit; 2439 2440 return 0; 2441 } 2442 2443 static inline int cq_validate_flags(u32 flags, u8 hw_rev) 2444 { 2445 /* GEN1/2 does not support CQ create flags */ 2446 if (hw_rev <= IRDMA_GEN_2) 2447 return flags ? -EOPNOTSUPP : 0; 2448 2449 return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; 2450 } 2451 2452 /** 2453 * irdma_create_cq - create cq 2454 * @ibcq: CQ allocated 2455 * @attr: attributes for cq 2456 * @attrs: uverbs attribute bundle 2457 */ 2458 static int irdma_create_cq(struct ib_cq *ibcq, 2459 const struct ib_cq_init_attr *attr, 2460 struct uverbs_attr_bundle *attrs) 2461 { 2462 #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf) 2463 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) 2464 struct ib_udata *udata = &attrs->driver_udata; 2465 struct ib_device *ibdev = ibcq->device; 2466 struct irdma_device *iwdev = to_iwdev(ibdev); 2467 struct irdma_pci_f *rf = iwdev->rf; 2468 struct irdma_cq *iwcq = to_iwcq(ibcq); 2469 u32 cq_num = 0; 2470 struct irdma_sc_cq *cq; 2471 struct irdma_sc_dev *dev = &rf->sc_dev; 2472 struct irdma_cq_init_info info = {}; 2473 struct irdma_cqp_request *cqp_request; 2474 struct cqp_cmds_info *cqp_info; 2475 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 2476 unsigned long flags; 2477 int err_code; 2478 int entries = attr->cqe; 2479 bool cqe_64byte_ena; 2480 u8 cqe_size; 2481 2482 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 2483 if (err_code) 2484 return err_code; 2485 2486 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || 2487 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) 2488 return -EINVAL; 2489 2490 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 2491 &rf->next_cq); 2492 if (err_code) 2493 return err_code; 2494 2495 cq = &iwcq->sc_cq; 2496 cq->back_cq = iwcq; 2497 refcount_set(&iwcq->refcnt, 1); 2498 spin_lock_init(&iwcq->lock); 2499 INIT_LIST_HEAD(&iwcq->resize_list); 2500 INIT_LIST_HEAD(&iwcq->cmpl_generated); 2501 iwcq->cq_num = cq_num; 2502 info.dev = dev; 2503 ukinfo->cq_size = max(entries, 4); 2504 ukinfo->cq_id = cq_num; 2505 cqe_64byte_ena = dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? 2506 true : false; 2507 cqe_size = cqe_64byte_ena ? 64 : 32; 2508 ukinfo->avoid_mem_cflct = cqe_64byte_ena; 2509 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 2510 if (attr->comp_vector < rf->ceqs_count) 2511 info.ceq_id = attr->comp_vector; 2512 info.ceq_id_valid = true; 2513 info.ceqe_mask = 1; 2514 info.type = IRDMA_CQ_TYPE_IWARP; 2515 info.vsi = &iwdev->vsi; 2516 2517 if (udata) { 2518 struct irdma_ucontext *ucontext; 2519 struct irdma_create_cq_req req = {}; 2520 struct irdma_cq_mr *cqmr; 2521 struct irdma_pbl *iwpbl; 2522 struct irdma_pbl *iwpbl_shadow; 2523 struct irdma_cq_mr *cqmr_shadow; 2524 2525 iwcq->user_mode = true; 2526 ucontext = 2527 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2528 ibucontext); 2529 if (ib_copy_from_udata(&req, udata, 2530 min(sizeof(req), udata->inlen))) { 2531 err_code = -EFAULT; 2532 goto cq_free_rsrc; 2533 } 2534 2535 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2536 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 2537 &ucontext->cq_reg_mem_list); 2538 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2539 if (!iwpbl) { 2540 err_code = -EPROTO; 2541 goto cq_free_rsrc; 2542 } 2543 2544 cqmr = &iwpbl->cq_mr; 2545 2546 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2547 IRDMA_FEATURE_CQ_RESIZE) { 2548 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2549 iwpbl_shadow = irdma_get_pbl( 2550 (unsigned long)req.user_shadow_area, 2551 &ucontext->cq_reg_mem_list); 2552 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2553 2554 if (!iwpbl_shadow) { 2555 err_code = -EPROTO; 2556 goto cq_free_rsrc; 2557 } 2558 cqmr_shadow = &iwpbl_shadow->cq_mr; 2559 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 2560 } else { 2561 info.shadow_area_pa = cqmr->shadow; 2562 } 2563 if (iwpbl->pbl_allocated) { 2564 info.virtual_map = true; 2565 info.pbl_chunk_size = 1; 2566 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 2567 } else { 2568 info.cq_base_pa = cqmr->cq_pbl.addr; 2569 } 2570 } else { 2571 /* Kmode allocations */ 2572 int rsize; 2573 2574 if (entries < 1 || entries > rf->max_cqe) { 2575 err_code = -EINVAL; 2576 goto cq_free_rsrc; 2577 } 2578 2579 entries += 2; 2580 if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2581 entries *= 2; 2582 2583 if (entries & 1) 2584 entries += 1; /* cq size must be an even number */ 2585 2586 if (entries * cqe_size == IRDMA_HW_PAGE_SIZE) 2587 entries += 2; 2588 2589 ukinfo->cq_size = entries; 2590 2591 if (cqe_64byte_ena) 2592 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe); 2593 else 2594 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 2595 iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256); 2596 iwcq->kmem.va = dma_alloc_coherent(dev->hw->device, 2597 iwcq->kmem.size, 2598 &iwcq->kmem.pa, GFP_KERNEL); 2599 if (!iwcq->kmem.va) { 2600 err_code = -ENOMEM; 2601 goto cq_free_rsrc; 2602 } 2603 2604 iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, 2605 64); 2606 iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device, 2607 iwcq->kmem_shadow.size, 2608 &iwcq->kmem_shadow.pa, 2609 GFP_KERNEL); 2610 if (!iwcq->kmem_shadow.va) { 2611 err_code = -ENOMEM; 2612 goto cq_free_rsrc; 2613 } 2614 info.shadow_area_pa = iwcq->kmem_shadow.pa; 2615 ukinfo->shadow_area = iwcq->kmem_shadow.va; 2616 ukinfo->cq_base = iwcq->kmem.va; 2617 info.cq_base_pa = iwcq->kmem.pa; 2618 } 2619 2620 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 2621 (u32)IRDMA_MAX_CQ_READ_THRESH); 2622 2623 if (irdma_sc_cq_init(cq, &info)) { 2624 ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n"); 2625 err_code = -EPROTO; 2626 goto cq_free_rsrc; 2627 } 2628 2629 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2630 if (!cqp_request) { 2631 err_code = -ENOMEM; 2632 goto cq_free_rsrc; 2633 } 2634 2635 cqp_info = &cqp_request->info; 2636 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 2637 cqp_info->post_sq = 1; 2638 cqp_info->in.u.cq_create.cq = cq; 2639 cqp_info->in.u.cq_create.check_overflow = true; 2640 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 2641 err_code = irdma_handle_cqp_op(rf, cqp_request); 2642 irdma_put_cqp_request(&rf->cqp, cqp_request); 2643 if (err_code) 2644 goto cq_free_rsrc; 2645 2646 if (udata) { 2647 struct irdma_create_cq_resp resp = {}; 2648 2649 resp.cq_id = info.cq_uk_init_info.cq_id; 2650 resp.cq_size = info.cq_uk_init_info.cq_size; 2651 err_code = ib_respond_udata(udata, resp); 2652 if (err_code) 2653 goto cq_destroy; 2654 } 2655 2656 init_completion(&iwcq->free_cq); 2657 2658 /* Populate table entry after CQ is fully created. */ 2659 smp_store_release(&rf->cq_table[cq_num], iwcq); 2660 2661 return 0; 2662 cq_destroy: 2663 irdma_cq_wq_destroy(rf, cq); 2664 cq_free_rsrc: 2665 irdma_cq_free_rsrc(rf, iwcq); 2666 2667 return err_code; 2668 } 2669 2670 /** 2671 * irdma_get_mr_access - get hw MR access permissions from IB access flags 2672 * @access: IB access flags 2673 * @hw_rev: Hardware version 2674 */ 2675 static inline u16 irdma_get_mr_access(int access, u8 hw_rev) 2676 { 2677 u16 hw_access = 0; 2678 2679 hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? 2680 IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; 2681 hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? 2682 IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; 2683 hw_access |= (access & IB_ACCESS_REMOTE_READ) ? 2684 IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; 2685 if (hw_rev >= IRDMA_GEN_3) { 2686 hw_access |= (access & IB_ACCESS_MW_BIND) ? 2687 IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; 2688 } 2689 hw_access |= (access & IB_ZERO_BASED) ? 2690 IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; 2691 hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; 2692 2693 return hw_access; 2694 } 2695 2696 /** 2697 * irdma_free_stag - free stag resource 2698 * @iwdev: irdma device 2699 * @stag: stag to free 2700 */ 2701 static void irdma_free_stag(struct irdma_device *iwdev, u32 stag) 2702 { 2703 u32 stag_idx; 2704 2705 stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; 2706 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); 2707 } 2708 2709 /** 2710 * irdma_create_stag - create random stag 2711 * @iwdev: irdma device 2712 */ 2713 static u32 irdma_create_stag(struct irdma_device *iwdev) 2714 { 2715 u32 stag = 0; 2716 u32 stag_index = 0; 2717 u32 next_stag_index; 2718 u32 driver_key; 2719 u32 random; 2720 u8 consumer_key; 2721 int ret; 2722 2723 get_random_bytes(&random, sizeof(random)); 2724 consumer_key = (u8)random; 2725 2726 driver_key = random & ~iwdev->rf->mr_stagmask; 2727 next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; 2728 next_stag_index %= iwdev->rf->max_mr; 2729 2730 ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, 2731 iwdev->rf->max_mr, &stag_index, 2732 &next_stag_index); 2733 if (ret) 2734 return stag; 2735 stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; 2736 stag |= driver_key; 2737 stag += (u32)consumer_key; 2738 2739 return stag; 2740 } 2741 2742 /** 2743 * irdma_next_pbl_addr - Get next pbl address 2744 * @pbl: pointer to a pble 2745 * @pinfo: info pointer 2746 * @idx: index 2747 */ 2748 static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, 2749 u32 *idx) 2750 { 2751 *idx += 1; 2752 if (!(*pinfo) || *idx != (*pinfo)->cnt) 2753 return ++pbl; 2754 *idx = 0; 2755 (*pinfo)++; 2756 2757 return (*pinfo)->addr; 2758 } 2759 2760 /** 2761 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 2762 * @iwmr: iwmr for IB's user page addresses 2763 * @pbl: ple pointer to save 1 level or 0 level pble 2764 * @pbl_len: Max number of PBL entries to populate 2765 * @level: indicated level 0, 1 or 2 2766 */ 2767 static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 2768 u32 pbl_len, enum irdma_pble_level level) 2769 { 2770 struct ib_umem *region = iwmr->region; 2771 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2772 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2773 struct irdma_pble_info *pinfo; 2774 struct ib_block_iter biter; 2775 u32 idx = 0; 2776 2777 if (!pbl_len) 2778 return; 2779 2780 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 2781 2782 if (iwmr->type == IRDMA_MEMREG_TYPE_QP) 2783 iwpbl->qp_mr.sq_page = sg_page(region->sgt_append.sgt.sgl); 2784 2785 rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { 2786 *pbl = rdma_block_iter_dma_address(&biter); 2787 if (!--pbl_len) 2788 break; 2789 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 2790 } 2791 } 2792 2793 /** 2794 * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous 2795 * @arr: lvl1 pbl array 2796 * @npages: page count 2797 * @pg_size: page size 2798 * 2799 */ 2800 static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) 2801 { 2802 u32 pg_idx; 2803 2804 for (pg_idx = 0; pg_idx < npages; pg_idx++) { 2805 if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) 2806 return false; 2807 } 2808 2809 return true; 2810 } 2811 2812 /** 2813 * irdma_check_mr_contiguous - check if MR is physically contiguous 2814 * @palloc: pbl allocation struct 2815 * @pg_size: page size 2816 */ 2817 static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, 2818 u32 pg_size) 2819 { 2820 struct irdma_pble_level2 *lvl2 = &palloc->level2; 2821 struct irdma_pble_info *leaf = lvl2->leaf; 2822 u64 *arr = NULL; 2823 u64 *start_addr = NULL; 2824 int i; 2825 bool ret; 2826 2827 if (palloc->level == PBLE_LEVEL_1) { 2828 arr = palloc->level1.addr; 2829 ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, 2830 pg_size); 2831 return ret; 2832 } 2833 2834 start_addr = leaf->addr; 2835 2836 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { 2837 arr = leaf->addr; 2838 if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) 2839 return false; 2840 ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); 2841 if (!ret) 2842 return false; 2843 } 2844 2845 return true; 2846 } 2847 2848 /** 2849 * irdma_setup_pbles - copy user pg address to pble's 2850 * @rf: RDMA PCI function 2851 * @iwmr: mr pointer for this memory registration 2852 * @lvl: requested pble levels 2853 */ 2854 static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, 2855 u8 lvl) 2856 { 2857 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2858 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2859 struct irdma_pble_info *pinfo; 2860 u64 *pbl; 2861 int status; 2862 enum irdma_pble_level level = PBLE_LEVEL_1; 2863 u32 pbl_len; 2864 2865 if (lvl) { 2866 status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, 2867 lvl); 2868 if (status) 2869 return status; 2870 2871 pbl_len = palloc->total_cnt; 2872 iwpbl->pbl_allocated = true; 2873 level = palloc->level; 2874 pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : 2875 palloc->level2.leaf; 2876 pbl = pinfo->addr; 2877 } else { 2878 pbl_len = IRDMA_MAX_SAVED_PHY_PGADDR; 2879 pbl = iwmr->pgaddrmem; 2880 } 2881 2882 irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level); 2883 2884 if (lvl) 2885 iwmr->pgaddrmem[0] = *pbl; 2886 2887 return 0; 2888 } 2889 2890 /** 2891 * irdma_handle_q_mem - handle memory for qp and cq 2892 * @iwdev: irdma device 2893 * @req: information for q memory management 2894 * @iwpbl: pble struct 2895 * @lvl: pble level mask 2896 */ 2897 static int irdma_handle_q_mem(struct irdma_device *iwdev, 2898 struct irdma_mem_reg_req *req, 2899 struct irdma_pbl *iwpbl, u8 lvl) 2900 { 2901 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2902 struct irdma_mr *iwmr = iwpbl->iwmr; 2903 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 2904 struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; 2905 struct irdma_srq_mr *srqmr = &iwpbl->srq_mr; 2906 struct irdma_hmc_pble *hmc_p; 2907 u64 *arr = iwmr->pgaddrmem; 2908 u32 pg_size, total; 2909 int err = 0; 2910 bool ret = true; 2911 2912 pg_size = iwmr->page_size; 2913 err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); 2914 if (err) 2915 return err; 2916 2917 if (lvl) 2918 arr = palloc->level1.addr; 2919 2920 switch (iwmr->type) { 2921 case IRDMA_MEMREG_TYPE_QP: 2922 total = req->sq_pages + req->rq_pages; 2923 hmc_p = &qpmr->sq_pbl; 2924 qpmr->shadow = (dma_addr_t)arr[total]; 2925 /* Need to use physical address for RQ of QP 2926 * in case it is associated with SRQ. 2927 */ 2928 qpmr->rq_pa = (dma_addr_t)arr[req->sq_pages]; 2929 if (lvl) { 2930 ret = irdma_check_mem_contiguous(arr, req->sq_pages, 2931 pg_size); 2932 if (ret) 2933 ret = irdma_check_mem_contiguous(&arr[req->sq_pages], 2934 req->rq_pages, 2935 pg_size); 2936 } 2937 2938 if (!ret) { 2939 hmc_p->idx = palloc->level1.idx; 2940 hmc_p = &qpmr->rq_pbl; 2941 hmc_p->idx = palloc->level1.idx + req->sq_pages; 2942 } else { 2943 hmc_p->addr = arr[0]; 2944 hmc_p = &qpmr->rq_pbl; 2945 hmc_p->addr = arr[req->sq_pages]; 2946 } 2947 break; 2948 case IRDMA_MEMREG_TYPE_SRQ: 2949 hmc_p = &srqmr->srq_pbl; 2950 srqmr->shadow = (dma_addr_t)arr[req->rq_pages]; 2951 if (lvl) 2952 ret = irdma_check_mem_contiguous(arr, req->rq_pages, 2953 pg_size); 2954 2955 if (!ret) 2956 hmc_p->idx = palloc->level1.idx; 2957 else 2958 hmc_p->addr = arr[0]; 2959 break; 2960 case IRDMA_MEMREG_TYPE_CQ: 2961 hmc_p = &cqmr->cq_pbl; 2962 2963 if (!(iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2964 IRDMA_FEATURE_CQ_RESIZE)) 2965 cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; 2966 2967 if (lvl) 2968 ret = irdma_check_mem_contiguous(arr, req->cq_pages, 2969 pg_size); 2970 2971 if (!ret) 2972 hmc_p->idx = palloc->level1.idx; 2973 else 2974 hmc_p->addr = arr[0]; 2975 break; 2976 default: 2977 ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n"); 2978 err = -EINVAL; 2979 } 2980 2981 if (lvl && ret) { 2982 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2983 iwpbl->pbl_allocated = false; 2984 } 2985 2986 return err; 2987 } 2988 2989 /** 2990 * irdma_hw_alloc_mw - create the hw memory window 2991 * @iwdev: irdma device 2992 * @iwmr: pointer to memory window info 2993 */ 2994 static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) 2995 { 2996 struct irdma_mw_alloc_info *info; 2997 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2998 struct irdma_cqp_request *cqp_request; 2999 struct cqp_cmds_info *cqp_info; 3000 int status; 3001 3002 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3003 if (!cqp_request) 3004 return -ENOMEM; 3005 3006 cqp_info = &cqp_request->info; 3007 info = &cqp_info->in.u.mw_alloc.info; 3008 memset(info, 0, sizeof(*info)); 3009 if (iwmr->ibmw.type == IB_MW_TYPE_1) 3010 info->mw_wide = true; 3011 3012 info->page_size = PAGE_SIZE; 3013 info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3014 info->pd_id = iwpd->sc_pd.pd_id; 3015 info->remote_access = true; 3016 cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; 3017 cqp_info->post_sq = 1; 3018 cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; 3019 cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; 3020 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3021 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3022 3023 return status; 3024 } 3025 3026 /** 3027 * irdma_alloc_mw - Allocate memory window 3028 * @ibmw: Memory Window 3029 * @udata: user data pointer 3030 */ 3031 static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 3032 { 3033 struct irdma_device *iwdev = to_iwdev(ibmw->device); 3034 struct irdma_mr *iwmr = to_iwmw(ibmw); 3035 int err_code; 3036 u32 stag; 3037 3038 stag = irdma_create_stag(iwdev); 3039 if (!stag) 3040 return -ENOMEM; 3041 3042 iwmr->stag = stag; 3043 ibmw->rkey = stag; 3044 3045 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 3046 if (err_code) { 3047 irdma_free_stag(iwdev, stag); 3048 return err_code; 3049 } 3050 3051 return 0; 3052 } 3053 3054 /** 3055 * irdma_dealloc_mw - Dealloc memory window 3056 * @ibmw: memory window structure. 3057 */ 3058 static int irdma_dealloc_mw(struct ib_mw *ibmw) 3059 { 3060 struct ib_pd *ibpd = ibmw->pd; 3061 struct irdma_pd *iwpd = to_iwpd(ibpd); 3062 struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); 3063 struct irdma_device *iwdev = to_iwdev(ibmw->device); 3064 struct irdma_cqp_request *cqp_request; 3065 struct cqp_cmds_info *cqp_info; 3066 struct irdma_dealloc_stag_info *info; 3067 3068 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3069 if (!cqp_request) 3070 return -ENOMEM; 3071 3072 cqp_info = &cqp_request->info; 3073 info = &cqp_info->in.u.dealloc_stag.info; 3074 memset(info, 0, sizeof(*info)); 3075 info->pd_id = iwpd->sc_pd.pd_id; 3076 info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3077 info->mr = false; 3078 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3079 cqp_info->post_sq = 1; 3080 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3081 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3082 irdma_handle_cqp_op(iwdev->rf, cqp_request); 3083 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3084 irdma_free_stag(iwdev, iwmr->stag); 3085 3086 return 0; 3087 } 3088 3089 /** 3090 * irdma_hw_alloc_stag - cqp command to allocate stag 3091 * @iwdev: irdma device 3092 * @iwmr: irdma mr pointer 3093 */ 3094 static int irdma_hw_alloc_stag(struct irdma_device *iwdev, 3095 struct irdma_mr *iwmr) 3096 { 3097 struct irdma_allocate_stag_info *info; 3098 struct ib_pd *pd = iwmr->ibmr.pd; 3099 struct irdma_pd *iwpd = to_iwpd(pd); 3100 int status; 3101 struct irdma_cqp_request *cqp_request; 3102 struct cqp_cmds_info *cqp_info; 3103 3104 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3105 if (!cqp_request) 3106 return -ENOMEM; 3107 3108 cqp_info = &cqp_request->info; 3109 info = &cqp_info->in.u.alloc_stag.info; 3110 info->page_size = PAGE_SIZE; 3111 info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3112 info->pd_id = iwpd->sc_pd.pd_id; 3113 info->total_len = iwmr->len; 3114 info->remote_access = true; 3115 cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; 3116 cqp_info->post_sq = 1; 3117 cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; 3118 cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; 3119 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3120 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3121 if (status) 3122 return status; 3123 3124 iwmr->is_hwreg = true; 3125 return 0; 3126 } 3127 3128 /** 3129 * irdma_alloc_mr - register stag for fast memory registration 3130 * @pd: ibpd pointer 3131 * @mr_type: memory for stag registrion 3132 * @max_num_sg: man number of pages 3133 */ 3134 static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 3135 u32 max_num_sg) 3136 { 3137 struct irdma_device *iwdev = to_iwdev(pd->device); 3138 struct irdma_pble_alloc *palloc; 3139 struct irdma_pbl *iwpbl; 3140 struct irdma_mr *iwmr; 3141 u32 stag; 3142 int err_code; 3143 3144 iwmr = kzalloc_obj(*iwmr); 3145 if (!iwmr) 3146 return ERR_PTR(-ENOMEM); 3147 3148 stag = irdma_create_stag(iwdev); 3149 if (!stag) { 3150 err_code = -ENOMEM; 3151 goto err; 3152 } 3153 3154 iwmr->stag = stag; 3155 iwmr->ibmr.rkey = stag; 3156 iwmr->ibmr.lkey = stag; 3157 iwmr->ibmr.pd = pd; 3158 iwmr->ibmr.device = pd->device; 3159 iwpbl = &iwmr->iwpbl; 3160 iwpbl->iwmr = iwmr; 3161 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 3162 palloc = &iwpbl->pble_alloc; 3163 iwmr->page_cnt = max_num_sg; 3164 /* Use system PAGE_SIZE as the sg page sizes are unknown at this point */ 3165 iwmr->len = max_num_sg * PAGE_SIZE; 3166 err_code = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 3167 false); 3168 if (err_code) 3169 goto err_get_pble; 3170 3171 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 3172 if (err_code) 3173 goto err_alloc_stag; 3174 3175 iwpbl->pbl_allocated = true; 3176 3177 return &iwmr->ibmr; 3178 err_alloc_stag: 3179 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3180 err_get_pble: 3181 irdma_free_stag(iwdev, stag); 3182 err: 3183 kfree(iwmr); 3184 3185 return ERR_PTR(err_code); 3186 } 3187 3188 /** 3189 * irdma_set_page - populate pbl list for fmr 3190 * @ibmr: ib mem to access iwarp mr pointer 3191 * @addr: page dma address fro pbl list 3192 */ 3193 static int irdma_set_page(struct ib_mr *ibmr, u64 addr) 3194 { 3195 struct irdma_mr *iwmr = to_iwmr(ibmr); 3196 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3197 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3198 u64 *pbl; 3199 3200 if (unlikely(iwmr->npages == iwmr->page_cnt)) 3201 return -ENOMEM; 3202 3203 if (palloc->level == PBLE_LEVEL_2) { 3204 struct irdma_pble_info *palloc_info = 3205 palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT); 3206 3207 palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr; 3208 } else { 3209 pbl = palloc->level1.addr; 3210 pbl[iwmr->npages] = addr; 3211 } 3212 iwmr->npages++; 3213 3214 return 0; 3215 } 3216 3217 /** 3218 * irdma_map_mr_sg - map of sg list for fmr 3219 * @ibmr: ib mem to access iwarp mr pointer 3220 * @sg: scatter gather list 3221 * @sg_nents: number of sg pages 3222 * @sg_offset: scatter gather list for fmr 3223 */ 3224 static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3225 int sg_nents, unsigned int *sg_offset) 3226 { 3227 struct irdma_mr *iwmr = to_iwmr(ibmr); 3228 3229 iwmr->npages = 0; 3230 3231 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); 3232 } 3233 3234 /** 3235 * irdma_hwreg_mr - send cqp command for memory registration 3236 * @iwdev: irdma device 3237 * @iwmr: irdma mr pointer 3238 * @access: access for MR 3239 */ 3240 static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, 3241 u16 access) 3242 { 3243 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3244 struct irdma_reg_ns_stag_info *stag_info; 3245 struct ib_pd *pd = iwmr->ibmr.pd; 3246 struct irdma_pd *iwpd = to_iwpd(pd); 3247 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3248 struct irdma_cqp_request *cqp_request; 3249 struct cqp_cmds_info *cqp_info; 3250 int ret; 3251 3252 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3253 if (!cqp_request) 3254 return -ENOMEM; 3255 3256 cqp_info = &cqp_request->info; 3257 stag_info = &cqp_info->in.u.mr_reg_non_shared.info; 3258 stag_info->va = iwpbl->user_base; 3259 stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3260 stag_info->stag_key = (u8)iwmr->stag; 3261 stag_info->total_len = iwmr->len; 3262 stag_info->access_rights = irdma_get_mr_access(access, 3263 iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev); 3264 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 3265 stag_info->remote_atomics_en = (access & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3266 stag_info->pd_id = iwpd->sc_pd.pd_id; 3267 stag_info->all_memory = iwmr->dma_mr; 3268 if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) 3269 stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; 3270 else 3271 stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; 3272 stag_info->page_size = iwmr->page_size; 3273 3274 if (iwpbl->pbl_allocated) { 3275 if (palloc->level == PBLE_LEVEL_1) { 3276 stag_info->first_pm_pbl_index = palloc->level1.idx; 3277 stag_info->chunk_size = 1; 3278 } else { 3279 stag_info->first_pm_pbl_index = palloc->level2.root.idx; 3280 stag_info->chunk_size = 3; 3281 } 3282 } else { 3283 stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; 3284 } 3285 3286 cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; 3287 cqp_info->post_sq = 1; 3288 cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; 3289 cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; 3290 ret = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3291 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3292 3293 if (!ret) 3294 iwmr->is_hwreg = true; 3295 3296 return ret; 3297 } 3298 3299 static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, 3300 bool create_stag) 3301 { 3302 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3303 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3304 u32 stag = 0; 3305 u8 lvl; 3306 int err; 3307 3308 lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; 3309 iwmr->access = access; 3310 3311 err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); 3312 if (err) 3313 return err; 3314 3315 if (lvl) { 3316 err = irdma_check_mr_contiguous(&iwpbl->pble_alloc, 3317 iwmr->page_size); 3318 if (err) { 3319 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 3320 iwpbl->pbl_allocated = false; 3321 } 3322 } 3323 3324 if (create_stag) { 3325 stag = irdma_create_stag(iwdev); 3326 if (!stag) { 3327 err = -ENOMEM; 3328 goto free_pble; 3329 } 3330 3331 iwmr->stag = stag; 3332 iwmr->ibmr.rkey = stag; 3333 iwmr->ibmr.lkey = stag; 3334 } 3335 3336 err = irdma_hwreg_mr(iwdev, iwmr, access); 3337 if (err) 3338 goto err_hwreg; 3339 3340 return 0; 3341 3342 err_hwreg: 3343 if (stag) 3344 irdma_free_stag(iwdev, stag); 3345 3346 free_pble: 3347 if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 3348 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 3349 3350 return err; 3351 } 3352 3353 static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, 3354 struct ib_pd *pd, u64 virt, 3355 enum irdma_memreg_type reg_type) 3356 { 3357 struct irdma_device *iwdev = to_iwdev(pd->device); 3358 struct irdma_pbl *iwpbl; 3359 struct irdma_mr *iwmr; 3360 unsigned long pgsz_bitmap; 3361 3362 iwmr = kzalloc_obj(*iwmr); 3363 if (!iwmr) 3364 return ERR_PTR(-ENOMEM); 3365 3366 iwpbl = &iwmr->iwpbl; 3367 iwpbl->iwmr = iwmr; 3368 iwmr->region = region; 3369 iwmr->ibmr.pd = pd; 3370 iwmr->ibmr.device = pd->device; 3371 iwmr->ibmr.iova = virt; 3372 iwmr->type = reg_type; 3373 3374 pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ? 3375 iwdev->rf->sc_dev.hw_attrs.page_size_cap : SZ_4K; 3376 3377 iwmr->page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt); 3378 if (unlikely(!iwmr->page_size)) { 3379 kfree(iwmr); 3380 return ERR_PTR(-EOPNOTSUPP); 3381 } 3382 3383 iwmr->len = region->length; 3384 iwpbl->user_base = virt; 3385 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 3386 3387 return iwmr; 3388 } 3389 3390 static void irdma_free_iwmr(struct irdma_mr *iwmr) 3391 { 3392 kfree(iwmr); 3393 } 3394 3395 static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req, 3396 struct ib_udata *udata, 3397 struct irdma_mr *iwmr) 3398 { 3399 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3400 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3401 struct irdma_ucontext *ucontext = NULL; 3402 unsigned long flags; 3403 u32 total; 3404 int err; 3405 u8 lvl; 3406 3407 /* iWarp: Catch page not starting on OS page boundary */ 3408 if (!rdma_protocol_roce(&iwdev->ibdev, 1) && 3409 ib_umem_offset(iwmr->region)) 3410 return -EINVAL; 3411 3412 total = req.sq_pages + req.rq_pages + 1; 3413 if (total > iwmr->page_cnt) 3414 return -EINVAL; 3415 3416 total = req.sq_pages + req.rq_pages; 3417 lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3418 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3419 if (err) 3420 return err; 3421 3422 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3423 ibucontext); 3424 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 3425 list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 3426 iwpbl->on_list = true; 3427 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 3428 3429 return 0; 3430 } 3431 3432 static int irdma_reg_user_mr_type_srq(struct irdma_mem_reg_req req, 3433 struct ib_udata *udata, 3434 struct irdma_mr *iwmr) 3435 { 3436 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3437 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3438 struct irdma_ucontext *ucontext; 3439 unsigned long flags; 3440 u32 total; 3441 int err; 3442 u8 lvl; 3443 3444 total = req.rq_pages + IRDMA_SHADOW_PGCNT; 3445 if (total > iwmr->page_cnt) 3446 return -EINVAL; 3447 3448 lvl = req.rq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3449 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3450 if (err) 3451 return err; 3452 3453 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3454 ibucontext); 3455 spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); 3456 list_add_tail(&iwpbl->list, &ucontext->srq_reg_mem_list); 3457 iwpbl->on_list = true; 3458 spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); 3459 3460 return 0; 3461 } 3462 3463 static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req, 3464 struct ib_udata *udata, 3465 struct irdma_mr *iwmr) 3466 { 3467 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3468 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3469 struct irdma_ucontext *ucontext = NULL; 3470 u8 shadow_pgcnt = 1; 3471 unsigned long flags; 3472 u32 total; 3473 int err; 3474 u8 lvl; 3475 3476 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 3477 shadow_pgcnt = 0; 3478 total = req.cq_pages + shadow_pgcnt; 3479 if (total > iwmr->page_cnt) 3480 return -EINVAL; 3481 3482 lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3483 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3484 if (err) 3485 return err; 3486 3487 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3488 ibucontext); 3489 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 3490 list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 3491 iwpbl->on_list = true; 3492 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 3493 3494 return 0; 3495 } 3496 3497 /** 3498 * irdma_reg_user_mr - Register a user memory region 3499 * @pd: ptr of pd 3500 * @start: virtual start address 3501 * @len: length of mr 3502 * @virt: virtual address 3503 * @access: access of mr 3504 * @dmah: dma handle 3505 * @udata: user data 3506 */ 3507 static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, 3508 u64 virt, int access, 3509 struct ib_dmah *dmah, 3510 struct ib_udata *udata) 3511 { 3512 #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) 3513 struct irdma_device *iwdev = to_iwdev(pd->device); 3514 struct irdma_mem_reg_req req = {}; 3515 struct ib_umem *region = NULL; 3516 struct irdma_mr *iwmr = NULL; 3517 int err; 3518 3519 if (dmah) 3520 return ERR_PTR(-EOPNOTSUPP); 3521 3522 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3523 return ERR_PTR(-EINVAL); 3524 3525 if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) 3526 return ERR_PTR(-EINVAL); 3527 3528 region = ib_umem_get_va(pd->device, start, len, access); 3529 3530 if (IS_ERR(region)) { 3531 ibdev_dbg(&iwdev->ibdev, 3532 "VERBS: Failed to create ib_umem region\n"); 3533 return (struct ib_mr *)region; 3534 } 3535 3536 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { 3537 ib_umem_release(region); 3538 return ERR_PTR(-EFAULT); 3539 } 3540 3541 iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type); 3542 if (IS_ERR(iwmr)) { 3543 ib_umem_release(region); 3544 return (struct ib_mr *)iwmr; 3545 } 3546 3547 switch (req.reg_type) { 3548 case IRDMA_MEMREG_TYPE_QP: 3549 err = irdma_reg_user_mr_type_qp(req, udata, iwmr); 3550 if (err) 3551 goto error; 3552 3553 break; 3554 case IRDMA_MEMREG_TYPE_SRQ: 3555 err = irdma_reg_user_mr_type_srq(req, udata, iwmr); 3556 if (err) 3557 goto error; 3558 3559 break; 3560 case IRDMA_MEMREG_TYPE_CQ: 3561 err = irdma_reg_user_mr_type_cq(req, udata, iwmr); 3562 if (err) 3563 goto error; 3564 break; 3565 case IRDMA_MEMREG_TYPE_MEM: 3566 err = irdma_reg_user_mr_type_mem(iwmr, access, true); 3567 if (err) 3568 goto error; 3569 3570 break; 3571 default: 3572 err = -EINVAL; 3573 goto error; 3574 } 3575 3576 return &iwmr->ibmr; 3577 error: 3578 ib_umem_release(region); 3579 irdma_free_iwmr(iwmr); 3580 3581 return ERR_PTR(err); 3582 } 3583 3584 static int irdma_hwdereg_mr(struct ib_mr *ib_mr); 3585 3586 static void irdma_umem_dmabuf_revoke(void *priv) 3587 { 3588 /* priv is guaranteed to be valid any time this callback is invoked 3589 * because we do not set the callback until after successful iwmr 3590 * allocation and initialization. 3591 */ 3592 struct irdma_mr *iwmr = priv; 3593 int err; 3594 3595 /* Invalidate the key in hardware. This does not actually release the 3596 * key for potential reuse - that only occurs when the region is fully 3597 * deregistered. 3598 * 3599 * The irdma_hwdereg_mr call is a no-op if the region is not currently 3600 * registered with hardware. 3601 */ 3602 err = irdma_hwdereg_mr(&iwmr->ibmr); 3603 if (err) { 3604 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3605 3606 ibdev_err(&iwdev->ibdev, "dmabuf mr revoke failed %d", err); 3607 if (!iwdev->rf->reset) { 3608 iwdev->rf->reset = true; 3609 iwdev->rf->gen_ops.request_reset(iwdev->rf); 3610 } 3611 } 3612 } 3613 3614 static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start, 3615 u64 len, u64 virt, 3616 int fd, int access, 3617 struct ib_dmah *dmah, 3618 struct uverbs_attr_bundle *attrs) 3619 { 3620 struct irdma_device *iwdev = to_iwdev(pd->device); 3621 struct ib_umem_dmabuf *umem_dmabuf; 3622 struct irdma_mr *iwmr; 3623 int err; 3624 3625 if (dmah) 3626 return ERR_PTR(-EOPNOTSUPP); 3627 3628 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3629 return ERR_PTR(-EINVAL); 3630 3631 umem_dmabuf = 3632 ib_umem_dmabuf_get_pinned_revocable_and_lock(pd->device, start, 3633 len, fd, access); 3634 if (IS_ERR(umem_dmabuf)) { 3635 ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%pe]\n", 3636 umem_dmabuf); 3637 return ERR_CAST(umem_dmabuf); 3638 } 3639 3640 iwmr = irdma_alloc_iwmr(&umem_dmabuf->umem, pd, virt, IRDMA_MEMREG_TYPE_MEM); 3641 if (IS_ERR(iwmr)) { 3642 err = PTR_ERR(iwmr); 3643 goto err_release; 3644 } 3645 3646 err = irdma_reg_user_mr_type_mem(iwmr, access, true); 3647 if (err) 3648 goto err_iwmr; 3649 3650 ib_umem_dmabuf_set_revoke_locked(umem_dmabuf, irdma_umem_dmabuf_revoke, 3651 iwmr); 3652 ib_umem_dmabuf_revoke_unlock(umem_dmabuf); 3653 return &iwmr->ibmr; 3654 3655 err_iwmr: 3656 irdma_free_iwmr(iwmr); 3657 3658 err_release: 3659 ib_umem_dmabuf_revoke_unlock(umem_dmabuf); 3660 3661 /* Will result in a call to revoke, but driver callback is not set and 3662 * is therefore skipped. 3663 */ 3664 ib_umem_release(&umem_dmabuf->umem); 3665 3666 return ERR_PTR(err); 3667 } 3668 3669 static int irdma_hwdereg_mr(struct ib_mr *ib_mr) 3670 { 3671 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3672 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3673 struct irdma_pd *iwpd = to_iwpd(ib_mr->pd); 3674 struct irdma_dealloc_stag_info *info; 3675 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3676 struct irdma_cqp_request *cqp_request; 3677 struct cqp_cmds_info *cqp_info; 3678 int status; 3679 3680 /* Skip HW MR de-register when it is already de-registered 3681 * during an MR re-reregister and the re-registration fails 3682 */ 3683 if (!iwmr->is_hwreg) 3684 return 0; 3685 3686 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3687 if (!cqp_request) 3688 return -ENOMEM; 3689 3690 cqp_info = &cqp_request->info; 3691 info = &cqp_info->in.u.dealloc_stag.info; 3692 info->pd_id = iwpd->sc_pd.pd_id; 3693 info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3694 info->mr = true; 3695 if (iwpbl->pbl_allocated) 3696 info->dealloc_pbl = true; 3697 3698 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3699 cqp_info->post_sq = 1; 3700 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3701 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3702 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3703 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3704 if (status) 3705 return status; 3706 3707 iwmr->is_hwreg = false; 3708 return 0; 3709 } 3710 3711 /* 3712 * irdma_rereg_mr_trans - Re-register a user MR for a change translation. 3713 * @iwmr: ptr of iwmr 3714 * @start: virtual start address 3715 * @len: length of mr 3716 * @virt: virtual address 3717 * 3718 * Re-register a user memory region when a change translation is requested. 3719 * Re-register a new region while reusing the stag from the original registration. 3720 */ 3721 static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, 3722 u64 virt) 3723 { 3724 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3725 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3726 struct ib_pd *pd = iwmr->ibmr.pd; 3727 struct ib_umem *region; 3728 int err; 3729 3730 region = ib_umem_get_va(pd->device, start, len, iwmr->access); 3731 if (IS_ERR(region)) 3732 return PTR_ERR(region); 3733 3734 iwmr->region = region; 3735 iwmr->ibmr.iova = virt; 3736 iwmr->ibmr.pd = pd; 3737 iwmr->page_size = ib_umem_find_best_pgsz(region, 3738 iwdev->rf->sc_dev.hw_attrs.page_size_cap, 3739 virt); 3740 if (unlikely(!iwmr->page_size)) { 3741 err = -EOPNOTSUPP; 3742 goto err; 3743 } 3744 3745 iwmr->len = region->length; 3746 iwpbl->user_base = virt; 3747 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 3748 3749 err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false); 3750 if (err) 3751 goto err; 3752 3753 return 0; 3754 3755 err: 3756 ib_umem_release(region); 3757 iwmr->region = NULL; 3758 return err; 3759 } 3760 3761 /* 3762 * irdma_rereg_user_mr - Re-Register a user memory region(MR) 3763 * @ibmr: ib mem to access iwarp mr pointer 3764 * @flags: bit mask to indicate which of the attr's of MR modified 3765 * @start: virtual start address 3766 * @len: length of mr 3767 * @virt: virtual address 3768 * @new_access: bit mask of access flags 3769 * @new_pd: ptr of pd 3770 * @udata: user data 3771 * 3772 * Return: 3773 * NULL - Success, existing MR updated 3774 * ERR_PTR - error occurred 3775 */ 3776 static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, 3777 u64 start, u64 len, u64 virt, 3778 int new_access, struct ib_pd *new_pd, 3779 struct ib_udata *udata) 3780 { 3781 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3782 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3783 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3784 bool dmabuf_revocable = iwmr->region && iwmr->region->is_dmabuf; 3785 struct ib_umem_dmabuf *umem_dmabuf; 3786 int ret; 3787 3788 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3789 return ERR_PTR(-EINVAL); 3790 3791 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) 3792 return ERR_PTR(-EOPNOTSUPP); 3793 3794 ret = ib_umem_check_rereg(iwmr->region, flags, new_access); 3795 if (ret) 3796 return ERR_PTR(ret); 3797 3798 if (dmabuf_revocable) { 3799 umem_dmabuf = to_ib_umem_dmabuf(iwmr->region); 3800 3801 ib_umem_dmabuf_revoke_lock(umem_dmabuf); 3802 3803 /* If the dmabuf has been revoked, it means that the region has 3804 * been invalidated in HW. We must not allow it to become valid 3805 * again unless the user is requesting a change in translation 3806 * which will end up dropping the umem dmabuf and allocating an 3807 * entirely new umem anyway. 3808 */ 3809 if (umem_dmabuf->revoked && !(flags & IB_MR_REREG_TRANS)) { 3810 ret = -EINVAL; 3811 goto err_unlock; 3812 } 3813 } 3814 3815 ret = irdma_hwdereg_mr(ib_mr); 3816 if (ret) 3817 goto err_unlock; 3818 3819 if (flags & IB_MR_REREG_ACCESS) 3820 iwmr->access = new_access; 3821 3822 if (flags & IB_MR_REREG_PD) { 3823 iwmr->ibmr.pd = new_pd; 3824 iwmr->ibmr.device = new_pd->device; 3825 } 3826 3827 if (flags & IB_MR_REREG_TRANS) { 3828 if (iwpbl->pbl_allocated) { 3829 irdma_free_pble(iwdev->rf->pble_rsrc, 3830 &iwpbl->pble_alloc); 3831 iwpbl->pbl_allocated = false; 3832 } 3833 3834 if (dmabuf_revocable) { 3835 /* Must unlock before release to prevent deadlock */ 3836 ib_umem_dmabuf_revoke_unlock(umem_dmabuf); 3837 dmabuf_revocable = false; 3838 } 3839 3840 if (iwmr->region) { 3841 ib_umem_release(iwmr->region); 3842 iwmr->region = NULL; 3843 } 3844 3845 ret = irdma_rereg_mr_trans(iwmr, start, len, virt); 3846 } else { 3847 ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access); 3848 } 3849 3850 err_unlock: 3851 if (dmabuf_revocable) 3852 ib_umem_dmabuf_revoke_unlock(umem_dmabuf); 3853 3854 return ret ? ERR_PTR(ret) : NULL; 3855 } 3856 3857 /** 3858 * irdma_reg_phys_mr - register kernel physical memory 3859 * @pd: ibpd pointer 3860 * @addr: physical address of memory to register 3861 * @size: size of memory to register 3862 * @access: Access rights 3863 * @iova_start: start of virtual address for physical buffers 3864 * @dma_mr: Flag indicating whether this region is a PD DMA MR 3865 */ 3866 struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, 3867 u64 *iova_start, bool dma_mr) 3868 { 3869 struct irdma_device *iwdev = to_iwdev(pd->device); 3870 struct irdma_pbl *iwpbl; 3871 struct irdma_mr *iwmr; 3872 u32 stag; 3873 int ret; 3874 3875 iwmr = kzalloc_obj(*iwmr); 3876 if (!iwmr) 3877 return ERR_PTR(-ENOMEM); 3878 3879 iwmr->ibmr.pd = pd; 3880 iwmr->ibmr.device = pd->device; 3881 iwpbl = &iwmr->iwpbl; 3882 iwpbl->iwmr = iwmr; 3883 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 3884 iwmr->dma_mr = dma_mr; 3885 iwpbl->user_base = *iova_start; 3886 stag = irdma_create_stag(iwdev); 3887 if (!stag) { 3888 ret = -ENOMEM; 3889 goto err; 3890 } 3891 3892 iwmr->stag = stag; 3893 iwmr->ibmr.iova = *iova_start; 3894 iwmr->ibmr.rkey = stag; 3895 iwmr->ibmr.lkey = stag; 3896 iwmr->page_cnt = 1; 3897 iwmr->pgaddrmem[0] = addr; 3898 iwmr->len = size; 3899 iwmr->page_size = SZ_4K; 3900 ret = irdma_hwreg_mr(iwdev, iwmr, access); 3901 if (ret) { 3902 irdma_free_stag(iwdev, stag); 3903 goto err; 3904 } 3905 3906 return &iwmr->ibmr; 3907 3908 err: 3909 kfree(iwmr); 3910 3911 return ERR_PTR(ret); 3912 } 3913 3914 /** 3915 * irdma_get_dma_mr - register physical mem 3916 * @pd: ptr of pd 3917 * @acc: access for memory 3918 */ 3919 static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) 3920 { 3921 u64 kva = 0; 3922 3923 return irdma_reg_phys_mr(pd, 0, 0, acc, &kva, true); 3924 } 3925 3926 /** 3927 * irdma_del_memlist - Deleting pbl list entries for CQ/QP 3928 * @iwmr: iwmr for IB's user page addresses 3929 * @ucontext: ptr to user context 3930 */ 3931 static void irdma_del_memlist(struct irdma_mr *iwmr, 3932 struct irdma_ucontext *ucontext) 3933 { 3934 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3935 unsigned long flags; 3936 3937 switch (iwmr->type) { 3938 case IRDMA_MEMREG_TYPE_CQ: 3939 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 3940 if (iwpbl->on_list) { 3941 iwpbl->on_list = false; 3942 list_del(&iwpbl->list); 3943 } 3944 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 3945 break; 3946 case IRDMA_MEMREG_TYPE_QP: 3947 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 3948 if (iwpbl->on_list) { 3949 iwpbl->on_list = false; 3950 list_del(&iwpbl->list); 3951 } 3952 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 3953 break; 3954 case IRDMA_MEMREG_TYPE_SRQ: 3955 spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); 3956 if (iwpbl->on_list) { 3957 iwpbl->on_list = false; 3958 list_del(&iwpbl->list); 3959 } 3960 spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); 3961 break; 3962 default: 3963 break; 3964 } 3965 } 3966 3967 /** 3968 * irdma_dereg_mr - deregister mr 3969 * @ib_mr: mr ptr for dereg 3970 * @udata: user data 3971 */ 3972 static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3973 { 3974 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3975 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3976 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3977 bool dmabuf_revocable = iwmr->region && iwmr->region->is_dmabuf; 3978 int ret; 3979 3980 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 3981 if (iwmr->region) { 3982 struct irdma_ucontext *ucontext; 3983 3984 ucontext = rdma_udata_to_drv_context(udata, 3985 struct irdma_ucontext, 3986 ibucontext); 3987 irdma_del_memlist(iwmr, ucontext); 3988 } 3989 goto done; 3990 } 3991 3992 if (!dmabuf_revocable) { 3993 ret = irdma_hwdereg_mr(ib_mr); 3994 if (ret) 3995 return ret; 3996 3997 irdma_free_stag(iwdev, iwmr->stag); 3998 } 3999 done: 4000 if (iwmr->region) 4001 /* For dmabuf MRs, ib_umem_release will trigger a synchronous 4002 * call to the revoke callback which will perform the actual HW 4003 * invalidation via irdma_hwdereg_mr. We rely on this for its 4004 * implicit serialization w.r.t. concurrent revocations. This 4005 * must be done before freeing the PBLEs. 4006 */ 4007 ib_umem_release(iwmr->region); 4008 4009 if (iwpbl->pbl_allocated) 4010 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 4011 4012 if (dmabuf_revocable) 4013 irdma_free_stag(iwdev, iwmr->stag); 4014 4015 kfree(iwmr); 4016 4017 return 0; 4018 } 4019 4020 /** 4021 * irdma_post_send - kernel application wr 4022 * @ibqp: qp ptr for wr 4023 * @ib_wr: work request ptr 4024 * @bad_wr: return of bad wr if err 4025 */ 4026 static int irdma_post_send(struct ib_qp *ibqp, 4027 const struct ib_send_wr *ib_wr, 4028 const struct ib_send_wr **bad_wr) 4029 { 4030 struct irdma_qp *iwqp; 4031 struct irdma_qp_uk *ukqp; 4032 struct irdma_sc_dev *dev; 4033 struct irdma_post_sq_info info; 4034 int err = 0; 4035 unsigned long flags; 4036 bool inv_stag; 4037 struct irdma_ah *ah; 4038 4039 iwqp = to_iwqp(ibqp); 4040 ukqp = &iwqp->sc_qp.qp_uk; 4041 dev = &iwqp->iwdev->rf->sc_dev; 4042 4043 spin_lock_irqsave(&iwqp->lock, flags); 4044 while (ib_wr) { 4045 memset(&info, 0, sizeof(info)); 4046 inv_stag = false; 4047 info.wr_id = (ib_wr->wr_id); 4048 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) 4049 info.signaled = true; 4050 if (ib_wr->send_flags & IB_SEND_FENCE) 4051 info.read_fence = true; 4052 switch (ib_wr->opcode) { 4053 case IB_WR_ATOMIC_CMP_AND_SWP: 4054 if (unlikely(!(dev->hw_attrs.uk_attrs.feature_flags & 4055 IRDMA_FEATURE_ATOMIC_OPS))) { 4056 err = -EINVAL; 4057 break; 4058 } 4059 info.op_type = IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP; 4060 info.op.atomic_compare_swap.tagged_offset = ib_wr->sg_list[0].addr; 4061 info.op.atomic_compare_swap.remote_tagged_offset = 4062 atomic_wr(ib_wr)->remote_addr; 4063 info.op.atomic_compare_swap.swap_data_bytes = atomic_wr(ib_wr)->swap; 4064 info.op.atomic_compare_swap.compare_data_bytes = 4065 atomic_wr(ib_wr)->compare_add; 4066 info.op.atomic_compare_swap.stag = ib_wr->sg_list[0].lkey; 4067 info.op.atomic_compare_swap.remote_stag = atomic_wr(ib_wr)->rkey; 4068 err = irdma_uk_atomic_compare_swap(ukqp, &info, false); 4069 break; 4070 case IB_WR_ATOMIC_FETCH_AND_ADD: 4071 if (unlikely(!(dev->hw_attrs.uk_attrs.feature_flags & 4072 IRDMA_FEATURE_ATOMIC_OPS))) { 4073 err = -EINVAL; 4074 break; 4075 } 4076 info.op_type = IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD; 4077 info.op.atomic_fetch_add.tagged_offset = ib_wr->sg_list[0].addr; 4078 info.op.atomic_fetch_add.remote_tagged_offset = 4079 atomic_wr(ib_wr)->remote_addr; 4080 info.op.atomic_fetch_add.fetch_add_data_bytes = 4081 atomic_wr(ib_wr)->compare_add; 4082 info.op.atomic_fetch_add.stag = ib_wr->sg_list[0].lkey; 4083 info.op.atomic_fetch_add.remote_stag = 4084 atomic_wr(ib_wr)->rkey; 4085 err = irdma_uk_atomic_fetch_add(ukqp, &info, false); 4086 break; 4087 case IB_WR_SEND_WITH_IMM: 4088 if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { 4089 info.imm_data_valid = true; 4090 info.imm_data = ntohl(ib_wr->ex.imm_data); 4091 } else { 4092 err = -EINVAL; 4093 break; 4094 } 4095 fallthrough; 4096 case IB_WR_SEND: 4097 case IB_WR_SEND_WITH_INV: 4098 if (ib_wr->opcode == IB_WR_SEND || 4099 ib_wr->opcode == IB_WR_SEND_WITH_IMM) { 4100 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4101 info.op_type = IRDMA_OP_TYPE_SEND_SOL; 4102 else 4103 info.op_type = IRDMA_OP_TYPE_SEND; 4104 } else { 4105 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4106 info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; 4107 else 4108 info.op_type = IRDMA_OP_TYPE_SEND_INV; 4109 info.stag_to_inv = ib_wr->ex.invalidate_rkey; 4110 } 4111 4112 info.op.send.num_sges = ib_wr->num_sge; 4113 info.op.send.sg_list = ib_wr->sg_list; 4114 if (iwqp->ibqp.qp_type == IB_QPT_UD || 4115 iwqp->ibqp.qp_type == IB_QPT_GSI) { 4116 ah = to_iwah(ud_wr(ib_wr)->ah); 4117 info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; 4118 info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; 4119 info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; 4120 } 4121 4122 if (ib_wr->send_flags & IB_SEND_INLINE) 4123 err = irdma_uk_inline_send(ukqp, &info, false); 4124 else 4125 err = irdma_uk_send(ukqp, &info, false); 4126 break; 4127 case IB_WR_RDMA_WRITE_WITH_IMM: 4128 if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { 4129 info.imm_data_valid = true; 4130 info.imm_data = ntohl(ib_wr->ex.imm_data); 4131 } else { 4132 err = -EINVAL; 4133 break; 4134 } 4135 fallthrough; 4136 case IB_WR_RDMA_WRITE: 4137 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4138 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; 4139 else 4140 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; 4141 4142 info.op.rdma_write.num_lo_sges = ib_wr->num_sge; 4143 info.op.rdma_write.lo_sg_list = ib_wr->sg_list; 4144 info.op.rdma_write.rem_addr.addr = 4145 rdma_wr(ib_wr)->remote_addr; 4146 info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 4147 if (ib_wr->send_flags & IB_SEND_INLINE) 4148 err = irdma_uk_inline_rdma_write(ukqp, &info, false); 4149 else 4150 err = irdma_uk_rdma_write(ukqp, &info, false); 4151 break; 4152 case IB_WR_RDMA_READ_WITH_INV: 4153 inv_stag = true; 4154 fallthrough; 4155 case IB_WR_RDMA_READ: 4156 if (ib_wr->num_sge > 4157 dev->hw_attrs.uk_attrs.max_hw_read_sges) { 4158 err = -EINVAL; 4159 break; 4160 } 4161 info.op_type = IRDMA_OP_TYPE_RDMA_READ; 4162 info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 4163 info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 4164 info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; 4165 info.op.rdma_read.num_lo_sges = ib_wr->num_sge; 4166 err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); 4167 break; 4168 case IB_WR_LOCAL_INV: 4169 info.op_type = IRDMA_OP_TYPE_INV_STAG; 4170 info.local_fence = true; 4171 info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; 4172 err = irdma_uk_stag_local_invalidate(ukqp, &info, true); 4173 break; 4174 case IB_WR_REG_MR: { 4175 struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); 4176 struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; 4177 struct irdma_fast_reg_stag_info stag_info = {}; 4178 4179 stag_info.signaled = info.signaled; 4180 stag_info.read_fence = info.read_fence; 4181 stag_info.access_rights = 4182 irdma_get_mr_access(reg_wr(ib_wr)->access, 4183 dev->hw_attrs.uk_attrs.hw_rev); 4184 stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; 4185 stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; 4186 stag_info.page_size = reg_wr(ib_wr)->mr->page_size; 4187 stag_info.wr_id = ib_wr->wr_id; 4188 stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; 4189 stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; 4190 stag_info.total_len = iwmr->ibmr.length; 4191 stag_info.reg_addr_pa = *palloc->level1.addr; 4192 stag_info.first_pm_pbl_index = palloc->level1.idx; 4193 stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; 4194 if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) 4195 stag_info.chunk_size = 1; 4196 err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, 4197 true); 4198 break; 4199 } 4200 default: 4201 err = -EINVAL; 4202 ibdev_dbg(&iwqp->iwdev->ibdev, 4203 "VERBS: upost_send bad opcode = 0x%x\n", 4204 ib_wr->opcode); 4205 break; 4206 } 4207 4208 if (err) 4209 break; 4210 ib_wr = ib_wr->next; 4211 } 4212 4213 if (!iwqp->flush_issued) { 4214 if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) 4215 irdma_uk_qp_post_wr(ukqp); 4216 spin_unlock_irqrestore(&iwqp->lock, flags); 4217 } else { 4218 spin_unlock_irqrestore(&iwqp->lock, flags); 4219 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 4220 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 4221 } 4222 4223 if (err) 4224 *bad_wr = ib_wr; 4225 4226 return err; 4227 } 4228 4229 /** 4230 * irdma_post_srq_recv - post receive wr for kernel application 4231 * @ibsrq: ib srq pointer 4232 * @ib_wr: work request for receive 4233 * @bad_wr: bad wr caused an error 4234 */ 4235 static int irdma_post_srq_recv(struct ib_srq *ibsrq, 4236 const struct ib_recv_wr *ib_wr, 4237 const struct ib_recv_wr **bad_wr) 4238 { 4239 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 4240 struct irdma_srq_uk *uksrq = &iwsrq->sc_srq.srq_uk; 4241 struct irdma_post_rq_info post_recv = {}; 4242 unsigned long flags; 4243 int err = 0; 4244 4245 spin_lock_irqsave(&iwsrq->lock, flags); 4246 while (ib_wr) { 4247 if (ib_wr->num_sge > uksrq->max_srq_frag_cnt) { 4248 err = -EINVAL; 4249 goto out; 4250 } 4251 post_recv.num_sges = ib_wr->num_sge; 4252 post_recv.wr_id = ib_wr->wr_id; 4253 post_recv.sg_list = ib_wr->sg_list; 4254 err = irdma_uk_srq_post_receive(uksrq, &post_recv); 4255 if (err) 4256 goto out; 4257 4258 ib_wr = ib_wr->next; 4259 } 4260 4261 out: 4262 spin_unlock_irqrestore(&iwsrq->lock, flags); 4263 4264 if (err) 4265 *bad_wr = ib_wr; 4266 4267 return err; 4268 } 4269 4270 /** 4271 * irdma_post_recv - post receive wr for kernel application 4272 * @ibqp: ib qp pointer 4273 * @ib_wr: work request for receive 4274 * @bad_wr: bad wr caused an error 4275 */ 4276 static int irdma_post_recv(struct ib_qp *ibqp, 4277 const struct ib_recv_wr *ib_wr, 4278 const struct ib_recv_wr **bad_wr) 4279 { 4280 struct irdma_qp *iwqp; 4281 struct irdma_qp_uk *ukqp; 4282 struct irdma_post_rq_info post_recv = {}; 4283 unsigned long flags; 4284 int err = 0; 4285 4286 iwqp = to_iwqp(ibqp); 4287 ukqp = &iwqp->sc_qp.qp_uk; 4288 4289 if (ukqp->srq_uk) { 4290 *bad_wr = ib_wr; 4291 return -EINVAL; 4292 } 4293 4294 spin_lock_irqsave(&iwqp->lock, flags); 4295 while (ib_wr) { 4296 post_recv.num_sges = ib_wr->num_sge; 4297 post_recv.wr_id = ib_wr->wr_id; 4298 post_recv.sg_list = ib_wr->sg_list; 4299 err = irdma_uk_post_receive(ukqp, &post_recv); 4300 if (err) { 4301 ibdev_dbg(&iwqp->iwdev->ibdev, 4302 "VERBS: post_recv err %d\n", err); 4303 goto out; 4304 } 4305 4306 ib_wr = ib_wr->next; 4307 } 4308 4309 out: 4310 spin_unlock_irqrestore(&iwqp->lock, flags); 4311 if (iwqp->flush_issued) 4312 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 4313 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 4314 4315 if (err) 4316 *bad_wr = ib_wr; 4317 4318 return err; 4319 } 4320 4321 /** 4322 * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status 4323 * @opcode: iwarp flush code 4324 */ 4325 static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) 4326 { 4327 switch (opcode) { 4328 case FLUSH_PROT_ERR: 4329 return IB_WC_LOC_PROT_ERR; 4330 case FLUSH_REM_ACCESS_ERR: 4331 return IB_WC_REM_ACCESS_ERR; 4332 case FLUSH_LOC_QP_OP_ERR: 4333 return IB_WC_LOC_QP_OP_ERR; 4334 case FLUSH_REM_OP_ERR: 4335 return IB_WC_REM_OP_ERR; 4336 case FLUSH_LOC_LEN_ERR: 4337 return IB_WC_LOC_LEN_ERR; 4338 case FLUSH_GENERAL_ERR: 4339 return IB_WC_WR_FLUSH_ERR; 4340 case FLUSH_RETRY_EXC_ERR: 4341 return IB_WC_RETRY_EXC_ERR; 4342 case FLUSH_MW_BIND_ERR: 4343 return IB_WC_MW_BIND_ERR; 4344 case FLUSH_REM_INV_REQ_ERR: 4345 return IB_WC_REM_INV_REQ_ERR; 4346 case FLUSH_RNR_RETRY_EXC_ERR: 4347 return IB_WC_RNR_RETRY_EXC_ERR; 4348 case FLUSH_FATAL_ERR: 4349 default: 4350 return IB_WC_FATAL_ERR; 4351 } 4352 } 4353 4354 /** 4355 * irdma_process_cqe - process cqe info 4356 * @entry: processed cqe 4357 * @cq_poll_info: cqe info 4358 */ 4359 static void irdma_process_cqe(struct ib_wc *entry, 4360 struct irdma_cq_poll_info *cq_poll_info) 4361 { 4362 struct irdma_sc_qp *qp; 4363 4364 entry->wc_flags = 0; 4365 entry->pkey_index = 0; 4366 entry->wr_id = cq_poll_info->wr_id; 4367 4368 qp = cq_poll_info->qp_handle; 4369 entry->qp = qp->qp_uk.back_qp; 4370 4371 if (cq_poll_info->error) { 4372 entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? 4373 irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; 4374 4375 entry->vendor_err = cq_poll_info->major_err << 16 | 4376 cq_poll_info->minor_err; 4377 } else { 4378 entry->status = IB_WC_SUCCESS; 4379 if (cq_poll_info->imm_valid) { 4380 entry->ex.imm_data = htonl(cq_poll_info->imm_data); 4381 entry->wc_flags |= IB_WC_WITH_IMM; 4382 } 4383 if (cq_poll_info->ud_smac_valid) { 4384 ether_addr_copy(entry->smac, cq_poll_info->ud_smac); 4385 entry->wc_flags |= IB_WC_WITH_SMAC; 4386 } 4387 4388 if (cq_poll_info->ud_vlan_valid) { 4389 u16 vlan = cq_poll_info->ud_vlan & VLAN_VID_MASK; 4390 4391 entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; 4392 if (vlan) { 4393 entry->vlan_id = vlan; 4394 entry->wc_flags |= IB_WC_WITH_VLAN; 4395 } 4396 } else { 4397 entry->sl = 0; 4398 } 4399 } 4400 4401 if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) { 4402 set_ib_wc_op_sq(cq_poll_info, entry); 4403 } else { 4404 if (qp->dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) 4405 set_ib_wc_op_rq(cq_poll_info, entry, 4406 qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM ? 4407 true : false); 4408 else 4409 set_ib_wc_op_rq_gen_3(cq_poll_info, entry); 4410 if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && 4411 cq_poll_info->stag_invalid_set) { 4412 entry->ex.invalidate_rkey = cq_poll_info->inv_stag; 4413 entry->wc_flags |= IB_WC_WITH_INVALIDATE; 4414 } 4415 } 4416 4417 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { 4418 entry->src_qp = cq_poll_info->ud_src_qpn; 4419 entry->slid = 0; 4420 entry->wc_flags |= 4421 (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); 4422 entry->network_hdr_type = cq_poll_info->ipv4 ? 4423 RDMA_NETWORK_IPV4 : 4424 RDMA_NETWORK_IPV6; 4425 } else { 4426 entry->src_qp = cq_poll_info->qp_id; 4427 } 4428 4429 entry->byte_len = cq_poll_info->bytes_xfered; 4430 } 4431 4432 /** 4433 * irdma_poll_one - poll one entry of the CQ 4434 * @ukcq: ukcq to poll 4435 * @cur_cqe: current CQE info to be filled in 4436 * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ 4437 * 4438 * Returns the internal irdma device error code or 0 on success 4439 */ 4440 static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, 4441 struct irdma_cq_poll_info *cur_cqe, 4442 struct ib_wc *entry) 4443 { 4444 int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); 4445 4446 if (ret) 4447 return ret; 4448 4449 irdma_process_cqe(entry, cur_cqe); 4450 4451 return 0; 4452 } 4453 4454 /** 4455 * __irdma_poll_cq - poll cq for completion (kernel apps) 4456 * @iwcq: cq to poll 4457 * @num_entries: number of entries to poll 4458 * @entry: wr of a completed entry 4459 */ 4460 static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) 4461 { 4462 struct list_head *tmp_node, *list_node; 4463 struct irdma_cq_buf *last_buf = NULL; 4464 struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; 4465 struct irdma_cq_buf *cq_buf; 4466 int ret; 4467 struct irdma_device *iwdev; 4468 struct irdma_cq_uk *ukcq; 4469 bool cq_new_cqe = false; 4470 int resized_bufs = 0; 4471 int npolled = 0; 4472 4473 iwdev = to_iwdev(iwcq->ibcq.device); 4474 ukcq = &iwcq->sc_cq.cq_uk; 4475 4476 /* go through the list of previously resized CQ buffers */ 4477 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 4478 cq_buf = container_of(list_node, struct irdma_cq_buf, list); 4479 while (npolled < num_entries) { 4480 ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); 4481 if (!ret) { 4482 ++npolled; 4483 cq_new_cqe = true; 4484 continue; 4485 } 4486 if (ret == -ENOENT) 4487 break; 4488 /* QP using the CQ is destroyed. Skip reporting this CQE */ 4489 if (ret == -EFAULT) { 4490 cq_new_cqe = true; 4491 continue; 4492 } 4493 goto error; 4494 } 4495 4496 /* save the resized CQ buffer which received the last cqe */ 4497 if (cq_new_cqe) 4498 last_buf = cq_buf; 4499 cq_new_cqe = false; 4500 } 4501 4502 /* check the current CQ for new cqes */ 4503 while (npolled < num_entries) { 4504 ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); 4505 if (ret == -ENOENT) { 4506 ret = irdma_generated_cmpls(iwcq, cur_cqe); 4507 if (!ret) 4508 irdma_process_cqe(entry + npolled, cur_cqe); 4509 } 4510 if (!ret) { 4511 ++npolled; 4512 cq_new_cqe = true; 4513 continue; 4514 } 4515 4516 if (ret == -ENOENT) 4517 break; 4518 /* QP using the CQ is destroyed. Skip reporting this CQE */ 4519 if (ret == -EFAULT) { 4520 cq_new_cqe = true; 4521 continue; 4522 } 4523 goto error; 4524 } 4525 4526 if (cq_new_cqe) 4527 /* all previous CQ resizes are complete */ 4528 resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); 4529 else if (last_buf) 4530 /* only CQ resizes up to the last_buf are complete */ 4531 resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); 4532 if (resized_bufs) 4533 /* report to the HW the number of complete CQ resizes */ 4534 irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); 4535 4536 return npolled; 4537 error: 4538 ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n", 4539 __func__, ret); 4540 4541 return ret; 4542 } 4543 4544 /** 4545 * irdma_poll_cq - poll cq for completion (kernel apps) 4546 * @ibcq: cq to poll 4547 * @num_entries: number of entries to poll 4548 * @entry: wr of a completed entry 4549 */ 4550 static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, 4551 struct ib_wc *entry) 4552 { 4553 struct irdma_cq *iwcq; 4554 unsigned long flags; 4555 int ret; 4556 4557 iwcq = to_iwcq(ibcq); 4558 4559 spin_lock_irqsave(&iwcq->lock, flags); 4560 ret = __irdma_poll_cq(iwcq, num_entries, entry); 4561 spin_unlock_irqrestore(&iwcq->lock, flags); 4562 4563 return ret; 4564 } 4565 4566 /** 4567 * irdma_req_notify_cq - arm cq kernel application 4568 * @ibcq: cq to arm 4569 * @notify_flags: notofication flags 4570 */ 4571 static int irdma_req_notify_cq(struct ib_cq *ibcq, 4572 enum ib_cq_notify_flags notify_flags) 4573 { 4574 struct irdma_cq *iwcq; 4575 struct irdma_cq_uk *ukcq; 4576 unsigned long flags; 4577 enum irdma_cmpl_notify cq_notify; 4578 bool promo_event = false; 4579 int ret = 0; 4580 4581 cq_notify = notify_flags == IB_CQ_SOLICITED ? 4582 IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT; 4583 iwcq = to_iwcq(ibcq); 4584 ukcq = &iwcq->sc_cq.cq_uk; 4585 4586 spin_lock_irqsave(&iwcq->lock, flags); 4587 /* Only promote to arm the CQ for any event if the last arm event was solicited. */ 4588 if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED && notify_flags != IB_CQ_SOLICITED) 4589 promo_event = true; 4590 4591 if (!atomic_cmpxchg(&iwcq->armed, 0, 1) || promo_event) { 4592 iwcq->last_notify = cq_notify; 4593 irdma_uk_cq_request_notification(ukcq, cq_notify); 4594 } 4595 4596 if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && 4597 (!irdma_uk_cq_empty(ukcq) || !list_empty(&iwcq->cmpl_generated))) 4598 ret = 1; 4599 spin_unlock_irqrestore(&iwcq->lock, flags); 4600 4601 return ret; 4602 } 4603 4604 static const struct rdma_stat_desc irdma_hw_stat_descs[] = { 4605 /* gen1 - 32-bit */ 4606 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards", 4607 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts", 4608 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes", 4609 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards", 4610 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts", 4611 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes", 4612 [IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors", 4613 /* gen1 - 64-bit */ 4614 [IRDMA_HW_STAT_INDEX_IP4RXOCTS].name = "ip4InOctets", 4615 [IRDMA_HW_STAT_INDEX_IP4RXPKTS].name = "ip4InPkts", 4616 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS].name = "ip4InReasmRqd", 4617 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS].name = "ip4InMcastPkts", 4618 [IRDMA_HW_STAT_INDEX_IP4TXOCTS].name = "ip4OutOctets", 4619 [IRDMA_HW_STAT_INDEX_IP4TXPKTS].name = "ip4OutPkts", 4620 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS].name = "ip4OutSegRqd", 4621 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS].name = "ip4OutMcastPkts", 4622 [IRDMA_HW_STAT_INDEX_IP6RXOCTS].name = "ip6InOctets", 4623 [IRDMA_HW_STAT_INDEX_IP6RXPKTS].name = "ip6InPkts", 4624 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS].name = "ip6InReasmRqd", 4625 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS].name = "ip6InMcastPkts", 4626 [IRDMA_HW_STAT_INDEX_IP6TXOCTS].name = "ip6OutOctets", 4627 [IRDMA_HW_STAT_INDEX_IP6TXPKTS].name = "ip6OutPkts", 4628 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS].name = "ip6OutSegRqd", 4629 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS].name = "ip6OutMcastPkts", 4630 [IRDMA_HW_STAT_INDEX_RDMARXRDS].name = "InRdmaReads", 4631 [IRDMA_HW_STAT_INDEX_RDMARXSNDS].name = "InRdmaSends", 4632 [IRDMA_HW_STAT_INDEX_RDMARXWRS].name = "InRdmaWrites", 4633 [IRDMA_HW_STAT_INDEX_RDMATXRDS].name = "OutRdmaReads", 4634 [IRDMA_HW_STAT_INDEX_RDMATXSNDS].name = "OutRdmaSends", 4635 [IRDMA_HW_STAT_INDEX_RDMATXWRS].name = "OutRdmaWrites", 4636 [IRDMA_HW_STAT_INDEX_RDMAVBND].name = "RdmaBnd", 4637 [IRDMA_HW_STAT_INDEX_RDMAVINV].name = "RdmaInv", 4638 4639 /* gen2 - 32-bit */ 4640 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled", 4641 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored", 4642 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent", 4643 /* gen2 - 64-bit */ 4644 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS].name = "ip4InMcastOctets", 4645 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS].name = "ip4OutMcastOctets", 4646 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS].name = "ip6InMcastOctets", 4647 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS].name = "ip6OutMcastOctets", 4648 [IRDMA_HW_STAT_INDEX_UDPRXPKTS].name = "RxUDP", 4649 [IRDMA_HW_STAT_INDEX_UDPTXPKTS].name = "TxUDP", 4650 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS].name = "RxECNMrkd", 4651 [IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "RetransSegs", 4652 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "InOptErrors", 4653 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "InProtoErrors", 4654 [IRDMA_HW_STAT_INDEX_TCPRXSEGS].name = "InSegs", 4655 [IRDMA_HW_STAT_INDEX_TCPTXSEG].name = "OutSegs", 4656 4657 /* gen3 */ 4658 [IRDMA_HW_STAT_INDEX_RNR_SENT].name = "RNR sent", 4659 [IRDMA_HW_STAT_INDEX_RNR_RCVD].name = "RNR received", 4660 [IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT].name = "ord limit count", 4661 [IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT].name = "ird limit count", 4662 [IRDMA_HW_STAT_INDEX_RDMARXATS].name = "Rx atomics", 4663 [IRDMA_HW_STAT_INDEX_RDMATXATS].name = "Tx atomics", 4664 [IRDMA_HW_STAT_INDEX_NAKSEQERR].name = "Nak Sequence Error", 4665 [IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED].name = "Nak Sequence Error Implied", 4666 [IRDMA_HW_STAT_INDEX_RTO].name = "RTO", 4667 [IRDMA_HW_STAT_INDEX_RXOOOPKTS].name = "Rcvd Out of order packets", 4668 [IRDMA_HW_STAT_INDEX_ICRCERR].name = "CRC errors", 4669 }; 4670 4671 static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, 4672 struct ib_port_immutable *immutable) 4673 { 4674 struct ib_port_attr attr; 4675 int err; 4676 4677 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 4678 err = ib_query_port(ibdev, port_num, &attr); 4679 if (err) 4680 return err; 4681 4682 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 4683 immutable->pkey_tbl_len = attr.pkey_tbl_len; 4684 immutable->gid_tbl_len = attr.gid_tbl_len; 4685 4686 return 0; 4687 } 4688 4689 static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num, 4690 struct ib_port_immutable *immutable) 4691 { 4692 struct ib_port_attr attr; 4693 int err; 4694 4695 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 4696 err = ib_query_port(ibdev, port_num, &attr); 4697 if (err) 4698 return err; 4699 immutable->gid_tbl_len = attr.gid_tbl_len; 4700 4701 return 0; 4702 } 4703 4704 static void irdma_get_dev_fw_str(struct ib_device *dev, char *str) 4705 { 4706 struct irdma_device *iwdev = to_iwdev(dev); 4707 4708 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", 4709 irdma_fw_major_ver(&iwdev->rf->sc_dev), 4710 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 4711 } 4712 4713 /** 4714 * irdma_alloc_hw_port_stats - Allocate a hw stats structure 4715 * @ibdev: device pointer from stack 4716 * @port_num: port number 4717 */ 4718 static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev, 4719 u32 port_num) 4720 { 4721 struct irdma_device *iwdev = to_iwdev(ibdev); 4722 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 4723 4724 int num_counters = dev->hw_attrs.max_stat_idx; 4725 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 4726 4727 return rdma_alloc_hw_stats_struct(irdma_hw_stat_descs, num_counters, 4728 lifespan); 4729 } 4730 4731 /** 4732 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 4733 * @ibdev: device pointer from stack 4734 * @stats: stats pointer from stack 4735 * @port_num: port number 4736 * @index: which hw counter the stack is requesting we update 4737 */ 4738 static int irdma_get_hw_stats(struct ib_device *ibdev, 4739 struct rdma_hw_stats *stats, u32 port_num, 4740 int index) 4741 { 4742 struct irdma_device *iwdev = to_iwdev(ibdev); 4743 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 4744 4745 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 4746 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 4747 else 4748 irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat); 4749 4750 memcpy(&stats->value[0], hw_stats, sizeof(u64) * stats->num_counters); 4751 4752 return stats->num_counters; 4753 } 4754 4755 /** 4756 * irdma_query_gid - Query port GID 4757 * @ibdev: device pointer from stack 4758 * @port: port number 4759 * @index: Entry index 4760 * @gid: Global ID 4761 */ 4762 static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index, 4763 union ib_gid *gid) 4764 { 4765 struct irdma_device *iwdev = to_iwdev(ibdev); 4766 4767 memset(gid->raw, 0, sizeof(gid->raw)); 4768 ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); 4769 4770 return 0; 4771 } 4772 4773 /** 4774 * mcast_list_add - Add a new mcast item to list 4775 * @rf: RDMA PCI function 4776 * @new_elem: pointer to element to add 4777 */ 4778 static void mcast_list_add(struct irdma_pci_f *rf, 4779 struct mc_table_list *new_elem) 4780 { 4781 list_add(&new_elem->list, &rf->mc_qht_list.list); 4782 } 4783 4784 /** 4785 * mcast_list_del - Remove an mcast item from list 4786 * @mc_qht_elem: pointer to mcast table list element 4787 */ 4788 static void mcast_list_del(struct mc_table_list *mc_qht_elem) 4789 { 4790 if (mc_qht_elem) 4791 list_del(&mc_qht_elem->list); 4792 } 4793 4794 /** 4795 * mcast_list_lookup_ip - Search mcast list for address 4796 * @rf: RDMA PCI function 4797 * @ip_mcast: pointer to mcast IP address 4798 */ 4799 static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf, 4800 u32 *ip_mcast) 4801 { 4802 struct mc_table_list *mc_qht_el; 4803 struct list_head *pos, *q; 4804 4805 list_for_each_safe (pos, q, &rf->mc_qht_list.list) { 4806 mc_qht_el = list_entry(pos, struct mc_table_list, list); 4807 if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, 4808 sizeof(mc_qht_el->mc_info.dest_ip))) 4809 return mc_qht_el; 4810 } 4811 4812 return NULL; 4813 } 4814 4815 /** 4816 * irdma_mcast_cqp_op - perform a mcast cqp operation 4817 * @iwdev: irdma device 4818 * @mc_grp_ctx: mcast group info 4819 * @op: operation 4820 * 4821 * returns error status 4822 */ 4823 static int irdma_mcast_cqp_op(struct irdma_device *iwdev, 4824 struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) 4825 { 4826 struct cqp_cmds_info *cqp_info; 4827 struct irdma_cqp_request *cqp_request; 4828 int status; 4829 4830 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 4831 if (!cqp_request) 4832 return -ENOMEM; 4833 4834 cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; 4835 cqp_info = &cqp_request->info; 4836 cqp_info->cqp_cmd = op; 4837 cqp_info->post_sq = 1; 4838 cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; 4839 cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; 4840 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 4841 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 4842 4843 return status; 4844 } 4845 4846 /** 4847 * irdma_mcast_mac - Get the multicast MAC for an IP address 4848 * @ip_addr: IPv4 or IPv6 address 4849 * @mac: pointer to result MAC address 4850 * @ipv4: flag indicating IPv4 or IPv6 4851 * 4852 */ 4853 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4) 4854 { 4855 u8 *ip = (u8 *)ip_addr; 4856 4857 if (ipv4) { 4858 unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00, 4859 0x00, 0x00}; 4860 4861 mac4[3] = ip[2] & 0x7F; 4862 mac4[4] = ip[1]; 4863 mac4[5] = ip[0]; 4864 ether_addr_copy(mac, mac4); 4865 } else { 4866 unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00, 4867 0x00, 0x00}; 4868 4869 mac6[2] = ip[3]; 4870 mac6[3] = ip[2]; 4871 mac6[4] = ip[1]; 4872 mac6[5] = ip[0]; 4873 ether_addr_copy(mac, mac6); 4874 } 4875 } 4876 4877 /** 4878 * irdma_attach_mcast - attach a qp to a multicast group 4879 * @ibqp: ptr to qp 4880 * @ibgid: pointer to global ID 4881 * @lid: local ID 4882 * 4883 * returns error status 4884 */ 4885 static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 4886 { 4887 struct irdma_qp *iwqp = to_iwqp(ibqp); 4888 struct irdma_device *iwdev = iwqp->iwdev; 4889 struct irdma_pci_f *rf = iwdev->rf; 4890 struct mc_table_list *mc_qht_elem; 4891 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 4892 unsigned long flags; 4893 u32 ip_addr[4] = {}; 4894 u32 mgn; 4895 u32 no_mgs; 4896 int ret = 0; 4897 bool ipv4; 4898 u16 vlan_id; 4899 union irdma_sockaddr sgid_addr; 4900 unsigned char dmac[ETH_ALEN]; 4901 4902 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 4903 4904 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { 4905 irdma_copy_ip_ntohl(ip_addr, 4906 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4907 irdma_get_vlan_mac_ipv6(ip_addr, &vlan_id, NULL); 4908 ipv4 = false; 4909 ibdev_dbg(&iwdev->ibdev, 4910 "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, 4911 ip_addr); 4912 irdma_mcast_mac(ip_addr, dmac, false); 4913 } else { 4914 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4915 ipv4 = true; 4916 vlan_id = irdma_get_vlan_ipv4(ip_addr); 4917 irdma_mcast_mac(ip_addr, dmac, true); 4918 ibdev_dbg(&iwdev->ibdev, 4919 "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n", 4920 ibqp->qp_num, ip_addr, dmac); 4921 } 4922 4923 spin_lock_irqsave(&rf->qh_list_lock, flags); 4924 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 4925 if (!mc_qht_elem) { 4926 struct irdma_dma_mem *dma_mem_mc; 4927 4928 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4929 mc_qht_elem = kzalloc_obj(*mc_qht_elem); 4930 if (!mc_qht_elem) 4931 return -ENOMEM; 4932 4933 mc_qht_elem->mc_info.ipv4_valid = ipv4; 4934 memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, 4935 sizeof(mc_qht_elem->mc_info.dest_ip)); 4936 ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, 4937 &mgn, &rf->next_mcg); 4938 if (ret) { 4939 kfree(mc_qht_elem); 4940 return -ENOMEM; 4941 } 4942 4943 mc_qht_elem->mc_info.mgn = mgn; 4944 dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; 4945 dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX, 4946 IRDMA_HW_PAGE_SIZE); 4947 dma_mem_mc->va = dma_alloc_coherent(rf->hw.device, 4948 dma_mem_mc->size, 4949 &dma_mem_mc->pa, 4950 GFP_KERNEL); 4951 if (!dma_mem_mc->va) { 4952 irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); 4953 kfree(mc_qht_elem); 4954 return -ENOMEM; 4955 } 4956 4957 mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; 4958 memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, 4959 sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); 4960 mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; 4961 mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; 4962 if (vlan_id < VLAN_N_VID) 4963 mc_qht_elem->mc_grp_ctx.vlan_valid = true; 4964 mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id; 4965 mc_qht_elem->mc_grp_ctx.qs_handle = 4966 iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; 4967 ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); 4968 4969 spin_lock_irqsave(&rf->qh_list_lock, flags); 4970 mcast_list_add(rf, mc_qht_elem); 4971 } else { 4972 if (mc_qht_elem->mc_grp_ctx.no_of_mgs == 4973 IRDMA_MAX_MGS_PER_CTX) { 4974 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4975 return -ENOMEM; 4976 } 4977 } 4978 4979 mcg_info.qp_id = iwqp->ibqp.qp_num; 4980 no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; 4981 irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4982 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4983 4984 /* Only if there is a change do we need to modify or create */ 4985 if (!no_mgs) { 4986 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4987 IRDMA_OP_MC_CREATE); 4988 } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4989 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4990 IRDMA_OP_MC_MODIFY); 4991 } else { 4992 return 0; 4993 } 4994 4995 if (ret) 4996 goto error; 4997 4998 return 0; 4999 5000 error: 5001 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 5002 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 5003 mcast_list_del(mc_qht_elem); 5004 dma_free_coherent(rf->hw.device, 5005 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 5006 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 5007 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 5008 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 5009 irdma_free_rsrc(rf, rf->allocated_mcgs, 5010 mc_qht_elem->mc_grp_ctx.mg_id); 5011 kfree(mc_qht_elem); 5012 } 5013 5014 return ret; 5015 } 5016 5017 /** 5018 * irdma_detach_mcast - detach a qp from a multicast group 5019 * @ibqp: ptr to qp 5020 * @ibgid: pointer to global ID 5021 * @lid: local ID 5022 * 5023 * returns error status 5024 */ 5025 static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 5026 { 5027 struct irdma_qp *iwqp = to_iwqp(ibqp); 5028 struct irdma_device *iwdev = iwqp->iwdev; 5029 struct irdma_pci_f *rf = iwdev->rf; 5030 u32 ip_addr[4] = {}; 5031 struct mc_table_list *mc_qht_elem; 5032 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 5033 int ret; 5034 unsigned long flags; 5035 union irdma_sockaddr sgid_addr; 5036 5037 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 5038 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) 5039 irdma_copy_ip_ntohl(ip_addr, 5040 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 5041 else 5042 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 5043 5044 spin_lock_irqsave(&rf->qh_list_lock, flags); 5045 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 5046 if (!mc_qht_elem) { 5047 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5048 ibdev_dbg(&iwdev->ibdev, 5049 "VERBS: address not found MCG\n"); 5050 return 0; 5051 } 5052 5053 mcg_info.qp_id = iwqp->ibqp.qp_num; 5054 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 5055 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 5056 mcast_list_del(mc_qht_elem); 5057 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5058 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 5059 IRDMA_OP_MC_DESTROY); 5060 if (ret) { 5061 ibdev_dbg(&iwdev->ibdev, 5062 "VERBS: failed MC_DESTROY MCG\n"); 5063 spin_lock_irqsave(&rf->qh_list_lock, flags); 5064 mcast_list_add(rf, mc_qht_elem); 5065 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5066 return -EAGAIN; 5067 } 5068 5069 dma_free_coherent(rf->hw.device, 5070 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 5071 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 5072 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 5073 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 5074 irdma_free_rsrc(rf, rf->allocated_mcgs, 5075 mc_qht_elem->mc_grp_ctx.mg_id); 5076 kfree(mc_qht_elem); 5077 } else { 5078 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5079 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 5080 IRDMA_OP_MC_MODIFY); 5081 if (ret) { 5082 ibdev_dbg(&iwdev->ibdev, 5083 "VERBS: failed Modify MCG\n"); 5084 return ret; 5085 } 5086 } 5087 5088 return 0; 5089 } 5090 5091 static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep) 5092 { 5093 struct irdma_pci_f *rf = iwdev->rf; 5094 int err; 5095 5096 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah->sc_ah.ah_info.ah_idx, 5097 &rf->next_ah); 5098 if (err) 5099 return err; 5100 5101 err = irdma_ah_cqp_op(rf, &ah->sc_ah, IRDMA_OP_AH_CREATE, sleep, 5102 irdma_gsi_ud_qp_ah_cb, &ah->sc_ah); 5103 5104 if (err) { 5105 ibdev_dbg(&iwdev->ibdev, "VERBS: CQP-OP Create AH fail"); 5106 goto err_ah_create; 5107 } 5108 5109 if (!sleep) { 5110 const u64 tmout_ms = irdma_get_timeout_threshold(&rf->sc_dev) * 5111 CQP_COMPL_WAIT_TIME_MS; 5112 5113 if (poll_timeout_us_atomic(irdma_cqp_ce_handler(rf, 5114 &rf->ccq.sc_cq), 5115 ah->sc_ah.ah_info.ah_valid, 1, 5116 tmout_ms * USEC_PER_MSEC, false)) { 5117 ibdev_dbg(&iwdev->ibdev, 5118 "VERBS: CQP create AH timed out"); 5119 err = -ETIMEDOUT; 5120 goto err_ah_create; 5121 } 5122 } 5123 return 0; 5124 5125 err_ah_create: 5126 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx); 5127 5128 return err; 5129 } 5130 5131 static int irdma_setup_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr) 5132 { 5133 struct irdma_pd *pd = to_iwpd(ibah->pd); 5134 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5135 struct rdma_ah_attr *ah_attr = attr->ah_attr; 5136 const struct ib_gid_attr *sgid_attr; 5137 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5138 struct irdma_pci_f *rf = iwdev->rf; 5139 struct irdma_sc_ah *sc_ah; 5140 struct irdma_ah_info *ah_info; 5141 union irdma_sockaddr sgid_addr, dgid_addr; 5142 int err; 5143 u8 dmac[ETH_ALEN]; 5144 5145 ah->pd = pd; 5146 sc_ah = &ah->sc_ah; 5147 sc_ah->ah_info.vsi = &iwdev->vsi; 5148 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 5149 ah->sgid_index = ah_attr->grh.sgid_index; 5150 sgid_attr = ah_attr->grh.sgid_attr; 5151 memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid)); 5152 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid_attr->gid); 5153 rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid); 5154 ah->av.attrs = *ah_attr; 5155 ah->av.net_type = rdma_gid_attr_network_type(sgid_attr); 5156 ah_info = &sc_ah->ah_info; 5157 ah_info->pd_idx = pd->sc_pd.pd_id; 5158 if (ah_attr->ah_flags & IB_AH_GRH) { 5159 ah_info->flow_label = ah_attr->grh.flow_label; 5160 ah_info->hop_ttl = ah_attr->grh.hop_limit; 5161 ah_info->tc_tos = ah_attr->grh.traffic_class; 5162 } 5163 5164 ether_addr_copy(dmac, ah_attr->roce.dmac); 5165 if (ah->av.net_type == RDMA_NETWORK_IPV4) { 5166 ah_info->ipv4_valid = true; 5167 ah_info->dest_ip_addr[0] = 5168 ntohl(dgid_addr.saddr_in.sin_addr.s_addr); 5169 ah_info->src_ip_addr[0] = 5170 ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 5171 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 5172 ah_info->dest_ip_addr[0]); 5173 if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) { 5174 ah_info->do_lpbk = true; 5175 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true); 5176 } 5177 } else { 5178 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 5179 dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 5180 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 5181 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 5182 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 5183 ah_info->dest_ip_addr); 5184 if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr)) { 5185 ah_info->do_lpbk = true; 5186 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false); 5187 } 5188 } 5189 5190 err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag, 5191 ah_info->mac_addr); 5192 if (err) 5193 return err; 5194 5195 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, 5196 ah_info->ipv4_valid, dmac); 5197 5198 if (ah_info->dst_arpindex == -1) 5199 return -EINVAL; 5200 5201 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) 5202 ah_info->vlan_tag = 0; 5203 5204 if (ah_info->vlan_tag < VLAN_N_VID) { 5205 u8 prio = rt_tos2priority(ah_info->tc_tos); 5206 5207 prio = irdma_roce_get_vlan_prio(sgid_attr, prio); 5208 5209 ah_info->vlan_tag |= (u16)prio << VLAN_PRIO_SHIFT; 5210 ah_info->insert_vlan_tag = true; 5211 } 5212 5213 return 0; 5214 } 5215 5216 /** 5217 * irdma_ah_exists - Check for existing identical AH 5218 * @iwdev: irdma device 5219 * @new_ah: AH to check for 5220 * 5221 * returns true if AH is found, false if not found. 5222 */ 5223 static bool irdma_ah_exists(struct irdma_device *iwdev, 5224 struct irdma_ah *new_ah) 5225 { 5226 struct irdma_ah *ah; 5227 u32 key = new_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 5228 new_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 5229 new_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 5230 new_ah->sc_ah.ah_info.dest_ip_addr[3]; 5231 5232 hash_for_each_possible(iwdev->rf->ah_hash_tbl, ah, list, key) { 5233 /* Set ah_valid and ah_id the same so memcmp can work */ 5234 new_ah->sc_ah.ah_info.ah_idx = ah->sc_ah.ah_info.ah_idx; 5235 new_ah->sc_ah.ah_info.ah_valid = ah->sc_ah.ah_info.ah_valid; 5236 if (!memcmp(&ah->sc_ah.ah_info, &new_ah->sc_ah.ah_info, 5237 sizeof(ah->sc_ah.ah_info))) { 5238 refcount_inc(&ah->refcnt); 5239 new_ah->parent_ah = ah; 5240 return true; 5241 } 5242 } 5243 5244 return false; 5245 } 5246 5247 /** 5248 * irdma_destroy_ah - Destroy address handle 5249 * @ibah: pointer to address handle 5250 * @ah_flags: flags for sleepable 5251 */ 5252 static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 5253 { 5254 struct irdma_device *iwdev = to_iwdev(ibah->device); 5255 struct irdma_ah *ah = to_iwah(ibah); 5256 5257 if ((ah_flags & RDMA_DESTROY_AH_SLEEPABLE) && ah->parent_ah) { 5258 mutex_lock(&iwdev->rf->ah_tbl_lock); 5259 if (!refcount_dec_and_test(&ah->parent_ah->refcnt)) { 5260 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5261 return 0; 5262 } 5263 hash_del(&ah->parent_ah->list); 5264 kfree(ah->parent_ah); 5265 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5266 } 5267 5268 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 5269 false, NULL, ah); 5270 5271 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 5272 ah->sc_ah.ah_info.ah_idx); 5273 5274 return 0; 5275 } 5276 5277 /** 5278 * irdma_create_user_ah - create user address handle 5279 * @ibah: address handle 5280 * @attr: address handle attributes 5281 * @udata: User data 5282 * 5283 * returns 0 on success, error otherwise 5284 */ 5285 static int irdma_create_user_ah(struct ib_ah *ibah, 5286 struct rdma_ah_init_attr *attr, 5287 struct ib_udata *udata) 5288 { 5289 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) 5290 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5291 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5292 struct irdma_create_ah_resp uresp = {}; 5293 struct irdma_ah *parent_ah; 5294 int err; 5295 5296 if (udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) 5297 return -EINVAL; 5298 5299 err = irdma_setup_ah(ibah, attr); 5300 if (err) 5301 return err; 5302 mutex_lock(&iwdev->rf->ah_tbl_lock); 5303 if (!irdma_ah_exists(iwdev, ah)) { 5304 err = irdma_create_hw_ah(iwdev, ah, true); 5305 if (err) { 5306 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5307 return err; 5308 } 5309 /* Add new AH to list */ 5310 parent_ah = kmemdup(ah, sizeof(*ah), GFP_KERNEL); 5311 if (parent_ah) { 5312 u32 key = parent_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 5313 parent_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 5314 parent_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 5315 parent_ah->sc_ah.ah_info.dest_ip_addr[3]; 5316 5317 ah->parent_ah = parent_ah; 5318 hash_add(iwdev->rf->ah_hash_tbl, &parent_ah->list, key); 5319 refcount_set(&parent_ah->refcnt, 1); 5320 } 5321 } 5322 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5323 5324 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 5325 err = ib_respond_udata(udata, uresp); 5326 if (err) 5327 irdma_destroy_ah(ibah, attr->flags); 5328 5329 return err; 5330 } 5331 5332 /** 5333 * irdma_create_ah - create address handle 5334 * @ibah: address handle 5335 * @attr: address handle attributes 5336 * @udata: NULL 5337 * 5338 * returns 0 on success, error otherwise 5339 */ 5340 static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr, 5341 struct ib_udata *udata) 5342 { 5343 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5344 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5345 int err; 5346 5347 err = irdma_setup_ah(ibah, attr); 5348 if (err) 5349 return err; 5350 err = irdma_create_hw_ah(iwdev, ah, attr->flags & RDMA_CREATE_AH_SLEEPABLE); 5351 5352 return err; 5353 } 5354 5355 /** 5356 * irdma_query_ah - Query address handle 5357 * @ibah: pointer to address handle 5358 * @ah_attr: address handle attributes 5359 */ 5360 static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 5361 { 5362 struct irdma_ah *ah = to_iwah(ibah); 5363 5364 memset(ah_attr, 0, sizeof(*ah_attr)); 5365 if (ah->av.attrs.ah_flags & IB_AH_GRH) { 5366 ah_attr->ah_flags = IB_AH_GRH; 5367 ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; 5368 ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; 5369 ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; 5370 ah_attr->grh.sgid_index = ah->sgid_index; 5371 memcpy(&ah_attr->grh.dgid, &ah->dgid, 5372 sizeof(ah_attr->grh.dgid)); 5373 } 5374 5375 return 0; 5376 } 5377 5378 static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, 5379 u32 port_num) 5380 { 5381 return IB_LINK_LAYER_ETHERNET; 5382 } 5383 5384 static const struct ib_device_ops irdma_gen1_dev_ops = { 5385 .dealloc_driver = irdma_ib_dealloc_device, 5386 }; 5387 5388 static const struct ib_device_ops irdma_gen3_dev_ops = { 5389 .alloc_mw = irdma_alloc_mw, 5390 .create_srq = irdma_create_srq, 5391 .dealloc_mw = irdma_dealloc_mw, 5392 .destroy_srq = irdma_destroy_srq, 5393 .modify_srq = irdma_modify_srq, 5394 .post_srq_recv = irdma_post_srq_recv, 5395 .query_srq = irdma_query_srq, 5396 }; 5397 5398 static const struct ib_device_ops irdma_roce_dev_ops = { 5399 .attach_mcast = irdma_attach_mcast, 5400 .create_ah = irdma_create_ah, 5401 .create_user_ah = irdma_create_user_ah, 5402 .destroy_ah = irdma_destroy_ah, 5403 .detach_mcast = irdma_detach_mcast, 5404 .get_link_layer = irdma_get_link_layer, 5405 .get_port_immutable = irdma_roce_port_immutable, 5406 .modify_qp = irdma_modify_qp_roce, 5407 .query_ah = irdma_query_ah, 5408 .query_pkey = irdma_query_pkey, 5409 }; 5410 5411 static const struct ib_device_ops irdma_iw_dev_ops = { 5412 .get_port_immutable = irdma_iw_port_immutable, 5413 .iw_accept = irdma_accept, 5414 .iw_add_ref = irdma_qp_add_ref, 5415 .iw_connect = irdma_connect, 5416 .iw_create_listen = irdma_create_listen, 5417 .iw_destroy_listen = irdma_destroy_listen, 5418 .iw_get_qp = irdma_get_qp, 5419 .iw_reject = irdma_reject, 5420 .iw_rem_ref = irdma_qp_rem_ref, 5421 .modify_qp = irdma_modify_qp, 5422 .query_gid = irdma_query_gid, 5423 }; 5424 5425 static const struct ib_device_ops irdma_dev_ops = { 5426 .owner = THIS_MODULE, 5427 .driver_id = RDMA_DRIVER_IRDMA, 5428 .uverbs_abi_ver = IRDMA_ABI_VER, 5429 5430 .alloc_hw_port_stats = irdma_alloc_hw_port_stats, 5431 .alloc_mr = irdma_alloc_mr, 5432 .alloc_pd = irdma_alloc_pd, 5433 .alloc_ucontext = irdma_alloc_ucontext, 5434 .create_cq = irdma_create_cq, 5435 .create_qp = irdma_create_qp, 5436 .dealloc_driver = irdma_ib_dealloc_device, 5437 .dealloc_mw = irdma_dealloc_mw, 5438 .dealloc_pd = irdma_dealloc_pd, 5439 .dealloc_ucontext = irdma_dealloc_ucontext, 5440 .dereg_mr = irdma_dereg_mr, 5441 .destroy_cq = irdma_destroy_cq, 5442 .destroy_qp = irdma_destroy_qp, 5443 .disassociate_ucontext = irdma_disassociate_ucontext, 5444 .get_dev_fw_str = irdma_get_dev_fw_str, 5445 .get_dma_mr = irdma_get_dma_mr, 5446 .get_hw_stats = irdma_get_hw_stats, 5447 .map_mr_sg = irdma_map_mr_sg, 5448 .mmap = irdma_mmap, 5449 .mmap_free = irdma_mmap_free, 5450 .poll_cq = irdma_poll_cq, 5451 .post_recv = irdma_post_recv, 5452 .post_send = irdma_post_send, 5453 .query_device = irdma_query_device, 5454 .query_port = irdma_query_port, 5455 .query_qp = irdma_query_qp, 5456 .reg_user_mr = irdma_reg_user_mr, 5457 .reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf, 5458 .rereg_user_mr = irdma_rereg_user_mr, 5459 .req_notify_cq = irdma_req_notify_cq, 5460 .resize_user_cq = irdma_resize_cq, 5461 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd), 5462 INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext), 5463 INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), 5464 INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), 5465 INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), 5466 INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp), 5467 INIT_RDMA_OBJ_SIZE(ib_srq, irdma_srq, ibsrq), 5468 }; 5469 5470 /** 5471 * irdma_init_roce_device - initialization of roce rdma device 5472 * @iwdev: irdma device 5473 */ 5474 static void irdma_init_roce_device(struct irdma_device *iwdev) 5475 { 5476 iwdev->ibdev.node_type = RDMA_NODE_IB_CA; 5477 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 5478 iwdev->netdev->dev_addr); 5479 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); 5480 } 5481 5482 /** 5483 * irdma_init_iw_device - initialization of iwarp rdma device 5484 * @iwdev: irdma device 5485 */ 5486 static void irdma_init_iw_device(struct irdma_device *iwdev) 5487 { 5488 struct net_device *netdev = iwdev->netdev; 5489 5490 iwdev->ibdev.node_type = RDMA_NODE_RNIC; 5491 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 5492 netdev->dev_addr); 5493 memcpy(iwdev->ibdev.iw_ifname, netdev->name, 5494 sizeof(iwdev->ibdev.iw_ifname)); 5495 ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops); 5496 } 5497 5498 /** 5499 * irdma_init_rdma_device - initialization of rdma device 5500 * @iwdev: irdma device 5501 */ 5502 static void irdma_init_rdma_device(struct irdma_device *iwdev) 5503 { 5504 struct pci_dev *pcidev = iwdev->rf->pcidev; 5505 5506 if (iwdev->roce_mode) 5507 irdma_init_roce_device(iwdev); 5508 else 5509 irdma_init_iw_device(iwdev); 5510 5511 iwdev->ibdev.phys_port_cnt = 1; 5512 iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; 5513 iwdev->ibdev.dev.parent = &pcidev->dev; 5514 ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops); 5515 if (iwdev->rf->rdma_ver == IRDMA_GEN_1) 5516 ib_set_device_ops(&iwdev->ibdev, &irdma_gen1_dev_ops); 5517 if (iwdev->rf->rdma_ver >= IRDMA_GEN_3) 5518 ib_set_device_ops(&iwdev->ibdev, &irdma_gen3_dev_ops); 5519 } 5520 5521 /** 5522 * irdma_port_ibevent - indicate port event 5523 * @iwdev: irdma device 5524 */ 5525 void irdma_port_ibevent(struct irdma_device *iwdev) 5526 { 5527 struct ib_event event; 5528 5529 event.device = &iwdev->ibdev; 5530 event.element.port_num = 1; 5531 event.event = 5532 iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; 5533 ib_dispatch_event(&event); 5534 } 5535 5536 /** 5537 * irdma_ib_unregister_device - unregister rdma device from IB 5538 * core 5539 * @iwdev: irdma device 5540 */ 5541 void irdma_ib_unregister_device(struct irdma_device *iwdev) 5542 { 5543 iwdev->iw_status = 0; 5544 irdma_port_ibevent(iwdev); 5545 ib_unregister_device(&iwdev->ibdev); 5546 } 5547 5548 /** 5549 * irdma_ib_register_device - register irdma device to IB core 5550 * @iwdev: irdma device 5551 */ 5552 int irdma_ib_register_device(struct irdma_device *iwdev) 5553 { 5554 int ret; 5555 5556 irdma_init_rdma_device(iwdev); 5557 5558 ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1); 5559 if (ret) 5560 goto error; 5561 dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX); 5562 ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device); 5563 if (ret) 5564 goto error; 5565 5566 iwdev->iw_status = 1; 5567 irdma_port_ibevent(iwdev); 5568 5569 return 0; 5570 5571 error: 5572 if (ret) 5573 ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n"); 5574 5575 return ret; 5576 } 5577 5578 /** 5579 * irdma_ib_dealloc_device 5580 * @ibdev: ib device 5581 * 5582 * callback from ibdev dealloc_driver to deallocate resources 5583 * unber irdma device 5584 */ 5585 void irdma_ib_dealloc_device(struct ib_device *ibdev) 5586 { 5587 struct irdma_device *iwdev = to_iwdev(ibdev); 5588 5589 irdma_rt_deinit_hw(iwdev); 5590 if (!iwdev->is_vport) { 5591 irdma_ctrl_deinit_hw(iwdev->rf); 5592 if (iwdev->rf->vchnl_wq) { 5593 destroy_workqueue(iwdev->rf->vchnl_wq); 5594 mutex_destroy(&iwdev->rf->sc_dev.vchnl_mutex); 5595 } 5596 } 5597 } 5598