1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_query_device - get device attributes 7 * @ibdev: device pointer from stack 8 * @props: returning device attributes 9 * @udata: user data 10 */ 11 static int irdma_query_device(struct ib_device *ibdev, 12 struct ib_device_attr *props, 13 struct ib_udata *udata) 14 { 15 struct irdma_device *iwdev = to_iwdev(ibdev); 16 struct irdma_pci_f *rf = iwdev->rf; 17 struct pci_dev *pcidev = iwdev->rf->pcidev; 18 struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; 19 int err; 20 21 err = ib_no_udata_io(udata); 22 if (err) 23 return err; 24 25 addrconf_addr_eui48((u8 *)&props->sys_image_guid, 26 iwdev->netdev->dev_addr); 27 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | 28 irdma_fw_minor_ver(&rf->sc_dev); 29 props->device_cap_flags = IB_DEVICE_MEM_WINDOW | 30 IB_DEVICE_MEM_MGT_EXTENSIONS; 31 if (hw_attrs->uk_attrs.hw_rev < IRDMA_GEN_3) 32 props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 33 props->vendor_id = pcidev->vendor; 34 props->vendor_part_id = pcidev->device; 35 36 props->hw_ver = rf->pcidev->revision; 37 props->page_size_cap = hw_attrs->page_size_cap; 38 props->max_mr_size = hw_attrs->max_mr_size; 39 props->max_qp = rf->max_qp - rf->used_qps; 40 props->max_qp_wr = hw_attrs->max_qp_wr; 41 props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 42 props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 43 props->max_cq = rf->max_cq - rf->used_cqs; 44 props->max_cqe = rf->max_cqe - 1; 45 props->max_mr = rf->max_mr - rf->used_mrs; 46 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) 47 props->max_mw = props->max_mr; 48 props->max_pd = rf->max_pd - rf->used_pds; 49 props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; 50 props->max_qp_rd_atom = hw_attrs->max_hw_ird; 51 props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; 52 if (rdma_protocol_roce(ibdev, 1)) { 53 props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN; 54 props->max_pkeys = IRDMA_PKEY_TBL_SZ; 55 } 56 57 props->max_ah = rf->max_ah; 58 props->max_mcast_grp = rf->max_mcg; 59 props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; 60 props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; 61 props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; 62 props->max_srq = rf->max_srq - rf->used_srqs; 63 props->max_srq_wr = IRDMA_MAX_SRQ_WRS; 64 props->max_srq_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 65 if (hw_attrs->uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 66 props->atomic_cap = IB_ATOMIC_HCA; 67 else 68 props->atomic_cap = IB_ATOMIC_NONE; 69 props->masked_atomic_cap = props->atomic_cap; 70 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) { 71 #define HCA_CORE_CLOCK_KHZ 1000000UL 72 props->timestamp_mask = GENMASK(31, 0); 73 props->hca_core_clock = HCA_CORE_CLOCK_KHZ; 74 } 75 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) 76 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; 77 78 return 0; 79 } 80 81 /** 82 * irdma_query_port - get port attributes 83 * @ibdev: device pointer from stack 84 * @port: port number for query 85 * @props: returning device attributes 86 */ 87 static int irdma_query_port(struct ib_device *ibdev, u32 port, 88 struct ib_port_attr *props) 89 { 90 struct irdma_device *iwdev = to_iwdev(ibdev); 91 struct net_device *netdev = iwdev->netdev; 92 93 /* no need to zero out pros here. done by caller */ 94 95 props->max_mtu = IB_MTU_4096; 96 props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); 97 props->lid = 1; 98 props->lmc = 0; 99 props->sm_lid = 0; 100 props->sm_sl = 0; 101 if (netif_carrier_ok(netdev) && netif_running(netdev)) { 102 props->state = IB_PORT_ACTIVE; 103 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 104 } else { 105 props->state = IB_PORT_DOWN; 106 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 107 } 108 109 ib_get_eth_speed(ibdev, port, &props->active_speed, 110 &props->active_width); 111 112 if (rdma_protocol_roce(ibdev, 1)) { 113 props->gid_tbl_len = 32; 114 props->ip_gids = true; 115 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 116 } else { 117 props->gid_tbl_len = 1; 118 } 119 props->qkey_viol_cntr = 0; 120 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 121 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 122 123 return 0; 124 } 125 126 /** 127 * irdma_disassociate_ucontext - Disassociate user context 128 * @context: ib user context 129 */ 130 static void irdma_disassociate_ucontext(struct ib_ucontext *context) 131 { 132 } 133 134 static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, 135 struct vm_area_struct *vma) 136 { 137 u64 pfn; 138 139 if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) 140 return -EINVAL; 141 142 vma->vm_private_data = ucontext; 143 pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + 144 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 145 146 return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, 147 pgprot_noncached(vma->vm_page_prot), NULL); 148 } 149 150 static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 151 { 152 struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); 153 154 kfree(entry); 155 } 156 157 static struct rdma_user_mmap_entry* 158 irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, 159 enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) 160 { 161 struct irdma_user_mmap_entry *entry = kzalloc_obj(*entry); 162 int ret; 163 164 if (!entry) 165 return NULL; 166 167 entry->bar_offset = bar_offset; 168 entry->mmap_flag = mmap_flag; 169 170 ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, 171 &entry->rdma_entry, PAGE_SIZE); 172 if (ret) { 173 kfree(entry); 174 return NULL; 175 } 176 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 177 178 return &entry->rdma_entry; 179 } 180 181 /** 182 * irdma_mmap - user memory map 183 * @context: context created during alloc 184 * @vma: kernel info for user memory map 185 */ 186 static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 187 { 188 struct rdma_user_mmap_entry *rdma_entry; 189 struct irdma_user_mmap_entry *entry; 190 struct irdma_ucontext *ucontext; 191 u64 pfn; 192 int ret; 193 194 ucontext = to_ucontext(context); 195 196 /* Legacy support for libi40iw with hard-coded mmap key */ 197 if (ucontext->legacy_mode) 198 return irdma_mmap_legacy(ucontext, vma); 199 200 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 201 if (!rdma_entry) { 202 ibdev_dbg(&ucontext->iwdev->ibdev, 203 "VERBS: pgoff[0x%lx] does not have valid entry\n", 204 vma->vm_pgoff); 205 return -EINVAL; 206 } 207 208 entry = to_irdma_mmap_entry(rdma_entry); 209 ibdev_dbg(&ucontext->iwdev->ibdev, 210 "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n", 211 entry->bar_offset, entry->mmap_flag); 212 213 pfn = (entry->bar_offset + 214 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 215 216 switch (entry->mmap_flag) { 217 case IRDMA_MMAP_IO_NC: 218 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 219 pgprot_noncached(vma->vm_page_prot), 220 rdma_entry); 221 break; 222 case IRDMA_MMAP_IO_WC: 223 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 224 pgprot_writecombine(vma->vm_page_prot), 225 rdma_entry); 226 break; 227 default: 228 ret = -EINVAL; 229 } 230 231 if (ret) 232 ibdev_dbg(&ucontext->iwdev->ibdev, 233 "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n", 234 entry->bar_offset, entry->mmap_flag, ret); 235 rdma_user_mmap_entry_put(rdma_entry); 236 237 return ret; 238 } 239 240 /** 241 * irdma_alloc_push_page - allocate a push page for qp 242 * @iwqp: qp pointer 243 */ 244 static void irdma_alloc_push_page(struct irdma_qp *iwqp) 245 { 246 struct irdma_cqp_request *cqp_request; 247 struct cqp_cmds_info *cqp_info; 248 struct irdma_device *iwdev = iwqp->iwdev; 249 struct irdma_sc_qp *qp = &iwqp->sc_qp; 250 int status; 251 252 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 253 if (!cqp_request) 254 return; 255 256 cqp_info = &cqp_request->info; 257 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 258 cqp_info->post_sq = 1; 259 cqp_info->in.u.manage_push_page.info.push_idx = 0; 260 cqp_info->in.u.manage_push_page.info.qs_handle = 261 qp->vsi->qos[qp->user_pri].qs_handle; 262 cqp_info->in.u.manage_push_page.info.free_page = 0; 263 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 264 cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; 265 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 266 267 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 268 if (!status && cqp_request->compl_info.op_ret_val < 269 iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { 270 qp->push_idx = cqp_request->compl_info.op_ret_val; 271 qp->push_offset = 0; 272 } 273 274 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 275 } 276 277 /** 278 * irdma_alloc_ucontext - Allocate the user context data structure 279 * @uctx: uverbs context pointer 280 * @udata: user data 281 * 282 * This keeps track of all objects associated with a particular 283 * user-mode client. 284 */ 285 static int irdma_alloc_ucontext(struct ib_ucontext *uctx, 286 struct ib_udata *udata) 287 { 288 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) 289 struct ib_device *ibdev = uctx->device; 290 struct irdma_device *iwdev = to_iwdev(ibdev); 291 struct irdma_alloc_ucontext_req req = {}; 292 struct irdma_alloc_ucontext_resp uresp = {}; 293 struct irdma_ucontext *ucontext = to_ucontext(uctx); 294 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 295 int ret; 296 297 if (udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) 298 return -EINVAL; 299 300 ret = ib_copy_validate_udata_in_cm(udata, req, rsvd8, 301 IRDMA_ALLOC_UCTX_USE_RAW_ATTR | 302 IRDMA_SUPPORT_WQE_FORMAT_V2); 303 if (ret) 304 return ret; 305 306 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 307 goto ver_error; 308 309 ucontext->iwdev = iwdev; 310 ucontext->abi_ver = req.userspace_ver; 311 312 if (!(req.comp_mask & IRDMA_SUPPORT_WQE_FORMAT_V2) && 313 uk_attrs->hw_rev >= IRDMA_GEN_3) 314 return -EOPNOTSUPP; 315 316 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR) 317 ucontext->use_raw_attrs = true; 318 319 /* GEN_1 legacy support with libi40iw */ 320 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { 321 if (uk_attrs->hw_rev != IRDMA_GEN_1) 322 return -EOPNOTSUPP; 323 324 ucontext->legacy_mode = true; 325 uresp.max_qps = iwdev->rf->max_qp; 326 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 327 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 328 uresp.kernel_ver = req.userspace_ver; 329 ret = ib_respond_udata(udata, uresp); 330 if (ret) 331 return ret; 332 } else { 333 u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 334 335 ucontext->db_mmap_entry = 336 irdma_user_mmap_entry_insert(ucontext, bar_off, 337 IRDMA_MMAP_IO_NC, 338 &uresp.db_mmap_key); 339 if (!ucontext->db_mmap_entry) 340 return -ENOMEM; 341 342 uresp.kernel_ver = IRDMA_ABI_VER; 343 uresp.feature_flags = uk_attrs->feature_flags; 344 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 345 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 346 uresp.max_hw_inline = uk_attrs->max_hw_inline; 347 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 348 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 349 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 350 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 351 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 352 uresp.hw_rev = uk_attrs->hw_rev; 353 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; 354 uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size; 355 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE; 356 uresp.max_hw_srq_quanta = uk_attrs->max_hw_srq_quanta; 357 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA; 358 ret = ib_respond_udata(udata, uresp); 359 if (ret) { 360 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 361 return ret; 362 } 363 } 364 365 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 366 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 367 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 368 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 369 INIT_LIST_HEAD(&ucontext->srq_reg_mem_list); 370 spin_lock_init(&ucontext->srq_reg_mem_list_lock); 371 372 return 0; 373 374 ver_error: 375 ibdev_err(&iwdev->ibdev, 376 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 377 req.userspace_ver, IRDMA_ABI_VER); 378 return -EINVAL; 379 } 380 381 /** 382 * irdma_dealloc_ucontext - deallocate the user context data structure 383 * @context: user context created during alloc 384 */ 385 static void irdma_dealloc_ucontext(struct ib_ucontext *context) 386 { 387 struct irdma_ucontext *ucontext = to_ucontext(context); 388 389 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 390 } 391 392 /** 393 * irdma_alloc_pd - allocate protection domain 394 * @pd: PD pointer 395 * @udata: user data 396 */ 397 static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 398 { 399 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) 400 struct irdma_pd *iwpd = to_iwpd(pd); 401 struct irdma_device *iwdev = to_iwdev(pd->device); 402 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 403 struct irdma_pci_f *rf = iwdev->rf; 404 struct irdma_alloc_pd_resp uresp = {}; 405 struct irdma_sc_pd *sc_pd; 406 u32 pd_id = 0; 407 int err; 408 409 err = ib_is_udata_in_empty(udata); 410 if (err) 411 return err; 412 413 if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) 414 return -EINVAL; 415 416 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 417 &rf->next_pd); 418 if (err) 419 return err; 420 421 sc_pd = &iwpd->sc_pd; 422 if (udata) { 423 struct irdma_ucontext *ucontext = 424 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 425 ibucontext); 426 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 427 uresp.pd_id = pd_id; 428 err = ib_respond_udata(udata, uresp); 429 if (err) 430 goto error; 431 } else { 432 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 433 } 434 435 return 0; 436 error: 437 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 438 439 return err; 440 } 441 442 /** 443 * irdma_dealloc_pd - deallocate pd 444 * @ibpd: ptr of pd to be deallocated 445 * @udata: user data 446 */ 447 static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 448 { 449 struct irdma_pd *iwpd = to_iwpd(ibpd); 450 struct irdma_device *iwdev = to_iwdev(ibpd->device); 451 int ret; 452 453 ret = ib_no_udata_io(udata); 454 if (ret) 455 return ret; 456 457 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 458 459 return 0; 460 } 461 462 /** 463 * irdma_get_pbl - Retrieve pbl from a list given a virtual 464 * address 465 * @va: user virtual address 466 * @pbl_list: pbl list to search in (QP's or CQ's) 467 */ 468 static struct irdma_pbl *irdma_get_pbl(unsigned long va, 469 struct list_head *pbl_list) 470 { 471 struct irdma_pbl *iwpbl; 472 473 list_for_each_entry (iwpbl, pbl_list, list) { 474 if (iwpbl->user_base == va) { 475 struct irdma_mr *iwmr = iwpbl->iwmr; 476 477 refcount_inc(&iwmr->user_ring_refs); 478 list_del(&iwpbl->list); 479 iwpbl->on_list = false; 480 return iwpbl; 481 } 482 } 483 484 return NULL; 485 } 486 487 /** 488 * irdma_clean_cqes - clean cq entries for qp 489 * @iwqp: qp ptr (user or kernel) 490 * @iwcq: cq ptr 491 */ 492 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) 493 { 494 struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; 495 unsigned long flags; 496 497 spin_lock_irqsave(&iwcq->lock, flags); 498 irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); 499 spin_unlock_irqrestore(&iwcq->lock, flags); 500 } 501 502 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) 503 { 504 if (iwqp->push_db_mmap_entry) { 505 rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); 506 iwqp->push_db_mmap_entry = NULL; 507 } 508 if (iwqp->push_wqe_mmap_entry) { 509 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 510 iwqp->push_wqe_mmap_entry = NULL; 511 } 512 } 513 514 static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, 515 struct irdma_qp *iwqp, 516 u64 *push_wqe_mmap_key, 517 u64 *push_db_mmap_key) 518 { 519 struct irdma_device *iwdev = ucontext->iwdev; 520 u64 rsvd, bar_off; 521 522 rsvd = IRDMA_PF_BAR_RSVD; 523 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 524 /* skip over db page */ 525 bar_off += IRDMA_HW_PAGE_SIZE; 526 /* push wqe page */ 527 bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; 528 iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 529 bar_off, IRDMA_MMAP_IO_WC, 530 push_wqe_mmap_key); 531 if (!iwqp->push_wqe_mmap_entry) 532 return -ENOMEM; 533 534 /* push doorbell page */ 535 bar_off += IRDMA_HW_PAGE_SIZE; 536 iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 537 bar_off, IRDMA_MMAP_IO_NC, 538 push_db_mmap_key); 539 if (!iwqp->push_db_mmap_entry) { 540 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 541 return -ENOMEM; 542 } 543 544 return 0; 545 } 546 547 /** 548 * _irdma_destroy_qp - destroy qp 549 * @ibqp: qp's ib pointer also to get to device's qp address 550 */ 551 static void _irdma_destroy_qp(struct ib_qp *ibqp) 552 { 553 struct irdma_qp *iwqp = to_iwqp(ibqp); 554 struct irdma_device *iwdev = iwqp->iwdev; 555 556 iwqp->sc_qp.qp_uk.destroy_pending = true; 557 558 if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE) 559 irdma_modify_qp_to_err(&iwqp->sc_qp); 560 561 if (!iwqp->user_mode) 562 cancel_delayed_work_sync(&iwqp->dwork_flush); 563 564 if (!iwqp->user_mode) { 565 if (iwqp->iwscq) { 566 irdma_clean_cqes(iwqp, iwqp->iwscq); 567 if (iwqp->iwrcq != iwqp->iwscq) 568 irdma_clean_cqes(iwqp, iwqp->iwrcq); 569 } 570 } 571 572 irdma_qp_rem_ref(&iwqp->ibqp); 573 if (!iwdev->rf->reset) 574 wait_for_completion(&iwqp->free_qp); 575 irdma_free_lsmm_rsrc(iwqp); 576 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); 577 578 irdma_remove_push_mmap_entries(iwqp); 579 580 if (iwqp->sc_qp.qp_uk.qp_id == 1) 581 iwdev->rf->hwqp1_rsvd = false; 582 irdma_free_qp_rsrc(iwqp); 583 } 584 585 /** 586 * irdma_destroy_qp - destroy qp 587 * @ibqp: qp's ib pointer also to get to device's qp address 588 * @udata: user data 589 */ 590 static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 591 { 592 int ret; 593 594 ret = ib_no_udata_io(udata); 595 if (ret) 596 return ret; 597 598 _irdma_destroy_qp(ibqp); 599 600 return 0; 601 } 602 603 /** 604 * irdma_setup_virt_qp - setup for allocation of virtual qp 605 * @iwdev: irdma device 606 * @iwqp: qp ptr 607 * @init_info: initialize info to return 608 */ 609 static void irdma_setup_virt_qp(struct irdma_device *iwdev, 610 struct irdma_qp *iwqp, 611 struct irdma_qp_init_info *init_info) 612 { 613 struct irdma_pbl *iwpbl = iwqp->iwpbl; 614 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 615 616 iwqp->page = qpmr->sq_page; 617 init_info->shadow_area_pa = qpmr->shadow; 618 if (iwpbl->pbl_allocated) { 619 init_info->virtual_map = true; 620 init_info->sq_pa = qpmr->sq_pbl.idx; 621 /* Need to use contiguous buffer for RQ of QP 622 * in case it is associated with SRQ. 623 */ 624 init_info->rq_pa = init_info->qp_uk_init_info.srq_uk ? 625 qpmr->rq_pa : qpmr->rq_pbl.idx; 626 } else { 627 init_info->sq_pa = qpmr->sq_pbl.addr; 628 init_info->rq_pa = qpmr->rq_pbl.addr; 629 } 630 } 631 632 /** 633 * irdma_setup_umode_qp - setup sq and rq size in user mode qp 634 * @ucontext: user context 635 * @req: user request pointer 636 * @iwdev: iwarp device 637 * @iwqp: qp ptr (user or kernel) 638 * @info: initialize info to return 639 * @init_attr: Initial QP create attributes 640 */ 641 static int irdma_setup_umode_qp(struct irdma_ucontext *ucontext, 642 struct irdma_create_qp_req *req, 643 struct irdma_device *iwdev, 644 struct irdma_qp *iwqp, 645 struct irdma_qp_init_info *info, 646 struct ib_qp_init_attr *init_attr) 647 { 648 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 649 unsigned long flags; 650 int ret; 651 652 iwqp->ctx_info.qp_compl_ctx = req->user_compl_ctx; 653 iwqp->user_mode = 1; 654 655 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 656 iwqp->iwpbl = irdma_get_pbl((unsigned long)req->user_wqe_bufs, 657 &ucontext->qp_reg_mem_list); 658 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 659 660 if (!iwqp->iwpbl) { 661 ret = -ENODATA; 662 ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); 663 return ret; 664 } 665 666 if (!ucontext->use_raw_attrs) { 667 /** 668 * Maintain backward compat with older ABI which passes sq and 669 * rq depth in quanta in cap.max_send_wr and cap.max_recv_wr. 670 * There is no way to compute the correct value of 671 * iwqp->max_send_wr/max_recv_wr in the kernel. 672 */ 673 iwqp->max_send_wr = init_attr->cap.max_send_wr; 674 iwqp->max_recv_wr = init_attr->cap.max_recv_wr; 675 ukinfo->sq_size = init_attr->cap.max_send_wr; 676 ukinfo->rq_size = init_attr->cap.max_recv_wr; 677 irdma_uk_calc_shift_wq(ukinfo, &ukinfo->sq_shift, 678 &ukinfo->rq_shift); 679 } else { 680 ret = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth, 681 &ukinfo->sq_shift); 682 if (ret) 683 return ret; 684 685 ret = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth, 686 &ukinfo->rq_shift); 687 if (ret) 688 return ret; 689 690 iwqp->max_send_wr = 691 (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift; 692 iwqp->max_recv_wr = 693 (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift; 694 ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; 695 ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; 696 } 697 698 irdma_setup_virt_qp(iwdev, iwqp, info); 699 700 return 0; 701 } 702 703 /** 704 * irdma_setup_kmode_qp - setup initialization for kernel mode qp 705 * @iwdev: iwarp device 706 * @iwqp: qp ptr (user or kernel) 707 * @info: initialize info to return 708 * @init_attr: Initial QP create attributes 709 */ 710 static int irdma_setup_kmode_qp(struct irdma_device *iwdev, 711 struct irdma_qp *iwqp, 712 struct irdma_qp_init_info *info, 713 struct ib_qp_init_attr *init_attr) 714 { 715 struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; 716 u32 size; 717 int status; 718 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 719 720 status = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth, 721 &ukinfo->sq_shift); 722 if (status) 723 return status; 724 725 status = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth, 726 &ukinfo->rq_shift); 727 if (status) 728 return status; 729 730 iwqp->kqp.sq_wrid_mem = 731 kzalloc_objs(*iwqp->kqp.sq_wrid_mem, ukinfo->sq_depth); 732 if (!iwqp->kqp.sq_wrid_mem) 733 return -ENOMEM; 734 735 iwqp->kqp.rq_wrid_mem = 736 kzalloc_objs(*iwqp->kqp.rq_wrid_mem, ukinfo->rq_depth); 737 738 if (!iwqp->kqp.rq_wrid_mem) { 739 kfree(iwqp->kqp.sq_wrid_mem); 740 iwqp->kqp.sq_wrid_mem = NULL; 741 return -ENOMEM; 742 } 743 744 ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; 745 ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; 746 747 size = (ukinfo->sq_depth + ukinfo->rq_depth) * IRDMA_QP_WQE_MIN_SIZE; 748 size += (IRDMA_SHADOW_AREA_SIZE << 3); 749 750 mem->size = ALIGN(size, 256); 751 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 752 &mem->pa, GFP_KERNEL); 753 if (!mem->va) { 754 kfree(iwqp->kqp.sq_wrid_mem); 755 iwqp->kqp.sq_wrid_mem = NULL; 756 kfree(iwqp->kqp.rq_wrid_mem); 757 iwqp->kqp.rq_wrid_mem = NULL; 758 return -ENOMEM; 759 } 760 761 ukinfo->sq = mem->va; 762 info->sq_pa = mem->pa; 763 ukinfo->rq = &ukinfo->sq[ukinfo->sq_depth]; 764 info->rq_pa = info->sq_pa + (ukinfo->sq_depth * IRDMA_QP_WQE_MIN_SIZE); 765 ukinfo->shadow_area = ukinfo->rq[ukinfo->rq_depth].elem; 766 info->shadow_area_pa = 767 info->rq_pa + (ukinfo->rq_depth * IRDMA_QP_WQE_MIN_SIZE); 768 ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; 769 ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; 770 ukinfo->qp_id = info->qp_uk_init_info.qp_id; 771 772 iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift; 773 iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift; 774 init_attr->cap.max_send_wr = iwqp->max_send_wr; 775 init_attr->cap.max_recv_wr = iwqp->max_recv_wr; 776 777 return 0; 778 } 779 780 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) 781 { 782 struct irdma_pci_f *rf = iwqp->iwdev->rf; 783 struct irdma_cqp_request *cqp_request; 784 struct cqp_cmds_info *cqp_info; 785 struct irdma_create_qp_info *qp_info; 786 int status; 787 788 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 789 if (!cqp_request) 790 return -ENOMEM; 791 792 cqp_info = &cqp_request->info; 793 qp_info = &cqp_request->info.in.u.qp_create.info; 794 qp_info->mac_valid = true; 795 qp_info->cq_num_valid = true; 796 qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; 797 798 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 799 cqp_info->post_sq = 1; 800 cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; 801 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 802 status = irdma_handle_cqp_op(rf, cqp_request); 803 irdma_put_cqp_request(&rf->cqp, cqp_request); 804 805 return status; 806 } 807 808 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 809 struct irdma_qp_host_ctx_info *ctx_info) 810 { 811 struct irdma_device *iwdev = iwqp->iwdev; 812 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 813 struct irdma_roce_offload_info *roce_info; 814 struct irdma_udp_offload_info *udp_info; 815 816 udp_info = &iwqp->udp_info; 817 udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); 818 udp_info->cwnd = iwdev->roce_cwnd; 819 udp_info->rexmit_thresh = 2; 820 udp_info->rnr_nak_thresh = 2; 821 udp_info->src_port = 0xc000; 822 udp_info->dst_port = ROCE_V2_UDP_DPORT; 823 roce_info = &iwqp->roce_info; 824 ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr); 825 826 if (iwqp->ibqp.qp_type == IB_QPT_GSI && iwqp->ibqp.qp_num != 1) 827 roce_info->is_qp1 = true; 828 roce_info->rd_en = true; 829 roce_info->wr_rdresp_en = true; 830 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) 831 roce_info->bind_en = true; 832 roce_info->dcqcn_en = false; 833 roce_info->rtomin = 5; 834 835 roce_info->ack_credits = iwdev->roce_ackcreds; 836 roce_info->ird_size = dev->hw_attrs.max_hw_ird; 837 roce_info->ord_size = dev->hw_attrs.max_hw_ord; 838 839 if (!iwqp->user_mode) { 840 roce_info->priv_mode_en = true; 841 roce_info->fast_reg_en = true; 842 roce_info->udprivcq_en = true; 843 } 844 roce_info->roce_tver = 0; 845 846 ctx_info->roce_info = &iwqp->roce_info; 847 ctx_info->udp_info = &iwqp->udp_info; 848 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 849 } 850 851 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 852 struct irdma_qp_host_ctx_info *ctx_info) 853 { 854 struct irdma_device *iwdev = iwqp->iwdev; 855 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 856 struct irdma_iwarp_offload_info *iwarp_info; 857 858 iwarp_info = &iwqp->iwarp_info; 859 ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr); 860 iwarp_info->rd_en = true; 861 iwarp_info->wr_rdresp_en = true; 862 iwarp_info->ecn_en = true; 863 iwarp_info->rtomin = 5; 864 865 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 866 iwarp_info->ib_rd_en = true; 867 if (!iwqp->user_mode) { 868 iwarp_info->priv_mode_en = true; 869 iwarp_info->fast_reg_en = true; 870 } 871 iwarp_info->ddp_ver = 1; 872 iwarp_info->rdmap_ver = 1; 873 874 ctx_info->iwarp_info = &iwqp->iwarp_info; 875 ctx_info->iwarp_info_valid = true; 876 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 877 ctx_info->iwarp_info_valid = false; 878 } 879 880 static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, 881 struct irdma_device *iwdev) 882 { 883 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 884 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 885 886 if (init_attr->create_flags) 887 return -EOPNOTSUPP; 888 889 if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || 890 init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || 891 init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags || 892 init_attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta || 893 init_attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta) 894 return -EINVAL; 895 896 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 897 if (init_attr->qp_type != IB_QPT_RC && 898 init_attr->qp_type != IB_QPT_UD && 899 init_attr->qp_type != IB_QPT_GSI) 900 return -EOPNOTSUPP; 901 } else { 902 if (init_attr->qp_type != IB_QPT_RC) 903 return -EOPNOTSUPP; 904 } 905 906 return 0; 907 } 908 909 static void irdma_flush_worker(struct work_struct *work) 910 { 911 struct delayed_work *dwork = to_delayed_work(work); 912 struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush); 913 914 irdma_generate_flush_completions(iwqp); 915 } 916 917 static int irdma_setup_gsi_qp_rsrc(struct irdma_qp *iwqp, u32 *qp_num) 918 { 919 struct irdma_device *iwdev = iwqp->iwdev; 920 struct irdma_pci_f *rf = iwdev->rf; 921 unsigned long flags; 922 int ret; 923 924 if (rf->rdma_ver <= IRDMA_GEN_2) { 925 *qp_num = 1; 926 return 0; 927 } 928 929 spin_lock_irqsave(&rf->rsrc_lock, flags); 930 if (!rf->hwqp1_rsvd) { 931 *qp_num = 1; 932 rf->hwqp1_rsvd = true; 933 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 934 } else { 935 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 936 ret = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 937 qp_num, &rf->next_qp); 938 if (ret) 939 return ret; 940 } 941 942 ret = irdma_vchnl_req_add_vport(&rf->sc_dev, iwdev->vport_id, *qp_num, 943 (&iwdev->vsi)->qos); 944 if (ret) { 945 if (*qp_num != 1) { 946 irdma_free_rsrc(rf, rf->allocated_qps, *qp_num); 947 } else { 948 spin_lock_irqsave(&rf->rsrc_lock, flags); 949 rf->hwqp1_rsvd = false; 950 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 951 } 952 return ret; 953 } 954 955 return 0; 956 } 957 958 /** 959 * irdma_create_qp - create qp 960 * @ibqp: ptr of qp 961 * @init_attr: attributes for qp 962 * @udata: user data for create qp 963 */ 964 static int irdma_create_qp(struct ib_qp *ibqp, 965 struct ib_qp_init_attr *init_attr, 966 struct ib_udata *udata) 967 { 968 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx) 969 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd) 970 struct ib_pd *ibpd = ibqp->pd; 971 struct irdma_pd *iwpd = to_iwpd(ibpd); 972 struct irdma_device *iwdev = to_iwdev(ibpd->device); 973 struct irdma_pci_f *rf = iwdev->rf; 974 struct irdma_qp *iwqp = to_iwqp(ibqp); 975 struct irdma_create_qp_resp uresp = {}; 976 u32 qp_num = 0; 977 int err_code; 978 struct irdma_sc_qp *qp; 979 struct irdma_sc_dev *dev = &rf->sc_dev; 980 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 981 struct irdma_qp_init_info init_info = {}; 982 struct irdma_qp_host_ctx_info *ctx_info; 983 struct irdma_create_qp_req ureq = {}; 984 struct irdma_srq *iwsrq; 985 bool srq_valid = false; 986 u32 srq_id = 0; 987 988 if (init_attr->srq) { 989 iwsrq = to_iwsrq(init_attr->srq); 990 srq_valid = true; 991 srq_id = iwsrq->srq_num; 992 init_attr->cap.max_recv_sge = uk_attrs->max_hw_wq_frags; 993 init_attr->cap.max_recv_wr = 4; 994 init_info.qp_uk_init_info.srq_uk = &iwsrq->sc_srq.srq_uk; 995 } 996 997 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 998 if (err_code) 999 return err_code; 1000 1001 if (udata) { 1002 if (udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN) 1003 return -EINVAL; 1004 1005 err_code = ib_copy_validate_udata_in(udata, ureq, user_compl_ctx); 1006 if (err_code) 1007 return err_code; 1008 } 1009 1010 init_info.vsi = &iwdev->vsi; 1011 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 1012 init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr; 1013 init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr; 1014 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 1015 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 1016 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 1017 1018 qp = &iwqp->sc_qp; 1019 qp->qp_uk.back_qp = iwqp; 1020 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1021 1022 iwqp->iwdev = iwdev; 1023 iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, 1024 256); 1025 iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, 1026 iwqp->q2_ctx_mem.size, 1027 &iwqp->q2_ctx_mem.pa, 1028 GFP_KERNEL); 1029 if (!iwqp->q2_ctx_mem.va) 1030 return -ENOMEM; 1031 1032 init_info.q2 = iwqp->q2_ctx_mem.va; 1033 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 1034 init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE); 1035 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 1036 1037 if (init_attr->qp_type == IB_QPT_GSI) { 1038 err_code = irdma_setup_gsi_qp_rsrc(iwqp, &qp_num); 1039 if (err_code) 1040 goto error; 1041 iwqp->ibqp.qp_num = 1; 1042 } else { 1043 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 1044 &qp_num, &rf->next_qp); 1045 if (err_code) 1046 goto error; 1047 iwqp->ibqp.qp_num = qp_num; 1048 } 1049 1050 iwqp->iwpd = iwpd; 1051 qp = &iwqp->sc_qp; 1052 iwqp->iwscq = to_iwcq(init_attr->send_cq); 1053 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 1054 iwqp->host_ctx.va = init_info.host_ctx; 1055 iwqp->host_ctx.pa = init_info.host_ctx_pa; 1056 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 1057 1058 init_info.pd = &iwpd->sc_pd; 1059 init_info.qp_uk_init_info.qp_id = qp_num; 1060 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 1061 init_info.qp_uk_init_info.first_sq_wq = 1; 1062 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 1063 init_waitqueue_head(&iwqp->waitq); 1064 init_waitqueue_head(&iwqp->mod_qp_waitq); 1065 1066 if (udata) { 1067 struct irdma_ucontext *ucontext = 1068 rdma_udata_to_drv_context(udata, 1069 struct irdma_ucontext, 1070 ibucontext); 1071 1072 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 1073 err_code = irdma_setup_umode_qp(ucontext, &ureq, iwdev, iwqp, 1074 &init_info, init_attr); 1075 } else { 1076 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); 1077 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 1078 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 1079 } 1080 1081 if (err_code) { 1082 ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n"); 1083 goto error; 1084 } 1085 1086 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 1087 if (init_attr->qp_type == IB_QPT_RC) { 1088 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 1089 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 1090 IRDMA_WRITE_WITH_IMM | 1091 IRDMA_ROCE; 1092 } else { 1093 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 1094 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 1095 IRDMA_ROCE; 1096 } 1097 } else { 1098 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 1099 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 1100 } 1101 1102 if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) 1103 init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE; 1104 1105 err_code = irdma_sc_qp_init(qp, &init_info); 1106 if (err_code) { 1107 ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n"); 1108 goto error; 1109 } 1110 1111 ctx_info = &iwqp->ctx_info; 1112 ctx_info->srq_valid = srq_valid; 1113 ctx_info->srq_id = srq_id; 1114 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1115 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1116 1117 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 1118 if (dev->ws_add(&iwdev->vsi, 0)) { 1119 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 1120 err_code = -EINVAL; 1121 goto error; 1122 } 1123 irdma_qp_add_qos(&iwqp->sc_qp); 1124 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 1125 } else { 1126 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 1127 } 1128 1129 err_code = irdma_cqp_create_qp_cmd(iwqp); 1130 if (err_code) 1131 goto error; 1132 1133 refcount_set(&iwqp->refcnt, 1); 1134 spin_lock_init(&iwqp->lock); 1135 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 1136 iwqp->sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1137 rf->qp_table[qp_num] = iwqp; 1138 init_completion(&iwqp->free_qp); 1139 1140 if (udata) { 1141 /* GEN_1 legacy support with libi40iw does not have expanded 1142 * uresp struct. Check for the exact legacy size (20 bytes) to 1143 * ensure that newer expanded uresp structs don't accidentally 1144 * trigger the legacy fallback. 1145 */ 1146 if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) { 1147 uresp.lsmm = 1; 1148 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 1149 } else { 1150 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 1151 uresp.lsmm = 1; 1152 } 1153 uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size; 1154 uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size; 1155 uresp.qp_id = qp_num; 1156 uresp.qp_caps = qp->qp_uk.qp_caps; 1157 1158 err_code = ib_respond_udata(udata, uresp); 1159 if (err_code) { 1160 _irdma_destroy_qp(&iwqp->ibqp); 1161 return err_code; 1162 } 1163 } 1164 1165 return 0; 1166 1167 error: 1168 irdma_free_qp_rsrc(iwqp); 1169 return err_code; 1170 } 1171 1172 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) 1173 { 1174 int acc_flags = 0; 1175 1176 if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { 1177 if (iwqp->roce_info.wr_rdresp_en) { 1178 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1179 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1180 } 1181 if (iwqp->roce_info.rd_en) 1182 acc_flags |= IB_ACCESS_REMOTE_READ; 1183 if (iwqp->roce_info.bind_en) 1184 acc_flags |= IB_ACCESS_MW_BIND; 1185 if (iwqp->ctx_info.remote_atomics_en) 1186 acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 1187 } else { 1188 if (iwqp->iwarp_info.wr_rdresp_en) { 1189 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1190 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1191 } 1192 if (iwqp->iwarp_info.rd_en) 1193 acc_flags |= IB_ACCESS_REMOTE_READ; 1194 if (iwqp->ctx_info.remote_atomics_en) 1195 acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 1196 } 1197 return acc_flags; 1198 } 1199 1200 /** 1201 * irdma_query_qp - query qp attributes 1202 * @ibqp: qp pointer 1203 * @attr: attributes pointer 1204 * @attr_mask: Not used 1205 * @init_attr: qp attributes to return 1206 */ 1207 static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1208 int attr_mask, struct ib_qp_init_attr *init_attr) 1209 { 1210 struct irdma_qp *iwqp = to_iwqp(ibqp); 1211 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1212 1213 memset(attr, 0, sizeof(*attr)); 1214 memset(init_attr, 0, sizeof(*init_attr)); 1215 1216 attr->qp_state = iwqp->ibqp_state; 1217 attr->cur_qp_state = iwqp->ibqp_state; 1218 attr->cap.max_send_wr = iwqp->max_send_wr; 1219 attr->cap.max_recv_wr = iwqp->max_recv_wr; 1220 attr->cap.max_inline_data = qp->qp_uk.max_inline_data; 1221 attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; 1222 attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; 1223 attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); 1224 attr->port_num = 1; 1225 if (rdma_protocol_roce(ibqp->device, 1)) { 1226 attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); 1227 attr->qkey = iwqp->roce_info.qkey; 1228 attr->rq_psn = iwqp->udp_info.epsn; 1229 attr->sq_psn = iwqp->udp_info.psn_nxt; 1230 attr->dest_qp_num = iwqp->roce_info.dest_qp; 1231 attr->pkey_index = iwqp->roce_info.p_key; 1232 attr->retry_cnt = iwqp->udp_info.rexmit_thresh; 1233 attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; 1234 attr->min_rnr_timer = iwqp->udp_info.min_rnr_timer; 1235 attr->max_rd_atomic = iwqp->roce_info.ord_size; 1236 attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; 1237 } 1238 1239 init_attr->event_handler = iwqp->ibqp.event_handler; 1240 init_attr->qp_context = iwqp->ibqp.qp_context; 1241 init_attr->send_cq = iwqp->ibqp.send_cq; 1242 init_attr->recv_cq = iwqp->ibqp.recv_cq; 1243 init_attr->srq = iwqp->ibqp.srq; 1244 init_attr->cap = attr->cap; 1245 1246 return 0; 1247 } 1248 1249 /** 1250 * irdma_query_pkey - Query partition key 1251 * @ibdev: device pointer from stack 1252 * @port: port number 1253 * @index: index of pkey 1254 * @pkey: pointer to store the pkey 1255 */ 1256 static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 1257 u16 *pkey) 1258 { 1259 if (index >= IRDMA_PKEY_TBL_SZ) 1260 return -EINVAL; 1261 1262 *pkey = IRDMA_DEFAULT_PKEY; 1263 return 0; 1264 } 1265 1266 static u8 irdma_roce_get_vlan_prio(const struct ib_gid_attr *attr, u8 prio) 1267 { 1268 struct net_device *ndev; 1269 1270 rcu_read_lock(); 1271 ndev = rcu_dereference(attr->ndev); 1272 if (!ndev) 1273 goto exit; 1274 if (is_vlan_dev(ndev)) { 1275 u16 vlan_qos = vlan_dev_get_egress_qos_mask(ndev, prio); 1276 1277 prio = (vlan_qos & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1278 } 1279 exit: 1280 rcu_read_unlock(); 1281 return prio; 1282 } 1283 1284 static int irdma_wait_for_suspend(struct irdma_qp *iwqp) 1285 { 1286 if (!wait_event_timeout(iwqp->iwdev->suspend_wq, 1287 !iwqp->suspend_pending, 1288 msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) { 1289 iwqp->suspend_pending = false; 1290 ibdev_warn(&iwqp->iwdev->ibdev, 1291 "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n", 1292 iwqp->ibqp.qp_num, iwqp->last_aeq); 1293 return -EBUSY; 1294 } 1295 1296 return 0; 1297 } 1298 1299 /** 1300 * irdma_modify_qp_roce - modify qp request 1301 * @ibqp: qp's pointer for modify 1302 * @attr: access attributes 1303 * @attr_mask: state mask 1304 * @udata: user data 1305 */ 1306 int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1307 int attr_mask, struct ib_udata *udata) 1308 { 1309 #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) 1310 struct irdma_pd *iwpd = to_iwpd(ibqp->pd); 1311 struct irdma_qp *iwqp = to_iwqp(ibqp); 1312 struct irdma_device *iwdev = iwqp->iwdev; 1313 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1314 struct irdma_qp_host_ctx_info *ctx_info; 1315 struct irdma_roce_offload_info *roce_info; 1316 struct irdma_udp_offload_info *udp_info; 1317 struct irdma_modify_qp_info info = {}; 1318 struct irdma_modify_qp_resp uresp = {}; 1319 struct irdma_modify_qp_req ureq = {}; 1320 unsigned long flags; 1321 u8 issue_modify_qp = 0; 1322 int ret = 0; 1323 1324 /* Clear the response buffer (if any). It may be updated again later. */ 1325 ret = ib_respond_empty_udata(udata); 1326 if (ret) 1327 return ret; 1328 1329 ctx_info = &iwqp->ctx_info; 1330 roce_info = &iwqp->roce_info; 1331 udp_info = &iwqp->udp_info; 1332 1333 if (udata) { 1334 /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ 1335 if (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN) 1336 return -EINVAL; 1337 1338 /* For current irdma, validate against ABI def. */ 1339 if (udata->inlen) { 1340 ret = ib_copy_validate_udata_in(udata, ureq, rsvd); 1341 if (ret) 1342 return ret; 1343 } 1344 } 1345 1346 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1347 return -EOPNOTSUPP; 1348 1349 if (attr_mask & IB_QP_DEST_QPN) 1350 roce_info->dest_qp = attr->dest_qp_num; 1351 1352 if (attr_mask & IB_QP_PKEY_INDEX) { 1353 ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, 1354 &roce_info->p_key); 1355 if (ret) 1356 return ret; 1357 } 1358 1359 if (attr_mask & IB_QP_QKEY) 1360 roce_info->qkey = attr->qkey; 1361 1362 if (attr_mask & IB_QP_PATH_MTU) 1363 udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); 1364 1365 if (attr_mask & IB_QP_SQ_PSN) { 1366 udp_info->psn_nxt = attr->sq_psn; 1367 udp_info->lsn = 0xffff; 1368 udp_info->psn_una = attr->sq_psn; 1369 udp_info->psn_max = attr->sq_psn; 1370 } 1371 1372 if (attr_mask & IB_QP_RQ_PSN) 1373 udp_info->epsn = attr->rq_psn; 1374 1375 if (attr_mask & IB_QP_RNR_RETRY) 1376 udp_info->rnr_nak_thresh = attr->rnr_retry; 1377 1378 if (attr_mask & IB_QP_MIN_RNR_TIMER && 1379 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) 1380 udp_info->min_rnr_timer = attr->min_rnr_timer; 1381 1382 if (attr_mask & IB_QP_RETRY_CNT) 1383 udp_info->rexmit_thresh = attr->retry_cnt; 1384 1385 ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; 1386 1387 if (attr_mask & IB_QP_AV) { 1388 struct irdma_av *av = &iwqp->roce_ah.av; 1389 const struct ib_gid_attr *sgid_attr = 1390 attr->ah_attr.grh.sgid_attr; 1391 u16 vlan_id = VLAN_N_VID; 1392 u32 local_ip[4]; 1393 1394 memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); 1395 if (attr->ah_attr.ah_flags & IB_AH_GRH) { 1396 udp_info->ttl = attr->ah_attr.grh.hop_limit; 1397 udp_info->flow_label = attr->ah_attr.grh.flow_label; 1398 udp_info->tos = attr->ah_attr.grh.traffic_class; 1399 udp_info->src_port = 1400 rdma_get_udp_sport(udp_info->flow_label, 1401 ibqp->qp_num, 1402 roce_info->dest_qp); 1403 irdma_qp_rem_qos(&iwqp->sc_qp); 1404 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); 1405 if (iwqp->sc_qp.vsi->dscp_mode) 1406 ctx_info->user_pri = 1407 iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(udp_info->tos)]; 1408 else 1409 ctx_info->user_pri = rt_tos2priority(udp_info->tos); 1410 } 1411 ret = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, 1412 ctx_info->roce_info->mac_addr); 1413 if (ret) 1414 return ret; 1415 ctx_info->user_pri = irdma_roce_get_vlan_prio(sgid_attr, 1416 ctx_info->user_pri); 1417 if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) 1418 return -ENOMEM; 1419 iwqp->sc_qp.user_pri = ctx_info->user_pri; 1420 irdma_qp_add_qos(&iwqp->sc_qp); 1421 1422 if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 1423 vlan_id = 0; 1424 if (vlan_id < VLAN_N_VID) { 1425 udp_info->insert_vlan_tag = true; 1426 udp_info->vlan_tag = vlan_id | 1427 ctx_info->user_pri << VLAN_PRIO_SHIFT; 1428 } else { 1429 udp_info->insert_vlan_tag = false; 1430 } 1431 1432 av->attrs = attr->ah_attr; 1433 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid); 1434 rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); 1435 av->net_type = rdma_gid_attr_network_type(sgid_attr); 1436 if (av->net_type == RDMA_NETWORK_IPV6) { 1437 __be32 *daddr = 1438 av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1439 __be32 *saddr = 1440 av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1441 1442 irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); 1443 irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); 1444 1445 udp_info->ipv4 = false; 1446 irdma_copy_ip_ntohl(local_ip, daddr); 1447 1448 } else if (av->net_type == RDMA_NETWORK_IPV4) { 1449 __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; 1450 __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; 1451 1452 local_ip[0] = ntohl(daddr); 1453 1454 udp_info->ipv4 = true; 1455 udp_info->dest_ip_addr[0] = 0; 1456 udp_info->dest_ip_addr[1] = 0; 1457 udp_info->dest_ip_addr[2] = 0; 1458 udp_info->dest_ip_addr[3] = local_ip[0]; 1459 1460 udp_info->local_ipaddr[0] = 0; 1461 udp_info->local_ipaddr[1] = 0; 1462 udp_info->local_ipaddr[2] = 0; 1463 udp_info->local_ipaddr[3] = ntohl(saddr); 1464 } 1465 udp_info->arp_idx = 1466 irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4, 1467 attr->ah_attr.roce.dmac); 1468 } 1469 1470 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1471 if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { 1472 ibdev_err(&iwdev->ibdev, 1473 "rd_atomic = %d, above max_hw_ord=%d\n", 1474 attr->max_rd_atomic, 1475 dev->hw_attrs.max_hw_ord); 1476 return -EINVAL; 1477 } 1478 if (attr->max_rd_atomic) 1479 roce_info->ord_size = attr->max_rd_atomic; 1480 info.ord_valid = true; 1481 } 1482 1483 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1484 if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { 1485 ibdev_err(&iwdev->ibdev, 1486 "rd_atomic = %d, above max_hw_ird=%d\n", 1487 attr->max_dest_rd_atomic, 1488 dev->hw_attrs.max_hw_ird); 1489 return -EINVAL; 1490 } 1491 if (attr->max_dest_rd_atomic) 1492 roce_info->ird_size = attr->max_dest_rd_atomic; 1493 } 1494 1495 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1496 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1497 roce_info->wr_rdresp_en = true; 1498 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1499 roce_info->wr_rdresp_en = true; 1500 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1501 roce_info->rd_en = true; 1502 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 1503 if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) 1504 ctx_info->remote_atomics_en = true; 1505 } 1506 1507 ibdev_dbg(&iwdev->ibdev, 1508 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", 1509 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1510 iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); 1511 1512 spin_lock_irqsave(&iwqp->lock, flags); 1513 if (attr_mask & IB_QP_STATE) { 1514 if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, 1515 iwqp->ibqp.qp_type, attr_mask)) { 1516 ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", 1517 iwqp->ibqp.qp_num, iwqp->ibqp_state, 1518 attr->qp_state); 1519 ret = -EINVAL; 1520 goto exit; 1521 } 1522 info.curr_iwarp_state = iwqp->iwarp_state; 1523 1524 switch (attr->qp_state) { 1525 case IB_QPS_INIT: 1526 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1527 ret = -EINVAL; 1528 goto exit; 1529 } 1530 1531 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1532 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1533 issue_modify_qp = 1; 1534 } 1535 break; 1536 case IB_QPS_RTR: 1537 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1538 ret = -EINVAL; 1539 goto exit; 1540 } 1541 info.arp_cache_idx_valid = true; 1542 info.cq_num_valid = true; 1543 info.next_iwarp_state = IRDMA_QP_STATE_RTR; 1544 issue_modify_qp = 1; 1545 break; 1546 case IB_QPS_RTS: 1547 if (iwqp->ibqp_state < IB_QPS_RTR || 1548 iwqp->ibqp_state == IB_QPS_ERR) { 1549 ret = -EINVAL; 1550 goto exit; 1551 } 1552 1553 info.arp_cache_idx_valid = true; 1554 info.cq_num_valid = true; 1555 info.ord_valid = true; 1556 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1557 issue_modify_qp = 1; 1558 if (iwdev->push_mode && udata && 1559 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1560 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1561 spin_unlock_irqrestore(&iwqp->lock, flags); 1562 irdma_alloc_push_page(iwqp); 1563 spin_lock_irqsave(&iwqp->lock, flags); 1564 } 1565 break; 1566 case IB_QPS_SQD: 1567 if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) 1568 goto exit; 1569 1570 if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { 1571 ret = -EINVAL; 1572 goto exit; 1573 } 1574 1575 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1576 issue_modify_qp = 1; 1577 iwqp->suspend_pending = true; 1578 break; 1579 case IB_QPS_RESET: 1580 /* 1581 * A QP that was never moved to INIT has nothing to 1582 * flush, so skip the hardware modify. Moving it to 1583 * ERROR would make the following INIT fail. 1584 */ 1585 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) 1586 break; 1587 fallthrough; 1588 case IB_QPS_SQE: 1589 case IB_QPS_ERR: 1590 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1591 iwqp->ibqp_state = attr->qp_state; 1592 spin_unlock_irqrestore(&iwqp->lock, flags); 1593 if (udata && udata->inlen) { 1594 irdma_flush_wqes(iwqp, 1595 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1596 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1597 IRDMA_REFLUSH); 1598 } 1599 return 0; 1600 } 1601 1602 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1603 issue_modify_qp = 1; 1604 break; 1605 default: 1606 ret = -EINVAL; 1607 goto exit; 1608 } 1609 1610 iwqp->ibqp_state = attr->qp_state; 1611 } 1612 1613 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1614 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1615 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1616 spin_unlock_irqrestore(&iwqp->lock, flags); 1617 1618 if (attr_mask & IB_QP_STATE) { 1619 if (issue_modify_qp) { 1620 ctx_info->rem_endpoint_idx = udp_info->arp_idx; 1621 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1622 return -EINVAL; 1623 if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) { 1624 ret = irdma_wait_for_suspend(iwqp); 1625 if (ret) 1626 return ret; 1627 } 1628 spin_lock_irqsave(&iwqp->lock, flags); 1629 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1630 iwqp->iwarp_state = info.next_iwarp_state; 1631 iwqp->ibqp_state = attr->qp_state; 1632 } 1633 if (iwqp->ibqp_state > IB_QPS_RTS && 1634 !iwqp->flush_issued) { 1635 spin_unlock_irqrestore(&iwqp->lock, flags); 1636 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | 1637 IRDMA_FLUSH_RQ | 1638 IRDMA_FLUSH_WAIT); 1639 iwqp->flush_issued = 1; 1640 } else { 1641 spin_unlock_irqrestore(&iwqp->lock, flags); 1642 } 1643 } else { 1644 iwqp->ibqp_state = attr->qp_state; 1645 } 1646 if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1647 struct irdma_ucontext *ucontext; 1648 1649 ucontext = rdma_udata_to_drv_context(udata, 1650 struct irdma_ucontext, ibucontext); 1651 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1652 !iwqp->push_wqe_mmap_entry && 1653 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1654 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1655 uresp.push_valid = 1; 1656 uresp.push_offset = iwqp->sc_qp.push_offset; 1657 } 1658 ret = ib_respond_udata(udata, uresp); 1659 if (ret) { 1660 irdma_remove_push_mmap_entries(iwqp); 1661 return ret; 1662 } 1663 } 1664 } 1665 1666 return 0; 1667 exit: 1668 spin_unlock_irqrestore(&iwqp->lock, flags); 1669 1670 return ret; 1671 } 1672 1673 /** 1674 * irdma_modify_qp - modify qp request 1675 * @ibqp: qp's pointer for modify 1676 * @attr: access attributes 1677 * @attr_mask: state mask 1678 * @udata: user data 1679 */ 1680 int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1681 struct ib_udata *udata) 1682 { 1683 #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) 1684 struct irdma_qp *iwqp = to_iwqp(ibqp); 1685 struct irdma_device *iwdev = iwqp->iwdev; 1686 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1687 struct irdma_qp_host_ctx_info *ctx_info; 1688 struct irdma_tcp_offload_info *tcp_info; 1689 struct irdma_iwarp_offload_info *offload_info; 1690 struct irdma_modify_qp_info info = {}; 1691 struct irdma_modify_qp_resp uresp = {}; 1692 struct irdma_modify_qp_req ureq = {}; 1693 u8 issue_modify_qp = 0; 1694 u8 dont_wait = 0; 1695 int err; 1696 unsigned long flags; 1697 1698 err = ib_respond_empty_udata(udata); 1699 if (err) 1700 return err; 1701 1702 if (udata) { 1703 /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ 1704 if (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN) 1705 return -EINVAL; 1706 1707 if (udata->inlen) { 1708 err = ib_copy_validate_udata_in(udata, ureq, rsvd); 1709 if (err) 1710 return err; 1711 } 1712 } 1713 1714 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1715 return -EOPNOTSUPP; 1716 1717 ctx_info = &iwqp->ctx_info; 1718 offload_info = &iwqp->iwarp_info; 1719 tcp_info = &iwqp->tcp_info; 1720 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1721 ibdev_dbg(&iwdev->ibdev, 1722 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", 1723 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1724 iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, 1725 iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); 1726 1727 spin_lock_irqsave(&iwqp->lock, flags); 1728 if (attr_mask & IB_QP_STATE) { 1729 info.curr_iwarp_state = iwqp->iwarp_state; 1730 switch (attr->qp_state) { 1731 case IB_QPS_INIT: 1732 case IB_QPS_RTR: 1733 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1734 err = -EINVAL; 1735 goto exit; 1736 } 1737 1738 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1739 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1740 issue_modify_qp = 1; 1741 } 1742 if (iwdev->push_mode && udata && 1743 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1744 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1745 spin_unlock_irqrestore(&iwqp->lock, flags); 1746 irdma_alloc_push_page(iwqp); 1747 spin_lock_irqsave(&iwqp->lock, flags); 1748 } 1749 break; 1750 case IB_QPS_RTS: 1751 if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || 1752 !iwqp->cm_id) { 1753 err = -EINVAL; 1754 goto exit; 1755 } 1756 1757 issue_modify_qp = 1; 1758 iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 1759 iwqp->hte_added = 1; 1760 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1761 info.tcp_ctx_valid = true; 1762 info.ord_valid = true; 1763 info.arp_cache_idx_valid = true; 1764 info.cq_num_valid = true; 1765 break; 1766 case IB_QPS_SQD: 1767 if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { 1768 err = 0; 1769 goto exit; 1770 } 1771 1772 if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || 1773 iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { 1774 err = 0; 1775 goto exit; 1776 } 1777 1778 if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { 1779 err = -EINVAL; 1780 goto exit; 1781 } 1782 1783 info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; 1784 issue_modify_qp = 1; 1785 break; 1786 case IB_QPS_SQE: 1787 if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { 1788 err = -EINVAL; 1789 goto exit; 1790 } 1791 1792 info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; 1793 issue_modify_qp = 1; 1794 break; 1795 case IB_QPS_ERR: 1796 case IB_QPS_RESET: 1797 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1798 iwqp->ibqp_state = attr->qp_state; 1799 spin_unlock_irqrestore(&iwqp->lock, flags); 1800 if (udata && udata->inlen) { 1801 irdma_flush_wqes(iwqp, 1802 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1803 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1804 IRDMA_REFLUSH); 1805 } 1806 return 0; 1807 } 1808 1809 if (iwqp->sc_qp.term_flags) { 1810 spin_unlock_irqrestore(&iwqp->lock, flags); 1811 irdma_terminate_del_timer(&iwqp->sc_qp); 1812 spin_lock_irqsave(&iwqp->lock, flags); 1813 } 1814 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1815 if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && 1816 iwdev->iw_status && 1817 iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) 1818 info.reset_tcp_conn = true; 1819 else 1820 dont_wait = 1; 1821 1822 issue_modify_qp = 1; 1823 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1824 break; 1825 default: 1826 err = -EINVAL; 1827 goto exit; 1828 } 1829 1830 iwqp->ibqp_state = attr->qp_state; 1831 } 1832 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1833 ctx_info->iwarp_info_valid = true; 1834 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1835 offload_info->wr_rdresp_en = true; 1836 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1837 offload_info->wr_rdresp_en = true; 1838 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1839 offload_info->rd_en = true; 1840 } 1841 1842 if (ctx_info->iwarp_info_valid) { 1843 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1844 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1845 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1846 } 1847 spin_unlock_irqrestore(&iwqp->lock, flags); 1848 1849 if (attr_mask & IB_QP_STATE) { 1850 if (issue_modify_qp) { 1851 ctx_info->rem_endpoint_idx = tcp_info->arp_idx; 1852 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1853 return -EINVAL; 1854 } 1855 1856 spin_lock_irqsave(&iwqp->lock, flags); 1857 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1858 iwqp->iwarp_state = info.next_iwarp_state; 1859 iwqp->ibqp_state = attr->qp_state; 1860 } 1861 spin_unlock_irqrestore(&iwqp->lock, flags); 1862 } 1863 1864 if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { 1865 if (dont_wait) { 1866 if (iwqp->hw_tcp_state) { 1867 spin_lock_irqsave(&iwqp->lock, flags); 1868 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1869 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1870 spin_unlock_irqrestore(&iwqp->lock, flags); 1871 } 1872 irdma_cm_disconn(iwqp); 1873 } else { 1874 int close_timer_started; 1875 1876 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 1877 1878 if (iwqp->cm_node) { 1879 refcount_inc(&iwqp->cm_node->refcnt); 1880 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1881 close_timer_started = atomic_inc_return(&iwqp->close_timer_started); 1882 if (iwqp->cm_id && close_timer_started == 1) 1883 irdma_schedule_cm_timer(iwqp->cm_node, 1884 (struct irdma_puda_buf *)iwqp, 1885 IRDMA_TIMER_TYPE_CLOSE, 1, 0); 1886 1887 irdma_rem_ref_cm_node(iwqp->cm_node); 1888 } else { 1889 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1890 } 1891 } 1892 } 1893 if (attr_mask & IB_QP_STATE && udata && udata->outlen && 1894 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1895 struct irdma_ucontext *ucontext; 1896 1897 ucontext = rdma_udata_to_drv_context(udata, 1898 struct irdma_ucontext, ibucontext); 1899 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1900 !iwqp->push_wqe_mmap_entry && 1901 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1902 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1903 uresp.push_valid = 1; 1904 uresp.push_offset = iwqp->sc_qp.push_offset; 1905 } 1906 1907 err = ib_respond_udata(udata, uresp); 1908 if (err) { 1909 irdma_remove_push_mmap_entries(iwqp); 1910 return err; 1911 } 1912 } 1913 1914 return 0; 1915 exit: 1916 spin_unlock_irqrestore(&iwqp->lock, flags); 1917 1918 return err; 1919 } 1920 1921 /** 1922 * irdma_srq_free_rsrc - free up resources for srq 1923 * @rf: RDMA PCI function 1924 * @iwsrq: srq ptr 1925 */ 1926 static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq) 1927 { 1928 struct irdma_sc_srq *srq = &iwsrq->sc_srq; 1929 1930 if (!iwsrq->user_mode) { 1931 dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size, 1932 iwsrq->kmem.va, iwsrq->kmem.pa); 1933 iwsrq->kmem.va = NULL; 1934 } else { 1935 /* Not called in any failure path, so iwpbl is valid. */ 1936 struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; 1937 1938 refcount_dec(&iwmr->user_ring_refs); 1939 } 1940 1941 irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id); 1942 } 1943 1944 /** 1945 * irdma_cq_free_rsrc - free up resources for cq 1946 * @rf: RDMA PCI function 1947 * @iwcq: cq ptr 1948 */ 1949 static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) 1950 { 1951 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1952 1953 if (!iwcq->user_mode) { 1954 dma_free_coherent(rf->sc_dev.hw->device, iwcq->kmem.size, 1955 iwcq->kmem.va, iwcq->kmem.pa); 1956 iwcq->kmem.va = NULL; 1957 dma_free_coherent(rf->sc_dev.hw->device, 1958 iwcq->kmem_shadow.size, 1959 iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); 1960 iwcq->kmem_shadow.va = NULL; 1961 } else { 1962 struct irdma_mr *iwmr; 1963 1964 /* May be called in a failure path before iwpbl is valid. */ 1965 if (iwcq->iwpbl) { 1966 iwmr = iwcq->iwpbl->iwmr; 1967 1968 refcount_dec(&iwmr->user_ring_refs); 1969 } 1970 1971 if (iwcq->iwpbl_shadow) { 1972 iwmr = iwcq->iwpbl_shadow->iwmr; 1973 1974 refcount_dec(&iwmr->user_ring_refs); 1975 } 1976 } 1977 1978 irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); 1979 } 1980 1981 /** 1982 * irdma_free_cqbuf - worker to free a cq buffer 1983 * @work: provides access to the cq buffer to free 1984 */ 1985 static void irdma_free_cqbuf(struct work_struct *work) 1986 { 1987 struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); 1988 1989 dma_free_coherent(cq_buf->hw->device, cq_buf->kmem_buf.size, 1990 cq_buf->kmem_buf.va, cq_buf->kmem_buf.pa); 1991 cq_buf->kmem_buf.va = NULL; 1992 kfree(cq_buf); 1993 } 1994 1995 /** 1996 * irdma_process_resize_list - remove resized cq buffers from the resize_list 1997 * @iwcq: cq which owns the resize_list 1998 * @iwdev: irdma device 1999 * @lcqe_buf: the buffer where the last cqe is received 2000 */ 2001 static int irdma_process_resize_list(struct irdma_cq *iwcq, 2002 struct irdma_device *iwdev, 2003 struct irdma_cq_buf *lcqe_buf) 2004 { 2005 struct list_head *tmp_node, *list_node; 2006 struct irdma_cq_buf *cq_buf; 2007 int cnt = 0; 2008 2009 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 2010 cq_buf = list_entry(list_node, struct irdma_cq_buf, list); 2011 if (cq_buf == lcqe_buf) 2012 return cnt; 2013 2014 list_del(&cq_buf->list); 2015 queue_work(iwdev->cleanup_wq, &cq_buf->work); 2016 cnt++; 2017 } 2018 2019 return cnt; 2020 } 2021 2022 /** 2023 * irdma_destroy_srq - destroy srq 2024 * @ibsrq: srq pointer 2025 * @udata: user data 2026 */ 2027 static int irdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 2028 { 2029 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 2030 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2031 struct irdma_sc_srq *srq = &iwsrq->sc_srq; 2032 int ret; 2033 2034 ret = ib_no_udata_io(udata); 2035 if (ret) 2036 return ret; 2037 2038 irdma_srq_wq_destroy(iwdev->rf, srq); 2039 irdma_srq_free_rsrc(iwdev->rf, iwsrq); 2040 return 0; 2041 } 2042 2043 /** 2044 * irdma_destroy_cq - destroy cq 2045 * @ib_cq: cq pointer 2046 * @udata: user data 2047 */ 2048 static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 2049 { 2050 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 2051 struct irdma_cq *iwcq = to_iwcq(ib_cq); 2052 struct irdma_sc_cq *cq = &iwcq->sc_cq; 2053 struct irdma_sc_dev *dev = cq->dev; 2054 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 2055 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 2056 unsigned long flags; 2057 int ret; 2058 2059 ret = ib_no_udata_io(udata); 2060 if (ret) 2061 return ret; 2062 2063 spin_lock_irqsave(&iwcq->lock, flags); 2064 if (!list_empty(&iwcq->cmpl_generated)) 2065 irdma_remove_cmpls_list(iwcq); 2066 if (!list_empty(&iwcq->resize_list)) 2067 irdma_process_resize_list(iwcq, iwdev, NULL); 2068 spin_unlock_irqrestore(&iwcq->lock, flags); 2069 2070 irdma_cq_rem_ref(ib_cq); 2071 wait_for_completion(&iwcq->free_cq); 2072 2073 irdma_cq_wq_destroy(iwdev->rf, cq); 2074 2075 spin_lock_irqsave(&iwceq->ce_lock, flags); 2076 irdma_sc_cleanup_ceqes(cq, ceq); 2077 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 2078 irdma_cq_free_rsrc(iwdev->rf, iwcq); 2079 2080 return 0; 2081 } 2082 2083 /** 2084 * irdma_resize_cq - resize cq 2085 * @ibcq: cq to be resized 2086 * @entries: desired cq size 2087 * @udata: user data 2088 */ 2089 static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, 2090 struct ib_udata *udata) 2091 { 2092 struct irdma_cq *iwcq = to_iwcq(ibcq); 2093 struct irdma_sc_dev *dev = iwcq->sc_cq.dev; 2094 struct irdma_cqp_request *cqp_request; 2095 struct cqp_cmds_info *cqp_info; 2096 struct irdma_modify_cq_info *m_info; 2097 struct irdma_modify_cq_info info = {}; 2098 struct irdma_dma_mem kmem_buf; 2099 struct irdma_cq_mr *cqmr_buf; 2100 struct irdma_pbl *iwpbl_buf = NULL; 2101 struct irdma_device *iwdev; 2102 struct irdma_pci_f *rf; 2103 struct irdma_cq_buf *cq_buf = NULL; 2104 unsigned long flags; 2105 u8 cqe_size; 2106 int ret; 2107 2108 iwdev = to_iwdev(ibcq->device); 2109 rf = iwdev->rf; 2110 2111 if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2112 IRDMA_FEATURE_CQ_RESIZE)) 2113 return -EOPNOTSUPP; 2114 2115 if (entries > rf->max_cqe) 2116 return -EINVAL; 2117 2118 ret = ib_respond_empty_udata(udata); 2119 if (ret) 2120 return ret; 2121 2122 if (!iwcq->user_mode) { 2123 entries += 2; 2124 2125 if (!iwcq->sc_cq.cq_uk.avoid_mem_cflct && 2126 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2127 entries *= 2; 2128 2129 if (entries & 1) 2130 entries += 1; /* cq size must be an even number */ 2131 2132 cqe_size = iwcq->sc_cq.cq_uk.avoid_mem_cflct ? 64 : 32; 2133 if (entries * cqe_size == IRDMA_HW_PAGE_SIZE) 2134 entries += 2; 2135 } 2136 2137 info.cq_size = max(entries, 4); 2138 2139 if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) 2140 return 0; 2141 2142 if (udata) { 2143 struct irdma_resize_cq_req req = {}; 2144 struct irdma_ucontext *ucontext = 2145 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2146 ibucontext); 2147 2148 ret = ib_copy_validate_udata_in(udata, req, user_cq_buffer); 2149 if (ret) 2150 return ret; 2151 2152 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2153 iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, 2154 &ucontext->cq_reg_mem_list); 2155 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2156 2157 if (!iwpbl_buf) 2158 return -ENOMEM; 2159 2160 cqmr_buf = &iwpbl_buf->cq_mr; 2161 if (iwpbl_buf->pbl_allocated) { 2162 info.virtual_map = true; 2163 info.pbl_chunk_size = 1; 2164 info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; 2165 } else { 2166 info.cq_pa = cqmr_buf->cq_pbl.addr; 2167 } 2168 } else { 2169 /* Kmode CQ resize */ 2170 int rsize; 2171 2172 rsize = info.cq_size * sizeof(struct irdma_cqe); 2173 kmem_buf.size = ALIGN(round_up(rsize, 256), 256); 2174 kmem_buf.va = dma_alloc_coherent(dev->hw->device, 2175 kmem_buf.size, &kmem_buf.pa, 2176 GFP_KERNEL); 2177 if (!kmem_buf.va) 2178 return -ENOMEM; 2179 2180 info.cq_base = kmem_buf.va; 2181 info.cq_pa = kmem_buf.pa; 2182 cq_buf = kzalloc_obj(*cq_buf); 2183 if (!cq_buf) { 2184 ret = -ENOMEM; 2185 goto error; 2186 } 2187 } 2188 2189 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2190 if (!cqp_request) { 2191 ret = -ENOMEM; 2192 goto error; 2193 } 2194 2195 info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; 2196 info.cq_resize = true; 2197 2198 cqp_info = &cqp_request->info; 2199 m_info = &cqp_info->in.u.cq_modify.info; 2200 memcpy(m_info, &info, sizeof(*m_info)); 2201 2202 cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; 2203 cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; 2204 cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; 2205 cqp_info->post_sq = 1; 2206 ret = irdma_handle_cqp_op(rf, cqp_request); 2207 irdma_put_cqp_request(&rf->cqp, cqp_request); 2208 if (ret) 2209 goto error; 2210 2211 spin_lock_irqsave(&iwcq->lock, flags); 2212 if (udata) { 2213 struct irdma_pbl *old_iwpbl = iwcq->iwpbl; 2214 2215 /* Only update if the resize was successful. Otherwise, HW is 2216 * still pointing to the old PBL. 2217 */ 2218 iwcq->iwpbl = iwpbl_buf; 2219 if (old_iwpbl) { 2220 struct irdma_mr *old_iwmr = old_iwpbl->iwmr; 2221 2222 refcount_dec(&old_iwmr->user_ring_refs); 2223 } 2224 } 2225 if (cq_buf) { 2226 cq_buf->kmem_buf = iwcq->kmem; 2227 cq_buf->hw = dev->hw; 2228 memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); 2229 INIT_WORK(&cq_buf->work, irdma_free_cqbuf); 2230 list_add_tail(&cq_buf->list, &iwcq->resize_list); 2231 iwcq->kmem = kmem_buf; 2232 } 2233 2234 irdma_sc_cq_resize(&iwcq->sc_cq, &info); 2235 ibcq->cqe = info.cq_size - 1; 2236 spin_unlock_irqrestore(&iwcq->lock, flags); 2237 2238 return 0; 2239 error: 2240 if (iwpbl_buf) { 2241 struct irdma_mr *iwmr = iwpbl_buf->iwmr; 2242 2243 refcount_dec(&iwmr->user_ring_refs); 2244 } 2245 if (!udata) { 2246 dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, 2247 kmem_buf.pa); 2248 kmem_buf.va = NULL; 2249 } 2250 kfree(cq_buf); 2251 2252 return ret; 2253 } 2254 2255 /** 2256 * irdma_srq_event - event notification for srq limit 2257 * @srq: shared srq struct 2258 */ 2259 void irdma_srq_event(struct irdma_sc_srq *srq) 2260 { 2261 struct irdma_srq *iwsrq = container_of(srq, struct irdma_srq, sc_srq); 2262 struct ib_srq *ibsrq = &iwsrq->ibsrq; 2263 struct ib_event event; 2264 2265 srq->srq_limit = 0; 2266 2267 if (!ibsrq->event_handler) 2268 return; 2269 2270 event.device = ibsrq->device; 2271 event.element.port_num = 1; 2272 event.element.srq = ibsrq; 2273 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 2274 ibsrq->event_handler(&event, ibsrq->srq_context); 2275 } 2276 2277 /** 2278 * irdma_modify_srq - modify srq request 2279 * @ibsrq: srq's pointer for modify 2280 * @attr: access attributes 2281 * @attr_mask: state mask 2282 * @udata: user data 2283 */ 2284 static int irdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 2285 enum ib_srq_attr_mask attr_mask, 2286 struct ib_udata *udata) 2287 { 2288 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 2289 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2290 struct irdma_cqp_request *cqp_request; 2291 struct irdma_pci_f *rf = iwdev->rf; 2292 struct irdma_modify_srq_info *info; 2293 struct cqp_cmds_info *cqp_info; 2294 int status; 2295 2296 status = ib_no_udata_io(udata); 2297 if (status) 2298 return status; 2299 2300 if (attr_mask & IB_SRQ_MAX_WR) 2301 return -EINVAL; 2302 2303 if (!(attr_mask & IB_SRQ_LIMIT)) 2304 return 0; 2305 2306 if (attr->srq_limit > iwsrq->sc_srq.srq_uk.srq_size) 2307 return -EINVAL; 2308 2309 /* Execute this cqp op synchronously, so we can update srq_limit 2310 * upon successful completion. 2311 */ 2312 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2313 if (!cqp_request) 2314 return -ENOMEM; 2315 2316 cqp_info = &cqp_request->info; 2317 info = &cqp_info->in.u.srq_modify.info; 2318 info->srq_limit = attr->srq_limit; 2319 if (info->srq_limit > 0xFFF) 2320 info->srq_limit = 0xFFF; 2321 info->arm_limit_event = 1; 2322 2323 cqp_info->cqp_cmd = IRDMA_OP_SRQ_MODIFY; 2324 cqp_info->post_sq = 1; 2325 cqp_info->in.u.srq_modify.srq = &iwsrq->sc_srq; 2326 cqp_info->in.u.srq_modify.scratch = (uintptr_t)cqp_request; 2327 status = irdma_handle_cqp_op(rf, cqp_request); 2328 irdma_put_cqp_request(&rf->cqp, cqp_request); 2329 if (status) 2330 return status; 2331 2332 iwsrq->sc_srq.srq_limit = info->srq_limit; 2333 2334 return 0; 2335 } 2336 2337 static int irdma_setup_umode_srq(struct irdma_device *iwdev, 2338 struct irdma_srq *iwsrq, 2339 struct irdma_srq_init_info *info, 2340 struct ib_udata *udata) 2341 { 2342 struct irdma_create_srq_req req = {}; 2343 struct irdma_ucontext *ucontext; 2344 struct irdma_srq_mr *srqmr; 2345 struct irdma_pbl *iwpbl; 2346 unsigned long flags; 2347 int ret; 2348 2349 iwsrq->user_mode = true; 2350 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2351 ibucontext); 2352 2353 ret = ib_copy_validate_udata_in(udata, req, user_shadow_area); 2354 if (ret) 2355 return ret; 2356 2357 spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); 2358 iwpbl = irdma_get_pbl((unsigned long)req.user_srq_buf, 2359 &ucontext->srq_reg_mem_list); 2360 spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); 2361 if (!iwpbl) 2362 return -EPROTO; 2363 2364 iwsrq->iwpbl = iwpbl; 2365 srqmr = &iwpbl->srq_mr; 2366 2367 if (iwpbl->pbl_allocated) { 2368 info->virtual_map = true; 2369 info->pbl_chunk_size = 1; 2370 info->first_pm_pbl_idx = srqmr->srq_pbl.idx; 2371 info->leaf_pbl_size = 1; 2372 } else { 2373 info->srq_pa = srqmr->srq_pbl.addr; 2374 } 2375 info->shadow_area_pa = srqmr->shadow; 2376 2377 return 0; 2378 } 2379 2380 static int irdma_setup_kmode_srq(struct irdma_device *iwdev, 2381 struct irdma_srq *iwsrq, 2382 struct irdma_srq_init_info *info, u32 depth, 2383 u8 shift) 2384 { 2385 struct irdma_srq_uk_init_info *ukinfo = &info->srq_uk_init_info; 2386 struct irdma_dma_mem *mem = &iwsrq->kmem; 2387 u32 size, ring_size; 2388 2389 ring_size = depth * IRDMA_QP_WQE_MIN_SIZE; 2390 size = ring_size + (IRDMA_SHADOW_AREA_SIZE << 3); 2391 2392 mem->size = ALIGN(size, 256); 2393 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 2394 &mem->pa, GFP_KERNEL); 2395 if (!mem->va) 2396 return -ENOMEM; 2397 2398 ukinfo->srq = mem->va; 2399 ukinfo->srq_size = depth >> shift; 2400 ukinfo->shadow_area = mem->va + ring_size; 2401 2402 info->srq_pa = mem->pa; 2403 info->shadow_area_pa = info->srq_pa + ring_size; 2404 2405 return 0; 2406 } 2407 2408 /** 2409 * irdma_create_srq - create srq 2410 * @ibsrq: ib's srq pointer 2411 * @initattrs: attributes for srq 2412 * @udata: user data for create srq 2413 */ 2414 static int irdma_create_srq(struct ib_srq *ibsrq, 2415 struct ib_srq_init_attr *initattrs, 2416 struct ib_udata *udata) 2417 { 2418 #define IRDMA_CREATE_SRQ_MIN_RESP_LEN offsetofend(struct irdma_create_srq_resp, srq_size) 2419 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 2420 struct ib_srq_attr *attr = &initattrs->attr; 2421 struct irdma_pd *iwpd = to_iwpd(ibsrq->pd); 2422 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2423 struct irdma_srq_uk_init_info *ukinfo; 2424 struct irdma_cqp_request *cqp_request; 2425 struct irdma_srq_init_info info = {}; 2426 struct irdma_pci_f *rf = iwdev->rf; 2427 struct irdma_uk_attrs *uk_attrs; 2428 struct cqp_cmds_info *cqp_info; 2429 int err_code = 0; 2430 u32 depth; 2431 u8 shift; 2432 2433 uk_attrs = &rf->sc_dev.hw_attrs.uk_attrs; 2434 ukinfo = &info.srq_uk_init_info; 2435 2436 if (initattrs->srq_type != IB_SRQT_BASIC) 2437 return -EOPNOTSUPP; 2438 2439 if (udata && udata->outlen < IRDMA_CREATE_SRQ_MIN_RESP_LEN) 2440 return -EINVAL; 2441 2442 if (!(uk_attrs->feature_flags & IRDMA_FEATURE_SRQ) || 2443 attr->max_sge > uk_attrs->max_hw_wq_frags) 2444 return -EINVAL; 2445 2446 refcount_set(&iwsrq->refcnt, 1); 2447 spin_lock_init(&iwsrq->lock); 2448 err_code = irdma_alloc_rsrc(rf, rf->allocated_srqs, rf->max_srq, 2449 &iwsrq->srq_num, &rf->next_srq); 2450 if (err_code) 2451 return err_code; 2452 2453 ukinfo->max_srq_frag_cnt = attr->max_sge; 2454 ukinfo->uk_attrs = uk_attrs; 2455 ukinfo->srq_id = iwsrq->srq_num; 2456 2457 irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_srq_frag_cnt, 0, 2458 &shift); 2459 2460 err_code = irdma_get_srqdepth(ukinfo->uk_attrs, attr->max_wr, 2461 shift, &depth); 2462 if (err_code) 2463 return err_code; 2464 2465 /* Actual SRQ size in WRs for ring and HW */ 2466 ukinfo->srq_size = depth >> shift; 2467 2468 /* Max postable WRs to SRQ */ 2469 iwsrq->max_wr = (depth - IRDMA_RQ_RSVD) >> shift; 2470 attr->max_wr = iwsrq->max_wr; 2471 2472 if (udata) 2473 err_code = irdma_setup_umode_srq(iwdev, iwsrq, &info, udata); 2474 else 2475 err_code = irdma_setup_kmode_srq(iwdev, iwsrq, &info, depth, 2476 shift); 2477 2478 if (err_code) 2479 goto free_rsrc; 2480 2481 info.vsi = &iwdev->vsi; 2482 info.pd = &iwpd->sc_pd; 2483 2484 iwsrq->sc_srq.srq_uk.lock = &iwsrq->lock; 2485 err_code = irdma_sc_srq_init(&iwsrq->sc_srq, &info); 2486 if (err_code) 2487 goto free_dmem; 2488 2489 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2490 if (!cqp_request) { 2491 err_code = -ENOMEM; 2492 goto free_dmem; 2493 } 2494 2495 cqp_info = &cqp_request->info; 2496 cqp_info->cqp_cmd = IRDMA_OP_SRQ_CREATE; 2497 cqp_info->post_sq = 1; 2498 cqp_info->in.u.srq_create.srq = &iwsrq->sc_srq; 2499 cqp_info->in.u.srq_create.scratch = (uintptr_t)cqp_request; 2500 err_code = irdma_handle_cqp_op(rf, cqp_request); 2501 irdma_put_cqp_request(&rf->cqp, cqp_request); 2502 if (err_code) 2503 goto free_dmem; 2504 2505 if (udata) { 2506 struct irdma_create_srq_resp resp = {}; 2507 2508 resp.srq_id = iwsrq->srq_num; 2509 resp.srq_size = ukinfo->srq_size; 2510 err_code = ib_respond_udata(udata, resp); 2511 if (err_code) 2512 goto srq_destroy; 2513 } 2514 2515 return 0; 2516 2517 srq_destroy: 2518 irdma_srq_wq_destroy(rf, &iwsrq->sc_srq); 2519 2520 free_dmem: 2521 if (!iwsrq->user_mode) 2522 dma_free_coherent(rf->hw.device, iwsrq->kmem.size, 2523 iwsrq->kmem.va, iwsrq->kmem.pa); 2524 free_rsrc: 2525 if (iwsrq->user_mode && iwsrq->iwpbl) { 2526 struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; 2527 2528 refcount_dec(&iwmr->user_ring_refs); 2529 } 2530 irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num); 2531 return err_code; 2532 } 2533 2534 /** 2535 * irdma_query_srq - get SRQ attributes 2536 * @ibsrq: the SRQ to query 2537 * @attr: the attributes of the SRQ 2538 */ 2539 static int irdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 2540 { 2541 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2542 2543 attr->max_wr = iwsrq->max_wr; 2544 attr->max_sge = iwsrq->sc_srq.srq_uk.max_srq_frag_cnt; 2545 attr->srq_limit = iwsrq->sc_srq.srq_limit; 2546 2547 return 0; 2548 } 2549 2550 static inline int cq_validate_flags(u32 flags, u8 hw_rev) 2551 { 2552 /* GEN1/2 does not support CQ create flags */ 2553 if (hw_rev <= IRDMA_GEN_2) 2554 return flags ? -EOPNOTSUPP : 0; 2555 2556 return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; 2557 } 2558 2559 /** 2560 * irdma_create_cq - create cq 2561 * @ibcq: CQ allocated 2562 * @attr: attributes for cq 2563 * @attrs: uverbs attribute bundle 2564 */ 2565 static int irdma_create_cq(struct ib_cq *ibcq, 2566 const struct ib_cq_init_attr *attr, 2567 struct uverbs_attr_bundle *attrs) 2568 { 2569 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) 2570 struct ib_udata *udata = &attrs->driver_udata; 2571 struct ib_device *ibdev = ibcq->device; 2572 struct irdma_device *iwdev = to_iwdev(ibdev); 2573 struct irdma_pci_f *rf = iwdev->rf; 2574 struct irdma_cq *iwcq = to_iwcq(ibcq); 2575 u32 cq_num = 0; 2576 struct irdma_sc_cq *cq; 2577 struct irdma_sc_dev *dev = &rf->sc_dev; 2578 struct irdma_cq_init_info info = {}; 2579 struct irdma_cqp_request *cqp_request; 2580 struct cqp_cmds_info *cqp_info; 2581 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 2582 unsigned long flags; 2583 int err_code; 2584 int entries = attr->cqe; 2585 bool cqe_64byte_ena; 2586 u8 cqe_size; 2587 2588 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 2589 if (err_code) 2590 return err_code; 2591 2592 if (udata && udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN) 2593 return -EINVAL; 2594 2595 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 2596 &rf->next_cq); 2597 if (err_code) 2598 return err_code; 2599 2600 cq = &iwcq->sc_cq; 2601 cq->back_cq = iwcq; 2602 refcount_set(&iwcq->refcnt, 1); 2603 spin_lock_init(&iwcq->lock); 2604 INIT_LIST_HEAD(&iwcq->resize_list); 2605 INIT_LIST_HEAD(&iwcq->cmpl_generated); 2606 iwcq->cq_num = cq_num; 2607 iwcq->iwpbl = NULL; 2608 iwcq->iwpbl_shadow = NULL; 2609 info.dev = dev; 2610 ukinfo->cq_size = max(entries, 4); 2611 ukinfo->cq_id = cq_num; 2612 cqe_64byte_ena = dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? 2613 true : false; 2614 cqe_size = cqe_64byte_ena ? 64 : 32; 2615 ukinfo->avoid_mem_cflct = cqe_64byte_ena; 2616 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 2617 if (attr->comp_vector < rf->ceqs_count) 2618 info.ceq_id = attr->comp_vector; 2619 info.ceq_id_valid = true; 2620 info.ceqe_mask = 1; 2621 info.type = IRDMA_CQ_TYPE_IWARP; 2622 info.vsi = &iwdev->vsi; 2623 2624 if (udata) { 2625 struct irdma_ucontext *ucontext; 2626 struct irdma_create_cq_req req = {}; 2627 struct irdma_cq_mr *cqmr; 2628 struct irdma_cq_mr *cqmr_shadow; 2629 2630 iwcq->user_mode = true; 2631 ucontext = 2632 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2633 ibucontext); 2634 /* Even though the last member of struct irdma_create_cq_req 2635 * was always user_shadow_area, we need backwards compat with 2636 * the legacy i40iw struct i40iw_ucreate_cq which stopped 2637 * at user_cq_buffer. 2638 */ 2639 err_code = ib_copy_validate_udata_in(udata, req, user_cq_buf); 2640 if (err_code) 2641 goto cq_free_rsrc; 2642 2643 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2644 iwcq->iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 2645 &ucontext->cq_reg_mem_list); 2646 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2647 if (!iwcq->iwpbl) { 2648 err_code = -EPROTO; 2649 goto cq_free_rsrc; 2650 } 2651 2652 cqmr = &iwcq->iwpbl->cq_mr; 2653 2654 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2655 IRDMA_FEATURE_CQ_RESIZE) { 2656 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2657 iwcq->iwpbl_shadow = irdma_get_pbl( 2658 (unsigned long)req.user_shadow_area, 2659 &ucontext->cq_reg_mem_list); 2660 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2661 2662 if (!iwcq->iwpbl_shadow) { 2663 err_code = -EPROTO; 2664 goto cq_free_rsrc; 2665 } 2666 cqmr_shadow = &iwcq->iwpbl_shadow->cq_mr; 2667 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 2668 } else { 2669 info.shadow_area_pa = cqmr->shadow; 2670 } 2671 if (iwcq->iwpbl->pbl_allocated) { 2672 info.virtual_map = true; 2673 info.pbl_chunk_size = 1; 2674 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 2675 } else { 2676 info.cq_base_pa = cqmr->cq_pbl.addr; 2677 } 2678 } else { 2679 /* Kmode allocations */ 2680 int rsize; 2681 2682 if (entries < 1 || entries > rf->max_cqe) { 2683 err_code = -EINVAL; 2684 goto cq_free_rsrc; 2685 } 2686 2687 entries += 2; 2688 if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2689 entries *= 2; 2690 2691 if (entries & 1) 2692 entries += 1; /* cq size must be an even number */ 2693 2694 if (entries * cqe_size == IRDMA_HW_PAGE_SIZE) 2695 entries += 2; 2696 2697 ukinfo->cq_size = entries; 2698 2699 if (cqe_64byte_ena) 2700 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe); 2701 else 2702 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 2703 iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256); 2704 iwcq->kmem.va = dma_alloc_coherent(dev->hw->device, 2705 iwcq->kmem.size, 2706 &iwcq->kmem.pa, GFP_KERNEL); 2707 if (!iwcq->kmem.va) { 2708 err_code = -ENOMEM; 2709 goto cq_free_rsrc; 2710 } 2711 2712 iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, 2713 64); 2714 iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device, 2715 iwcq->kmem_shadow.size, 2716 &iwcq->kmem_shadow.pa, 2717 GFP_KERNEL); 2718 if (!iwcq->kmem_shadow.va) { 2719 err_code = -ENOMEM; 2720 goto cq_free_rsrc; 2721 } 2722 info.shadow_area_pa = iwcq->kmem_shadow.pa; 2723 ukinfo->shadow_area = iwcq->kmem_shadow.va; 2724 ukinfo->cq_base = iwcq->kmem.va; 2725 info.cq_base_pa = iwcq->kmem.pa; 2726 } 2727 2728 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 2729 (u32)IRDMA_MAX_CQ_READ_THRESH); 2730 2731 if (irdma_sc_cq_init(cq, &info)) { 2732 ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n"); 2733 err_code = -EPROTO; 2734 goto cq_free_rsrc; 2735 } 2736 2737 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2738 if (!cqp_request) { 2739 err_code = -ENOMEM; 2740 goto cq_free_rsrc; 2741 } 2742 2743 cqp_info = &cqp_request->info; 2744 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 2745 cqp_info->post_sq = 1; 2746 cqp_info->in.u.cq_create.cq = cq; 2747 cqp_info->in.u.cq_create.check_overflow = true; 2748 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 2749 err_code = irdma_handle_cqp_op(rf, cqp_request); 2750 irdma_put_cqp_request(&rf->cqp, cqp_request); 2751 if (err_code) 2752 goto cq_free_rsrc; 2753 2754 if (udata) { 2755 struct irdma_create_cq_resp resp = {}; 2756 2757 resp.cq_id = info.cq_uk_init_info.cq_id; 2758 resp.cq_size = info.cq_uk_init_info.cq_size; 2759 err_code = ib_respond_udata(udata, resp); 2760 if (err_code) 2761 goto cq_destroy; 2762 } 2763 2764 init_completion(&iwcq->free_cq); 2765 2766 /* Populate table entry after CQ is fully created. */ 2767 smp_store_release(&rf->cq_table[cq_num], iwcq); 2768 2769 return 0; 2770 cq_destroy: 2771 irdma_cq_wq_destroy(rf, cq); 2772 cq_free_rsrc: 2773 irdma_cq_free_rsrc(rf, iwcq); 2774 2775 return err_code; 2776 } 2777 2778 /** 2779 * irdma_get_mr_access - get hw MR access permissions from IB access flags 2780 * @access: IB access flags 2781 * @hw_rev: Hardware version 2782 */ 2783 static inline u16 irdma_get_mr_access(int access, u8 hw_rev) 2784 { 2785 u16 hw_access = 0; 2786 2787 hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? 2788 IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; 2789 hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? 2790 IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; 2791 hw_access |= (access & IB_ACCESS_REMOTE_READ) ? 2792 IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; 2793 if (hw_rev >= IRDMA_GEN_3) { 2794 hw_access |= (access & IB_ACCESS_MW_BIND) ? 2795 IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; 2796 } 2797 hw_access |= (access & IB_ZERO_BASED) ? 2798 IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; 2799 hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; 2800 2801 return hw_access; 2802 } 2803 2804 /** 2805 * irdma_free_stag - free stag resource 2806 * @iwdev: irdma device 2807 * @stag: stag to free 2808 */ 2809 static void irdma_free_stag(struct irdma_device *iwdev, u32 stag) 2810 { 2811 u32 stag_idx; 2812 2813 stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; 2814 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); 2815 } 2816 2817 /** 2818 * irdma_create_stag - create random stag 2819 * @iwdev: irdma device 2820 */ 2821 static u32 irdma_create_stag(struct irdma_device *iwdev) 2822 { 2823 u32 stag = 0; 2824 u32 stag_index = 0; 2825 u32 next_stag_index; 2826 u32 driver_key; 2827 u32 random; 2828 u8 consumer_key; 2829 int ret; 2830 2831 get_random_bytes(&random, sizeof(random)); 2832 consumer_key = (u8)random; 2833 2834 driver_key = random & ~iwdev->rf->mr_stagmask; 2835 next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; 2836 next_stag_index %= iwdev->rf->max_mr; 2837 2838 ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, 2839 iwdev->rf->max_mr, &stag_index, 2840 &next_stag_index); 2841 if (ret) 2842 return stag; 2843 stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; 2844 stag |= driver_key; 2845 stag += (u32)consumer_key; 2846 2847 return stag; 2848 } 2849 2850 /** 2851 * irdma_next_pbl_addr - Get next pbl address 2852 * @pbl: pointer to a pble 2853 * @pinfo: info pointer 2854 * @idx: index 2855 */ 2856 static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, 2857 u32 *idx) 2858 { 2859 *idx += 1; 2860 if (!(*pinfo) || *idx != (*pinfo)->cnt) 2861 return ++pbl; 2862 *idx = 0; 2863 (*pinfo)++; 2864 2865 return (*pinfo)->addr; 2866 } 2867 2868 /** 2869 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 2870 * @iwmr: iwmr for IB's user page addresses 2871 * @pbl: ple pointer to save 1 level or 0 level pble 2872 * @pbl_len: Max number of PBL entries to populate 2873 * @level: indicated level 0, 1 or 2 2874 */ 2875 static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 2876 u32 pbl_len, enum irdma_pble_level level) 2877 { 2878 struct ib_umem *region = iwmr->region; 2879 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2880 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2881 struct irdma_pble_info *pinfo; 2882 struct ib_block_iter biter; 2883 u32 idx = 0; 2884 2885 if (!pbl_len) 2886 return; 2887 2888 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 2889 2890 if (iwmr->type == IRDMA_MEMREG_TYPE_QP) 2891 iwpbl->qp_mr.sq_page = sg_page(region->sgt_append.sgt.sgl); 2892 2893 rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { 2894 *pbl = rdma_block_iter_dma_address(&biter); 2895 if (!--pbl_len) 2896 break; 2897 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 2898 } 2899 } 2900 2901 /** 2902 * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous 2903 * @arr: lvl1 pbl array 2904 * @npages: page count 2905 * @pg_size: page size 2906 * 2907 */ 2908 static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) 2909 { 2910 u32 pg_idx; 2911 2912 for (pg_idx = 0; pg_idx < npages; pg_idx++) { 2913 if ((*arr + ((u64)pg_size * pg_idx)) != arr[pg_idx]) 2914 return false; 2915 } 2916 2917 return true; 2918 } 2919 2920 /** 2921 * irdma_setup_pbles - copy user pg address to pble's 2922 * @rf: RDMA PCI function 2923 * @iwmr: mr pointer for this memory registration 2924 * @lvl: requested pble levels 2925 */ 2926 static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, 2927 u8 lvl) 2928 { 2929 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2930 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2931 struct irdma_pble_info *pinfo; 2932 u64 *pbl; 2933 int status; 2934 enum irdma_pble_level level = PBLE_LEVEL_1; 2935 u32 pbl_len; 2936 2937 if (lvl) { 2938 status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, 2939 lvl); 2940 if (status) 2941 return status; 2942 2943 pbl_len = palloc->total_cnt; 2944 iwpbl->pbl_allocated = true; 2945 level = palloc->level; 2946 pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : 2947 palloc->level2.leaf; 2948 pbl = pinfo->addr; 2949 } else { 2950 pbl_len = IRDMA_MAX_SAVED_PHY_PGADDR; 2951 pbl = iwmr->pgaddrmem; 2952 } 2953 2954 irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level); 2955 2956 return 0; 2957 } 2958 2959 /** 2960 * irdma_handle_q_mem - handle memory for qp and cq 2961 * @iwdev: irdma device 2962 * @req: information for q memory management 2963 * @iwpbl: pble struct 2964 * @lvl: pble level mask 2965 */ 2966 static int irdma_handle_q_mem(struct irdma_device *iwdev, 2967 struct irdma_mem_reg_req *req, 2968 struct irdma_pbl *iwpbl, u8 lvl) 2969 { 2970 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2971 struct irdma_mr *iwmr = iwpbl->iwmr; 2972 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 2973 struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; 2974 struct irdma_srq_mr *srqmr = &iwpbl->srq_mr; 2975 struct irdma_hmc_pble *hmc_p; 2976 u64 *arr = iwmr->pgaddrmem; 2977 u32 pg_size, total; 2978 int err = 0; 2979 bool ret = true; 2980 2981 pg_size = iwmr->page_size; 2982 err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); 2983 if (err) 2984 return err; 2985 2986 if (lvl) 2987 arr = palloc->level1.addr; 2988 2989 switch (iwmr->type) { 2990 case IRDMA_MEMREG_TYPE_QP: 2991 total = req->sq_pages + req->rq_pages; 2992 hmc_p = &qpmr->sq_pbl; 2993 qpmr->shadow = (dma_addr_t)arr[total]; 2994 /* Need to use physical address for RQ of QP 2995 * in case it is associated with SRQ. 2996 */ 2997 qpmr->rq_pa = (dma_addr_t)arr[req->sq_pages]; 2998 if (lvl) { 2999 ret = irdma_check_mem_contiguous(arr, req->sq_pages, 3000 pg_size); 3001 if (ret) 3002 ret = irdma_check_mem_contiguous(&arr[req->sq_pages], 3003 req->rq_pages, 3004 pg_size); 3005 } 3006 3007 if (!ret) { 3008 hmc_p->idx = palloc->level1.idx; 3009 hmc_p = &qpmr->rq_pbl; 3010 hmc_p->idx = palloc->level1.idx + req->sq_pages; 3011 } else { 3012 hmc_p->addr = arr[0]; 3013 hmc_p = &qpmr->rq_pbl; 3014 hmc_p->addr = arr[req->sq_pages]; 3015 } 3016 break; 3017 case IRDMA_MEMREG_TYPE_SRQ: 3018 hmc_p = &srqmr->srq_pbl; 3019 srqmr->shadow = (dma_addr_t)arr[req->rq_pages]; 3020 if (lvl) 3021 ret = irdma_check_mem_contiguous(arr, req->rq_pages, 3022 pg_size); 3023 3024 if (!ret) 3025 hmc_p->idx = palloc->level1.idx; 3026 else 3027 hmc_p->addr = arr[0]; 3028 break; 3029 case IRDMA_MEMREG_TYPE_CQ: 3030 hmc_p = &cqmr->cq_pbl; 3031 3032 if (!(iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 3033 IRDMA_FEATURE_CQ_RESIZE)) 3034 cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; 3035 3036 if (lvl) 3037 ret = irdma_check_mem_contiguous(arr, req->cq_pages, 3038 pg_size); 3039 3040 if (!ret) 3041 hmc_p->idx = palloc->level1.idx; 3042 else 3043 hmc_p->addr = arr[0]; 3044 break; 3045 default: 3046 ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n"); 3047 err = -EINVAL; 3048 } 3049 3050 if (lvl && ret) { 3051 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3052 iwpbl->pbl_allocated = false; 3053 } 3054 3055 return err; 3056 } 3057 3058 /** 3059 * irdma_hw_alloc_mw - create the hw memory window 3060 * @iwdev: irdma device 3061 * @iwmr: pointer to memory window info 3062 */ 3063 static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) 3064 { 3065 struct irdma_mw_alloc_info *info; 3066 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 3067 struct irdma_cqp_request *cqp_request; 3068 struct cqp_cmds_info *cqp_info; 3069 int status; 3070 3071 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3072 if (!cqp_request) 3073 return -ENOMEM; 3074 3075 cqp_info = &cqp_request->info; 3076 info = &cqp_info->in.u.mw_alloc.info; 3077 memset(info, 0, sizeof(*info)); 3078 if (iwmr->ibmw.type == IB_MW_TYPE_1) 3079 info->mw_wide = true; 3080 3081 info->page_size = PAGE_SIZE; 3082 info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3083 info->pd_id = iwpd->sc_pd.pd_id; 3084 info->remote_access = true; 3085 cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; 3086 cqp_info->post_sq = 1; 3087 cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; 3088 cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; 3089 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3090 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3091 3092 return status; 3093 } 3094 3095 /** 3096 * irdma_alloc_mw - Allocate memory window 3097 * @ibmw: Memory Window 3098 * @udata: user data pointer 3099 */ 3100 static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 3101 { 3102 struct irdma_device *iwdev = to_iwdev(ibmw->device); 3103 struct irdma_mr *iwmr = to_iwmw(ibmw); 3104 int err_code; 3105 u32 stag; 3106 3107 err_code = ib_no_udata_io(udata); 3108 if (err_code) 3109 return err_code; 3110 3111 stag = irdma_create_stag(iwdev); 3112 if (!stag) 3113 return -ENOMEM; 3114 3115 iwmr->stag = stag; 3116 ibmw->rkey = stag; 3117 3118 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 3119 if (err_code) { 3120 irdma_free_stag(iwdev, stag); 3121 return err_code; 3122 } 3123 3124 return 0; 3125 } 3126 3127 /** 3128 * irdma_dealloc_mw - Dealloc memory window 3129 * @ibmw: memory window structure. 3130 */ 3131 static int irdma_dealloc_mw(struct ib_mw *ibmw) 3132 { 3133 struct ib_pd *ibpd = ibmw->pd; 3134 struct irdma_pd *iwpd = to_iwpd(ibpd); 3135 struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); 3136 struct irdma_device *iwdev = to_iwdev(ibmw->device); 3137 struct irdma_cqp_request *cqp_request; 3138 struct cqp_cmds_info *cqp_info; 3139 struct irdma_dealloc_stag_info *info; 3140 3141 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3142 if (!cqp_request) 3143 return -ENOMEM; 3144 3145 cqp_info = &cqp_request->info; 3146 info = &cqp_info->in.u.dealloc_stag.info; 3147 memset(info, 0, sizeof(*info)); 3148 info->pd_id = iwpd->sc_pd.pd_id; 3149 info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3150 info->mr = false; 3151 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3152 cqp_info->post_sq = 1; 3153 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3154 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3155 irdma_handle_cqp_op(iwdev->rf, cqp_request); 3156 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3157 irdma_free_stag(iwdev, iwmr->stag); 3158 3159 return 0; 3160 } 3161 3162 /** 3163 * irdma_hw_alloc_stag - cqp command to allocate stag 3164 * @iwdev: irdma device 3165 * @iwmr: irdma mr pointer 3166 */ 3167 static int irdma_hw_alloc_stag(struct irdma_device *iwdev, 3168 struct irdma_mr *iwmr) 3169 { 3170 struct irdma_allocate_stag_info *info; 3171 struct ib_pd *pd = iwmr->ibmr.pd; 3172 struct irdma_pd *iwpd = to_iwpd(pd); 3173 int status; 3174 struct irdma_cqp_request *cqp_request; 3175 struct cqp_cmds_info *cqp_info; 3176 3177 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3178 if (!cqp_request) 3179 return -ENOMEM; 3180 3181 cqp_info = &cqp_request->info; 3182 info = &cqp_info->in.u.alloc_stag.info; 3183 info->page_size = PAGE_SIZE; 3184 info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3185 info->pd_id = iwpd->sc_pd.pd_id; 3186 info->total_len = iwmr->len; 3187 info->remote_access = true; 3188 cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; 3189 cqp_info->post_sq = 1; 3190 cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; 3191 cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; 3192 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3193 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3194 if (status) 3195 return status; 3196 3197 iwmr->is_hwreg = true; 3198 return 0; 3199 } 3200 3201 /** 3202 * irdma_alloc_mr - register stag for fast memory registration 3203 * @pd: ibpd pointer 3204 * @mr_type: memory for stag registrion 3205 * @max_num_sg: man number of pages 3206 */ 3207 static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 3208 u32 max_num_sg) 3209 { 3210 struct irdma_device *iwdev = to_iwdev(pd->device); 3211 struct irdma_pble_alloc *palloc; 3212 struct irdma_pbl *iwpbl; 3213 struct irdma_mr *iwmr; 3214 u32 stag; 3215 int err_code; 3216 3217 iwmr = kzalloc_obj(*iwmr); 3218 if (!iwmr) 3219 return ERR_PTR(-ENOMEM); 3220 3221 stag = irdma_create_stag(iwdev); 3222 if (!stag) { 3223 err_code = -ENOMEM; 3224 goto err; 3225 } 3226 3227 iwmr->stag = stag; 3228 iwmr->ibmr.rkey = stag; 3229 iwmr->ibmr.lkey = stag; 3230 iwmr->ibmr.pd = pd; 3231 iwmr->ibmr.device = pd->device; 3232 iwpbl = &iwmr->iwpbl; 3233 iwpbl->iwmr = iwmr; 3234 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 3235 palloc = &iwpbl->pble_alloc; 3236 iwmr->page_cnt = max_num_sg; 3237 /* Use system PAGE_SIZE as the sg page sizes are unknown at this point */ 3238 iwmr->len = max_num_sg * PAGE_SIZE; 3239 err_code = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 3240 false); 3241 if (err_code) 3242 goto err_get_pble; 3243 3244 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 3245 if (err_code) 3246 goto err_alloc_stag; 3247 3248 iwpbl->pbl_allocated = true; 3249 3250 return &iwmr->ibmr; 3251 err_alloc_stag: 3252 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3253 err_get_pble: 3254 irdma_free_stag(iwdev, stag); 3255 err: 3256 kfree(iwmr); 3257 3258 return ERR_PTR(err_code); 3259 } 3260 3261 /** 3262 * irdma_set_page - populate pbl list for fmr 3263 * @ibmr: ib mem to access iwarp mr pointer 3264 * @addr: page dma address fro pbl list 3265 */ 3266 static int irdma_set_page(struct ib_mr *ibmr, u64 addr) 3267 { 3268 struct irdma_mr *iwmr = to_iwmr(ibmr); 3269 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3270 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3271 u64 *pbl; 3272 3273 if (unlikely(iwmr->npages == iwmr->page_cnt)) 3274 return -ENOMEM; 3275 3276 if (palloc->level == PBLE_LEVEL_2) { 3277 struct irdma_pble_info *palloc_info = 3278 palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT); 3279 3280 palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr; 3281 } else { 3282 pbl = palloc->level1.addr; 3283 pbl[iwmr->npages] = addr; 3284 } 3285 iwmr->npages++; 3286 3287 return 0; 3288 } 3289 3290 /** 3291 * irdma_map_mr_sg - map of sg list for fmr 3292 * @ibmr: ib mem to access iwarp mr pointer 3293 * @sg: scatter gather list 3294 * @sg_nents: number of sg pages 3295 * @sg_offset: scatter gather list for fmr 3296 */ 3297 static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3298 int sg_nents, unsigned int *sg_offset) 3299 { 3300 struct irdma_mr *iwmr = to_iwmr(ibmr); 3301 3302 iwmr->npages = 0; 3303 3304 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); 3305 } 3306 3307 /** 3308 * irdma_hwreg_mr - send cqp command for memory registration 3309 * @iwdev: irdma device 3310 * @iwmr: irdma mr pointer 3311 * @access: access for MR 3312 */ 3313 static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, 3314 u16 access) 3315 { 3316 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3317 struct irdma_reg_ns_stag_info *stag_info; 3318 struct ib_pd *pd = iwmr->ibmr.pd; 3319 struct irdma_pd *iwpd = to_iwpd(pd); 3320 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3321 struct irdma_cqp_request *cqp_request; 3322 struct cqp_cmds_info *cqp_info; 3323 int ret; 3324 3325 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3326 if (!cqp_request) 3327 return -ENOMEM; 3328 3329 cqp_info = &cqp_request->info; 3330 stag_info = &cqp_info->in.u.mr_reg_non_shared.info; 3331 stag_info->va = iwpbl->user_base; 3332 stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3333 stag_info->stag_key = (u8)iwmr->stag; 3334 stag_info->total_len = iwmr->len; 3335 stag_info->access_rights = irdma_get_mr_access(access, 3336 iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev); 3337 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 3338 stag_info->remote_atomics_en = (access & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3339 stag_info->pd_id = iwpd->sc_pd.pd_id; 3340 stag_info->all_memory = iwmr->dma_mr; 3341 if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) 3342 stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; 3343 else 3344 stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; 3345 stag_info->page_size = iwmr->page_size; 3346 3347 if (iwpbl->pbl_allocated) { 3348 if (palloc->level == PBLE_LEVEL_1) { 3349 stag_info->first_pm_pbl_index = palloc->level1.idx; 3350 stag_info->chunk_size = 1; 3351 } else { 3352 stag_info->first_pm_pbl_index = palloc->level2.root.idx; 3353 stag_info->chunk_size = 3; 3354 } 3355 } else { 3356 stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; 3357 } 3358 3359 cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; 3360 cqp_info->post_sq = 1; 3361 cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; 3362 cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; 3363 ret = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3364 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3365 3366 if (!ret) 3367 iwmr->is_hwreg = true; 3368 3369 return ret; 3370 } 3371 3372 static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, 3373 bool create_stag) 3374 { 3375 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3376 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3377 struct ib_umem *region = iwmr->region; 3378 u32 stag = 0; 3379 int err; 3380 3381 iwmr->access = access; 3382 3383 if (ib_umem_is_contiguous(region)) { 3384 /* HW needs a valid page size even without PBLEs, so use 4K. It 3385 * behaves like a PBLE-backed region where the PBA must be page 3386 * aligned. HW takes the page offset from the lower 12 VA bits. 3387 */ 3388 iwmr->page_size = SZ_4K; 3389 iwmr->pgaddrmem[0] = ALIGN_DOWN(ib_umem_start_dma_addr(region), 3390 iwmr->page_size); 3391 } else { 3392 iwmr->page_size = ib_umem_find_best_pgsz(region, 3393 iwdev->rf->sc_dev.hw_attrs.page_size_cap, 3394 iwmr->ibmr.iova); 3395 if (unlikely(!iwmr->page_size)) 3396 return -EOPNOTSUPP; 3397 3398 iwmr->page_cnt = ib_umem_num_dma_blocks(region, 3399 iwmr->page_size); 3400 err = irdma_setup_pbles(iwdev->rf, iwmr, 3401 PBLE_LEVEL_1 | PBLE_LEVEL_2); 3402 if (err) 3403 return err; 3404 } 3405 3406 if (create_stag) { 3407 stag = irdma_create_stag(iwdev); 3408 if (!stag) { 3409 err = -ENOMEM; 3410 goto free_pble; 3411 } 3412 3413 iwmr->stag = stag; 3414 iwmr->ibmr.rkey = stag; 3415 iwmr->ibmr.lkey = stag; 3416 } 3417 3418 err = irdma_hwreg_mr(iwdev, iwmr, access); 3419 if (err) 3420 goto err_hwreg; 3421 3422 return 0; 3423 3424 err_hwreg: 3425 if (stag) 3426 irdma_free_stag(iwdev, stag); 3427 3428 free_pble: 3429 if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 3430 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 3431 3432 return err; 3433 } 3434 3435 static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, 3436 struct ib_pd *pd, u64 virt, 3437 enum irdma_memreg_type reg_type) 3438 { 3439 struct irdma_pbl *iwpbl; 3440 struct irdma_mr *iwmr; 3441 3442 iwmr = kzalloc_obj(*iwmr); 3443 if (!iwmr) 3444 return ERR_PTR(-ENOMEM); 3445 3446 refcount_set(&iwmr->user_ring_refs, 1); 3447 iwpbl = &iwmr->iwpbl; 3448 iwpbl->iwmr = iwmr; 3449 iwmr->region = region; 3450 iwmr->ibmr.pd = pd; 3451 iwmr->ibmr.device = pd->device; 3452 iwmr->ibmr.iova = virt; 3453 iwmr->type = reg_type; 3454 iwmr->len = region->length; 3455 iwpbl->user_base = virt; 3456 3457 /* Page size and count are calculated in irdma_reg_user_mr_type_mem() 3458 * for MEM type MRs. 3459 */ 3460 if (reg_type == IRDMA_MEMREG_TYPE_MEM) 3461 return iwmr; 3462 3463 iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K, virt); 3464 if (unlikely(!iwmr->page_size)) { 3465 kfree(iwmr); 3466 return ERR_PTR(-EOPNOTSUPP); 3467 } 3468 3469 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 3470 3471 return iwmr; 3472 } 3473 3474 static void irdma_free_iwmr(struct irdma_mr *iwmr) 3475 { 3476 kfree(iwmr); 3477 } 3478 3479 static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req, 3480 struct ib_udata *udata, 3481 struct irdma_mr *iwmr) 3482 { 3483 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3484 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3485 struct irdma_ucontext *ucontext = NULL; 3486 unsigned long flags; 3487 u32 total; 3488 int err; 3489 u8 lvl; 3490 3491 /* iWarp: Catch page not starting on OS page boundary */ 3492 if (!rdma_protocol_roce(&iwdev->ibdev, 1) && 3493 ib_umem_offset(iwmr->region)) 3494 return -EINVAL; 3495 3496 total = req.sq_pages + req.rq_pages + 1; 3497 if (total > iwmr->page_cnt) 3498 return -EINVAL; 3499 3500 total = req.sq_pages + req.rq_pages; 3501 lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3502 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3503 if (err) 3504 return err; 3505 3506 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3507 ibucontext); 3508 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 3509 list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 3510 iwpbl->on_list = true; 3511 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 3512 3513 return 0; 3514 } 3515 3516 static int irdma_reg_user_mr_type_srq(struct irdma_mem_reg_req req, 3517 struct ib_udata *udata, 3518 struct irdma_mr *iwmr) 3519 { 3520 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3521 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3522 struct irdma_ucontext *ucontext; 3523 unsigned long flags; 3524 u32 total; 3525 int err; 3526 u8 lvl; 3527 3528 total = req.rq_pages + IRDMA_SHADOW_PGCNT; 3529 if (total > iwmr->page_cnt) 3530 return -EINVAL; 3531 3532 lvl = req.rq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3533 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3534 if (err) 3535 return err; 3536 3537 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3538 ibucontext); 3539 spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); 3540 list_add_tail(&iwpbl->list, &ucontext->srq_reg_mem_list); 3541 iwpbl->on_list = true; 3542 spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); 3543 3544 return 0; 3545 } 3546 3547 static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req, 3548 struct ib_udata *udata, 3549 struct irdma_mr *iwmr) 3550 { 3551 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3552 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3553 struct irdma_ucontext *ucontext = NULL; 3554 u8 shadow_pgcnt = 1; 3555 unsigned long flags; 3556 u32 total; 3557 int err; 3558 u8 lvl; 3559 3560 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 3561 shadow_pgcnt = 0; 3562 total = req.cq_pages + shadow_pgcnt; 3563 if (total > iwmr->page_cnt) 3564 return -EINVAL; 3565 3566 lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3567 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3568 if (err) 3569 return err; 3570 3571 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3572 ibucontext); 3573 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 3574 list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 3575 iwpbl->on_list = true; 3576 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 3577 3578 return 0; 3579 } 3580 3581 /** 3582 * irdma_reg_user_mr - Register a user memory region 3583 * @pd: ptr of pd 3584 * @start: virtual start address 3585 * @len: length of mr 3586 * @virt: virtual address 3587 * @access: access of mr 3588 * @dmah: dma handle 3589 * @udata: user data 3590 */ 3591 static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, 3592 u64 virt, int access, 3593 struct ib_dmah *dmah, 3594 struct ib_udata *udata) 3595 { 3596 struct irdma_device *iwdev = to_iwdev(pd->device); 3597 struct irdma_mem_reg_req req = {}; 3598 struct ib_umem *region = NULL; 3599 struct irdma_mr *iwmr = NULL; 3600 int err; 3601 3602 if (dmah) 3603 return ERR_PTR(-EOPNOTSUPP); 3604 3605 err = ib_copy_validate_udata_in(udata, req, sq_pages); 3606 if (err) 3607 return ERR_PTR(err); 3608 3609 err = ib_respond_empty_udata(udata); 3610 if (err) 3611 return ERR_PTR(err); 3612 3613 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3614 return ERR_PTR(-EINVAL); 3615 3616 region = ib_umem_get_va(pd->device, start, len, access); 3617 3618 if (IS_ERR(region)) { 3619 ibdev_dbg(&iwdev->ibdev, 3620 "VERBS: Failed to create ib_umem region\n"); 3621 return (struct ib_mr *)region; 3622 } 3623 3624 iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type); 3625 if (IS_ERR(iwmr)) { 3626 ib_umem_release(region); 3627 return (struct ib_mr *)iwmr; 3628 } 3629 3630 switch (req.reg_type) { 3631 case IRDMA_MEMREG_TYPE_QP: 3632 err = irdma_reg_user_mr_type_qp(req, udata, iwmr); 3633 if (err) 3634 goto error; 3635 3636 break; 3637 case IRDMA_MEMREG_TYPE_SRQ: 3638 err = irdma_reg_user_mr_type_srq(req, udata, iwmr); 3639 if (err) 3640 goto error; 3641 3642 break; 3643 case IRDMA_MEMREG_TYPE_CQ: 3644 err = irdma_reg_user_mr_type_cq(req, udata, iwmr); 3645 if (err) 3646 goto error; 3647 break; 3648 case IRDMA_MEMREG_TYPE_MEM: 3649 err = irdma_reg_user_mr_type_mem(iwmr, access, true); 3650 if (err) 3651 goto error; 3652 3653 break; 3654 default: 3655 err = -EINVAL; 3656 goto error; 3657 } 3658 3659 return &iwmr->ibmr; 3660 error: 3661 ib_umem_release(region); 3662 irdma_free_iwmr(iwmr); 3663 3664 return ERR_PTR(err); 3665 } 3666 3667 static int irdma_hwdereg_mr(struct ib_mr *ib_mr); 3668 3669 static void irdma_umem_dmabuf_revoke(void *priv) 3670 { 3671 /* priv is guaranteed to be valid any time this callback is invoked 3672 * because we do not set the callback until after successful iwmr 3673 * allocation and initialization. 3674 */ 3675 struct irdma_mr *iwmr = priv; 3676 int err; 3677 3678 /* Invalidate the key in hardware. This does not actually release the 3679 * key for potential reuse - that only occurs when the region is fully 3680 * deregistered. 3681 * 3682 * The irdma_hwdereg_mr call is a no-op if the region is not currently 3683 * registered with hardware. 3684 */ 3685 err = irdma_hwdereg_mr(&iwmr->ibmr); 3686 if (err) { 3687 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3688 3689 ibdev_err(&iwdev->ibdev, "dmabuf mr revoke failed %d", err); 3690 if (!iwdev->rf->reset) { 3691 iwdev->rf->reset = true; 3692 iwdev->rf->gen_ops.request_reset(iwdev->rf); 3693 } 3694 } 3695 } 3696 3697 static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start, 3698 u64 len, u64 virt, 3699 int fd, int access, 3700 struct ib_dmah *dmah, 3701 struct uverbs_attr_bundle *attrs) 3702 { 3703 struct irdma_device *iwdev = to_iwdev(pd->device); 3704 struct ib_umem_dmabuf *umem_dmabuf; 3705 struct irdma_mr *iwmr; 3706 int err; 3707 3708 if (dmah) 3709 return ERR_PTR(-EOPNOTSUPP); 3710 3711 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3712 return ERR_PTR(-EINVAL); 3713 3714 umem_dmabuf = 3715 ib_umem_dmabuf_get_pinned_revocable_and_lock(pd->device, start, 3716 len, fd, access); 3717 if (IS_ERR(umem_dmabuf)) { 3718 ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%pe]\n", 3719 umem_dmabuf); 3720 return ERR_CAST(umem_dmabuf); 3721 } 3722 3723 iwmr = irdma_alloc_iwmr(&umem_dmabuf->umem, pd, virt, IRDMA_MEMREG_TYPE_MEM); 3724 if (IS_ERR(iwmr)) { 3725 err = PTR_ERR(iwmr); 3726 goto err_release; 3727 } 3728 3729 err = irdma_reg_user_mr_type_mem(iwmr, access, true); 3730 if (err) 3731 goto err_iwmr; 3732 3733 ib_umem_dmabuf_set_revoke_locked(umem_dmabuf, irdma_umem_dmabuf_revoke, 3734 iwmr); 3735 ib_umem_dmabuf_revoke_unlock(umem_dmabuf); 3736 return &iwmr->ibmr; 3737 3738 err_iwmr: 3739 irdma_free_iwmr(iwmr); 3740 3741 err_release: 3742 ib_umem_dmabuf_revoke_unlock(umem_dmabuf); 3743 3744 /* Will result in a call to revoke, but driver callback is not set and 3745 * is therefore skipped. 3746 */ 3747 ib_umem_release(&umem_dmabuf->umem); 3748 3749 return ERR_PTR(err); 3750 } 3751 3752 static int irdma_hwdereg_mr(struct ib_mr *ib_mr) 3753 { 3754 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3755 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3756 struct irdma_pd *iwpd = to_iwpd(ib_mr->pd); 3757 struct irdma_dealloc_stag_info *info; 3758 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3759 struct irdma_cqp_request *cqp_request; 3760 struct cqp_cmds_info *cqp_info; 3761 int status; 3762 3763 /* Skip HW MR de-register when it is already de-registered 3764 * during an MR re-reregister and the re-registration fails 3765 */ 3766 if (!iwmr->is_hwreg) 3767 return 0; 3768 3769 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3770 if (!cqp_request) 3771 return -ENOMEM; 3772 3773 cqp_info = &cqp_request->info; 3774 info = &cqp_info->in.u.dealloc_stag.info; 3775 info->pd_id = iwpd->sc_pd.pd_id; 3776 info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3777 info->mr = true; 3778 if (iwpbl->pbl_allocated) 3779 info->dealloc_pbl = true; 3780 3781 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3782 cqp_info->post_sq = 1; 3783 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3784 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3785 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3786 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3787 if (status) 3788 return status; 3789 3790 iwmr->is_hwreg = false; 3791 return 0; 3792 } 3793 3794 /* 3795 * irdma_rereg_mr_trans - Re-register a user MR for a change translation. 3796 * @iwmr: ptr of iwmr 3797 * @start: virtual start address 3798 * @len: length of mr 3799 * @virt: virtual address 3800 * 3801 * Re-register a user memory region when a change translation is requested. 3802 * Re-register a new region while reusing the stag from the original registration. 3803 */ 3804 static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, 3805 u64 virt) 3806 { 3807 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3808 struct ib_pd *pd = iwmr->ibmr.pd; 3809 struct ib_umem *region; 3810 int err; 3811 3812 region = ib_umem_get_va(pd->device, start, len, iwmr->access); 3813 if (IS_ERR(region)) 3814 return PTR_ERR(region); 3815 3816 iwmr->region = region; 3817 iwmr->ibmr.iova = virt; 3818 iwmr->ibmr.pd = pd; 3819 iwmr->len = region->length; 3820 iwpbl->user_base = virt; 3821 3822 err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false); 3823 if (err) 3824 goto err; 3825 3826 return 0; 3827 3828 err: 3829 ib_umem_release(region); 3830 iwmr->region = NULL; 3831 return err; 3832 } 3833 3834 /* 3835 * irdma_rereg_user_mr - Re-Register a user memory region(MR) 3836 * @ibmr: ib mem to access iwarp mr pointer 3837 * @flags: bit mask to indicate which of the attr's of MR modified 3838 * @start: virtual start address 3839 * @len: length of mr 3840 * @virt: virtual address 3841 * @new_access: bit mask of access flags 3842 * @new_pd: ptr of pd 3843 * @udata: user data 3844 * 3845 * Return: 3846 * NULL - Success, existing MR updated 3847 * ERR_PTR - error occurred 3848 */ 3849 static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, 3850 u64 start, u64 len, u64 virt, 3851 int new_access, struct ib_pd *new_pd, 3852 struct ib_udata *udata) 3853 { 3854 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3855 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3856 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3857 int ret; 3858 3859 ret = ib_no_udata_io(udata); 3860 if (ret) 3861 return ERR_PTR(ret); 3862 3863 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3864 return ERR_PTR(-EINVAL); 3865 3866 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) 3867 return ERR_PTR(-EOPNOTSUPP); 3868 3869 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) 3870 return ERR_PTR(-EINVAL); 3871 3872 ret = ib_umem_check_rereg(iwmr->region, flags, new_access); 3873 if (ret) 3874 return ERR_PTR(ret); 3875 3876 ret = irdma_hwdereg_mr(ib_mr); 3877 if (ret) 3878 return ERR_PTR(ret); 3879 3880 if (flags & IB_MR_REREG_ACCESS) 3881 iwmr->access = new_access; 3882 3883 if (flags & IB_MR_REREG_PD) { 3884 iwmr->ibmr.pd = new_pd; 3885 iwmr->ibmr.device = new_pd->device; 3886 } 3887 3888 if (flags & IB_MR_REREG_TRANS) { 3889 if (iwpbl->pbl_allocated) { 3890 irdma_free_pble(iwdev->rf->pble_rsrc, 3891 &iwpbl->pble_alloc); 3892 iwpbl->pbl_allocated = false; 3893 } 3894 3895 if (iwmr->region) { 3896 ib_umem_release(iwmr->region); 3897 iwmr->region = NULL; 3898 } 3899 3900 ret = irdma_rereg_mr_trans(iwmr, start, len, virt); 3901 } else { 3902 ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access); 3903 } 3904 3905 return ret ? ERR_PTR(ret) : NULL; 3906 } 3907 3908 /** 3909 * irdma_reg_phys_mr - register kernel physical memory 3910 * @pd: ibpd pointer 3911 * @addr: physical address of memory to register 3912 * @size: size of memory to register 3913 * @access: Access rights 3914 * @iova_start: start of virtual address for physical buffers 3915 * @dma_mr: Flag indicating whether this region is a PD DMA MR 3916 */ 3917 struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, 3918 u64 *iova_start, bool dma_mr) 3919 { 3920 struct irdma_device *iwdev = to_iwdev(pd->device); 3921 struct irdma_pbl *iwpbl; 3922 struct irdma_mr *iwmr; 3923 u32 stag; 3924 int ret; 3925 3926 iwmr = kzalloc_obj(*iwmr); 3927 if (!iwmr) 3928 return ERR_PTR(-ENOMEM); 3929 3930 iwmr->ibmr.pd = pd; 3931 iwmr->ibmr.device = pd->device; 3932 iwpbl = &iwmr->iwpbl; 3933 iwpbl->iwmr = iwmr; 3934 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 3935 iwmr->dma_mr = dma_mr; 3936 iwpbl->user_base = *iova_start; 3937 stag = irdma_create_stag(iwdev); 3938 if (!stag) { 3939 ret = -ENOMEM; 3940 goto err; 3941 } 3942 3943 iwmr->stag = stag; 3944 iwmr->ibmr.iova = *iova_start; 3945 iwmr->ibmr.rkey = stag; 3946 iwmr->ibmr.lkey = stag; 3947 iwmr->page_cnt = 1; 3948 iwmr->pgaddrmem[0] = addr; 3949 iwmr->len = size; 3950 iwmr->page_size = SZ_4K; 3951 ret = irdma_hwreg_mr(iwdev, iwmr, access); 3952 if (ret) { 3953 irdma_free_stag(iwdev, stag); 3954 goto err; 3955 } 3956 3957 return &iwmr->ibmr; 3958 3959 err: 3960 kfree(iwmr); 3961 3962 return ERR_PTR(ret); 3963 } 3964 3965 /** 3966 * irdma_get_dma_mr - register physical mem 3967 * @pd: ptr of pd 3968 * @acc: access for memory 3969 */ 3970 static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) 3971 { 3972 u64 kva = 0; 3973 3974 return irdma_reg_phys_mr(pd, 0, 0, acc, &kva, true); 3975 } 3976 3977 /** 3978 * irdma_del_memlist - Deleting pbl list entries for CQ/QP 3979 * @iwmr: iwmr for IB's user page addresses 3980 * @ucontext: ptr to user context 3981 * 3982 * Return: True if the MR is currently in-use by a QP/CQ/SRQ ring. 3983 */ 3984 static bool irdma_del_memlist(struct irdma_mr *iwmr, 3985 struct irdma_ucontext *ucontext) 3986 { 3987 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3988 unsigned long flags; 3989 spinlock_t *lock; 3990 bool in_use = false; 3991 3992 switch (iwmr->type) { 3993 case IRDMA_MEMREG_TYPE_CQ: 3994 lock = &ucontext->cq_reg_mem_list_lock; 3995 break; 3996 case IRDMA_MEMREG_TYPE_QP: 3997 lock = &ucontext->qp_reg_mem_list_lock; 3998 break; 3999 case IRDMA_MEMREG_TYPE_SRQ: 4000 lock = &ucontext->srq_reg_mem_list_lock; 4001 break; 4002 default: 4003 return false; 4004 } 4005 4006 spin_lock_irqsave(lock, flags); 4007 if (!refcount_dec_if_one(&iwmr->user_ring_refs)) { 4008 in_use = true; 4009 } else if (iwpbl->on_list) { 4010 iwpbl->on_list = false; 4011 list_del(&iwpbl->list); 4012 } 4013 spin_unlock_irqrestore(lock, flags); 4014 4015 return in_use; 4016 } 4017 4018 /** 4019 * irdma_dereg_mr - deregister mr 4020 * @ib_mr: mr ptr for dereg 4021 * @udata: user data 4022 */ 4023 static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 4024 { 4025 struct irdma_mr *iwmr = to_iwmr(ib_mr); 4026 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 4027 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 4028 bool dmabuf_revocable = iwmr->region && iwmr->region->is_dmabuf; 4029 int ret; 4030 4031 ret = ib_no_udata_io(udata); 4032 if (ret) 4033 return ret; 4034 4035 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 4036 if (iwmr->region) { 4037 struct irdma_ucontext *ucontext; 4038 4039 ucontext = rdma_udata_to_drv_context(udata, 4040 struct irdma_ucontext, 4041 ibucontext); 4042 4043 /* Do not allow the MR to be unpinned if it is still 4044 * backing a user ring. 4045 */ 4046 if (irdma_del_memlist(iwmr, ucontext)) 4047 return -EBUSY; 4048 } 4049 goto done; 4050 } 4051 4052 if (!dmabuf_revocable) { 4053 ret = irdma_hwdereg_mr(ib_mr); 4054 if (ret) 4055 return ret; 4056 4057 irdma_free_stag(iwdev, iwmr->stag); 4058 } 4059 done: 4060 if (iwmr->region) 4061 /* For dmabuf MRs, ib_umem_release will trigger a synchronous 4062 * call to the revoke callback which will perform the actual HW 4063 * invalidation via irdma_hwdereg_mr. We rely on this for its 4064 * implicit serialization w.r.t. concurrent revocations. This 4065 * must be done before freeing the PBLEs. 4066 */ 4067 ib_umem_release(iwmr->region); 4068 4069 if (iwpbl->pbl_allocated) 4070 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 4071 4072 if (dmabuf_revocable) 4073 irdma_free_stag(iwdev, iwmr->stag); 4074 4075 kfree(iwmr); 4076 4077 return 0; 4078 } 4079 4080 /** 4081 * irdma_post_send - kernel application wr 4082 * @ibqp: qp ptr for wr 4083 * @ib_wr: work request ptr 4084 * @bad_wr: return of bad wr if err 4085 */ 4086 static int irdma_post_send(struct ib_qp *ibqp, 4087 const struct ib_send_wr *ib_wr, 4088 const struct ib_send_wr **bad_wr) 4089 { 4090 struct irdma_qp *iwqp; 4091 struct irdma_qp_uk *ukqp; 4092 struct irdma_sc_dev *dev; 4093 struct irdma_post_sq_info info; 4094 int err = 0; 4095 unsigned long flags; 4096 bool inv_stag; 4097 struct irdma_ah *ah; 4098 4099 iwqp = to_iwqp(ibqp); 4100 ukqp = &iwqp->sc_qp.qp_uk; 4101 dev = &iwqp->iwdev->rf->sc_dev; 4102 4103 spin_lock_irqsave(&iwqp->lock, flags); 4104 while (ib_wr) { 4105 memset(&info, 0, sizeof(info)); 4106 inv_stag = false; 4107 info.wr_id = (ib_wr->wr_id); 4108 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) 4109 info.signaled = true; 4110 if (ib_wr->send_flags & IB_SEND_FENCE) 4111 info.read_fence = true; 4112 switch (ib_wr->opcode) { 4113 case IB_WR_ATOMIC_CMP_AND_SWP: 4114 if (unlikely(!(dev->hw_attrs.uk_attrs.feature_flags & 4115 IRDMA_FEATURE_ATOMIC_OPS))) { 4116 err = -EINVAL; 4117 break; 4118 } 4119 info.op_type = IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP; 4120 info.op.atomic_compare_swap.tagged_offset = ib_wr->sg_list[0].addr; 4121 info.op.atomic_compare_swap.remote_tagged_offset = 4122 atomic_wr(ib_wr)->remote_addr; 4123 info.op.atomic_compare_swap.swap_data_bytes = atomic_wr(ib_wr)->swap; 4124 info.op.atomic_compare_swap.compare_data_bytes = 4125 atomic_wr(ib_wr)->compare_add; 4126 info.op.atomic_compare_swap.stag = ib_wr->sg_list[0].lkey; 4127 info.op.atomic_compare_swap.remote_stag = atomic_wr(ib_wr)->rkey; 4128 err = irdma_uk_atomic_compare_swap(ukqp, &info, false); 4129 break; 4130 case IB_WR_ATOMIC_FETCH_AND_ADD: 4131 if (unlikely(!(dev->hw_attrs.uk_attrs.feature_flags & 4132 IRDMA_FEATURE_ATOMIC_OPS))) { 4133 err = -EINVAL; 4134 break; 4135 } 4136 info.op_type = IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD; 4137 info.op.atomic_fetch_add.tagged_offset = ib_wr->sg_list[0].addr; 4138 info.op.atomic_fetch_add.remote_tagged_offset = 4139 atomic_wr(ib_wr)->remote_addr; 4140 info.op.atomic_fetch_add.fetch_add_data_bytes = 4141 atomic_wr(ib_wr)->compare_add; 4142 info.op.atomic_fetch_add.stag = ib_wr->sg_list[0].lkey; 4143 info.op.atomic_fetch_add.remote_stag = 4144 atomic_wr(ib_wr)->rkey; 4145 err = irdma_uk_atomic_fetch_add(ukqp, &info, false); 4146 break; 4147 case IB_WR_SEND_WITH_IMM: 4148 if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { 4149 info.imm_data_valid = true; 4150 info.imm_data = ntohl(ib_wr->ex.imm_data); 4151 } else { 4152 err = -EINVAL; 4153 break; 4154 } 4155 fallthrough; 4156 case IB_WR_SEND: 4157 case IB_WR_SEND_WITH_INV: 4158 if (ib_wr->opcode == IB_WR_SEND || 4159 ib_wr->opcode == IB_WR_SEND_WITH_IMM) { 4160 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4161 info.op_type = IRDMA_OP_TYPE_SEND_SOL; 4162 else 4163 info.op_type = IRDMA_OP_TYPE_SEND; 4164 } else { 4165 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4166 info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; 4167 else 4168 info.op_type = IRDMA_OP_TYPE_SEND_INV; 4169 info.stag_to_inv = ib_wr->ex.invalidate_rkey; 4170 } 4171 4172 info.op.send.num_sges = ib_wr->num_sge; 4173 info.op.send.sg_list = ib_wr->sg_list; 4174 if (iwqp->ibqp.qp_type == IB_QPT_UD || 4175 iwqp->ibqp.qp_type == IB_QPT_GSI) { 4176 ah = to_iwah(ud_wr(ib_wr)->ah); 4177 info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; 4178 info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; 4179 info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; 4180 } 4181 4182 if (ib_wr->send_flags & IB_SEND_INLINE) 4183 err = irdma_uk_inline_send(ukqp, &info, false); 4184 else 4185 err = irdma_uk_send(ukqp, &info, false); 4186 break; 4187 case IB_WR_RDMA_WRITE_WITH_IMM: 4188 if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { 4189 info.imm_data_valid = true; 4190 info.imm_data = ntohl(ib_wr->ex.imm_data); 4191 } else { 4192 err = -EINVAL; 4193 break; 4194 } 4195 fallthrough; 4196 case IB_WR_RDMA_WRITE: 4197 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4198 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; 4199 else 4200 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; 4201 4202 info.op.rdma_write.num_lo_sges = ib_wr->num_sge; 4203 info.op.rdma_write.lo_sg_list = ib_wr->sg_list; 4204 info.op.rdma_write.rem_addr.addr = 4205 rdma_wr(ib_wr)->remote_addr; 4206 info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 4207 if (ib_wr->send_flags & IB_SEND_INLINE) 4208 err = irdma_uk_inline_rdma_write(ukqp, &info, false); 4209 else 4210 err = irdma_uk_rdma_write(ukqp, &info, false); 4211 break; 4212 case IB_WR_RDMA_READ_WITH_INV: 4213 inv_stag = true; 4214 fallthrough; 4215 case IB_WR_RDMA_READ: 4216 if (ib_wr->num_sge > 4217 dev->hw_attrs.uk_attrs.max_hw_read_sges) { 4218 err = -EINVAL; 4219 break; 4220 } 4221 info.op_type = IRDMA_OP_TYPE_RDMA_READ; 4222 info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 4223 info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 4224 info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; 4225 info.op.rdma_read.num_lo_sges = ib_wr->num_sge; 4226 err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); 4227 break; 4228 case IB_WR_LOCAL_INV: 4229 info.op_type = IRDMA_OP_TYPE_INV_STAG; 4230 info.local_fence = true; 4231 info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; 4232 err = irdma_uk_stag_local_invalidate(ukqp, &info, true); 4233 break; 4234 case IB_WR_REG_MR: { 4235 struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); 4236 struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; 4237 struct irdma_fast_reg_stag_info stag_info = {}; 4238 4239 stag_info.signaled = info.signaled; 4240 stag_info.read_fence = info.read_fence; 4241 stag_info.access_rights = 4242 irdma_get_mr_access(reg_wr(ib_wr)->access, 4243 dev->hw_attrs.uk_attrs.hw_rev); 4244 stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; 4245 stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; 4246 stag_info.page_size = reg_wr(ib_wr)->mr->page_size; 4247 stag_info.wr_id = ib_wr->wr_id; 4248 stag_info.va = iwmr->ibmr.iova; 4249 stag_info.total_len = iwmr->ibmr.length; 4250 stag_info.local_fence = true; 4251 if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) { 4252 if (palloc->level == PBLE_LEVEL_1) { 4253 stag_info.first_pm_pbl_index = 4254 palloc->level1.idx; 4255 stag_info.chunk_size = 1; 4256 } else { 4257 stag_info.first_pm_pbl_index = 4258 palloc->level2.root.idx; 4259 stag_info.chunk_size = 3; 4260 } 4261 } else { 4262 stag_info.reg_addr_pa = 4263 (palloc->level == PBLE_LEVEL_1) ? 4264 *palloc->level1.addr : 4265 *palloc->level2.leaf->addr; 4266 } 4267 err = irdma_sc_mr_fast_register(&iwqp->sc_qp, 4268 &stag_info); 4269 break; 4270 } 4271 default: 4272 err = -EINVAL; 4273 ibdev_dbg(&iwqp->iwdev->ibdev, 4274 "VERBS: upost_send bad opcode = 0x%x\n", 4275 ib_wr->opcode); 4276 break; 4277 } 4278 4279 if (err) 4280 break; 4281 ib_wr = ib_wr->next; 4282 } 4283 4284 if (!iwqp->flush_issued) { 4285 if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) 4286 irdma_uk_qp_post_wr(ukqp); 4287 spin_unlock_irqrestore(&iwqp->lock, flags); 4288 } else { 4289 spin_unlock_irqrestore(&iwqp->lock, flags); 4290 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 4291 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 4292 } 4293 4294 if (err) 4295 *bad_wr = ib_wr; 4296 4297 return err; 4298 } 4299 4300 /** 4301 * irdma_post_srq_recv - post receive wr for kernel application 4302 * @ibsrq: ib srq pointer 4303 * @ib_wr: work request for receive 4304 * @bad_wr: bad wr caused an error 4305 */ 4306 static int irdma_post_srq_recv(struct ib_srq *ibsrq, 4307 const struct ib_recv_wr *ib_wr, 4308 const struct ib_recv_wr **bad_wr) 4309 { 4310 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 4311 struct irdma_srq_uk *uksrq = &iwsrq->sc_srq.srq_uk; 4312 struct irdma_post_rq_info post_recv = {}; 4313 unsigned long flags; 4314 int err = 0; 4315 4316 spin_lock_irqsave(&iwsrq->lock, flags); 4317 while (ib_wr) { 4318 if (ib_wr->num_sge > uksrq->max_srq_frag_cnt) { 4319 err = -EINVAL; 4320 goto out; 4321 } 4322 post_recv.num_sges = ib_wr->num_sge; 4323 post_recv.wr_id = ib_wr->wr_id; 4324 post_recv.sg_list = ib_wr->sg_list; 4325 err = irdma_uk_srq_post_receive(uksrq, &post_recv); 4326 if (err) 4327 goto out; 4328 4329 ib_wr = ib_wr->next; 4330 } 4331 4332 out: 4333 spin_unlock_irqrestore(&iwsrq->lock, flags); 4334 4335 if (err) 4336 *bad_wr = ib_wr; 4337 4338 return err; 4339 } 4340 4341 /** 4342 * irdma_post_recv - post receive wr for kernel application 4343 * @ibqp: ib qp pointer 4344 * @ib_wr: work request for receive 4345 * @bad_wr: bad wr caused an error 4346 */ 4347 static int irdma_post_recv(struct ib_qp *ibqp, 4348 const struct ib_recv_wr *ib_wr, 4349 const struct ib_recv_wr **bad_wr) 4350 { 4351 struct irdma_qp *iwqp; 4352 struct irdma_qp_uk *ukqp; 4353 struct irdma_post_rq_info post_recv = {}; 4354 unsigned long flags; 4355 int err = 0; 4356 4357 iwqp = to_iwqp(ibqp); 4358 ukqp = &iwqp->sc_qp.qp_uk; 4359 4360 if (ukqp->srq_uk) { 4361 *bad_wr = ib_wr; 4362 return -EINVAL; 4363 } 4364 4365 spin_lock_irqsave(&iwqp->lock, flags); 4366 while (ib_wr) { 4367 post_recv.num_sges = ib_wr->num_sge; 4368 post_recv.wr_id = ib_wr->wr_id; 4369 post_recv.sg_list = ib_wr->sg_list; 4370 err = irdma_uk_post_receive(ukqp, &post_recv); 4371 if (err) { 4372 ibdev_dbg(&iwqp->iwdev->ibdev, 4373 "VERBS: post_recv err %d\n", err); 4374 goto out; 4375 } 4376 4377 ib_wr = ib_wr->next; 4378 } 4379 4380 out: 4381 spin_unlock_irqrestore(&iwqp->lock, flags); 4382 if (iwqp->flush_issued) 4383 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 4384 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 4385 4386 if (err) 4387 *bad_wr = ib_wr; 4388 4389 return err; 4390 } 4391 4392 /** 4393 * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status 4394 * @opcode: iwarp flush code 4395 */ 4396 static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) 4397 { 4398 switch (opcode) { 4399 case FLUSH_PROT_ERR: 4400 return IB_WC_LOC_PROT_ERR; 4401 case FLUSH_REM_ACCESS_ERR: 4402 return IB_WC_REM_ACCESS_ERR; 4403 case FLUSH_LOC_QP_OP_ERR: 4404 return IB_WC_LOC_QP_OP_ERR; 4405 case FLUSH_REM_OP_ERR: 4406 return IB_WC_REM_OP_ERR; 4407 case FLUSH_LOC_LEN_ERR: 4408 return IB_WC_LOC_LEN_ERR; 4409 case FLUSH_GENERAL_ERR: 4410 return IB_WC_WR_FLUSH_ERR; 4411 case FLUSH_RETRY_EXC_ERR: 4412 return IB_WC_RETRY_EXC_ERR; 4413 case FLUSH_MW_BIND_ERR: 4414 return IB_WC_MW_BIND_ERR; 4415 case FLUSH_REM_INV_REQ_ERR: 4416 return IB_WC_REM_INV_REQ_ERR; 4417 case FLUSH_RNR_RETRY_EXC_ERR: 4418 return IB_WC_RNR_RETRY_EXC_ERR; 4419 case FLUSH_FATAL_ERR: 4420 default: 4421 return IB_WC_FATAL_ERR; 4422 } 4423 } 4424 4425 /** 4426 * irdma_process_cqe - process cqe info 4427 * @entry: processed cqe 4428 * @cq_poll_info: cqe info 4429 */ 4430 static void irdma_process_cqe(struct ib_wc *entry, 4431 struct irdma_cq_poll_info *cq_poll_info) 4432 { 4433 struct irdma_sc_qp *qp; 4434 4435 entry->wc_flags = 0; 4436 entry->pkey_index = 0; 4437 entry->wr_id = cq_poll_info->wr_id; 4438 4439 qp = cq_poll_info->qp_handle; 4440 entry->qp = qp->qp_uk.back_qp; 4441 4442 if (cq_poll_info->error) { 4443 entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? 4444 irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; 4445 4446 entry->vendor_err = cq_poll_info->major_err << 16 | 4447 cq_poll_info->minor_err; 4448 } else { 4449 entry->status = IB_WC_SUCCESS; 4450 if (cq_poll_info->imm_valid) { 4451 entry->ex.imm_data = htonl(cq_poll_info->imm_data); 4452 entry->wc_flags |= IB_WC_WITH_IMM; 4453 } 4454 if (cq_poll_info->ud_smac_valid) { 4455 ether_addr_copy(entry->smac, cq_poll_info->ud_smac); 4456 entry->wc_flags |= IB_WC_WITH_SMAC; 4457 } 4458 4459 if (cq_poll_info->ud_vlan_valid) { 4460 u16 vlan = cq_poll_info->ud_vlan & VLAN_VID_MASK; 4461 4462 entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; 4463 if (vlan) { 4464 entry->vlan_id = vlan; 4465 entry->wc_flags |= IB_WC_WITH_VLAN; 4466 } 4467 } else { 4468 entry->sl = 0; 4469 } 4470 } 4471 4472 if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) { 4473 set_ib_wc_op_sq(cq_poll_info, entry); 4474 } else { 4475 if (qp->dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) 4476 set_ib_wc_op_rq(cq_poll_info, entry, 4477 qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM ? 4478 true : false); 4479 else 4480 set_ib_wc_op_rq_gen_3(cq_poll_info, entry); 4481 if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && 4482 cq_poll_info->stag_invalid_set) { 4483 entry->ex.invalidate_rkey = cq_poll_info->inv_stag; 4484 entry->wc_flags |= IB_WC_WITH_INVALIDATE; 4485 } 4486 } 4487 4488 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { 4489 entry->src_qp = cq_poll_info->ud_src_qpn; 4490 entry->slid = 0; 4491 entry->wc_flags |= 4492 (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); 4493 entry->network_hdr_type = cq_poll_info->ipv4 ? 4494 RDMA_NETWORK_IPV4 : 4495 RDMA_NETWORK_IPV6; 4496 } else { 4497 entry->src_qp = cq_poll_info->qp_id; 4498 } 4499 4500 entry->byte_len = cq_poll_info->bytes_xfered; 4501 } 4502 4503 /** 4504 * irdma_poll_one - poll one entry of the CQ 4505 * @ukcq: ukcq to poll 4506 * @cur_cqe: current CQE info to be filled in 4507 * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ 4508 * 4509 * Returns the internal irdma device error code or 0 on success 4510 */ 4511 static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, 4512 struct irdma_cq_poll_info *cur_cqe, 4513 struct ib_wc *entry) 4514 { 4515 int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); 4516 4517 if (ret) 4518 return ret; 4519 4520 irdma_process_cqe(entry, cur_cqe); 4521 4522 return 0; 4523 } 4524 4525 /** 4526 * __irdma_poll_cq - poll cq for completion (kernel apps) 4527 * @iwcq: cq to poll 4528 * @num_entries: number of entries to poll 4529 * @entry: wr of a completed entry 4530 */ 4531 static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) 4532 { 4533 struct list_head *tmp_node, *list_node; 4534 struct irdma_cq_buf *last_buf = NULL; 4535 struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; 4536 struct irdma_cq_buf *cq_buf; 4537 int ret; 4538 struct irdma_device *iwdev; 4539 struct irdma_cq_uk *ukcq; 4540 bool cq_new_cqe = false; 4541 int resized_bufs = 0; 4542 int npolled = 0; 4543 4544 iwdev = to_iwdev(iwcq->ibcq.device); 4545 ukcq = &iwcq->sc_cq.cq_uk; 4546 4547 /* go through the list of previously resized CQ buffers */ 4548 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 4549 cq_buf = container_of(list_node, struct irdma_cq_buf, list); 4550 while (npolled < num_entries) { 4551 ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); 4552 if (!ret) { 4553 ++npolled; 4554 cq_new_cqe = true; 4555 continue; 4556 } 4557 if (ret == -ENOENT) 4558 break; 4559 /* QP using the CQ is destroyed. Skip reporting this CQE */ 4560 if (ret == -EFAULT) { 4561 cq_new_cqe = true; 4562 continue; 4563 } 4564 goto error; 4565 } 4566 4567 /* save the resized CQ buffer which received the last cqe */ 4568 if (cq_new_cqe) 4569 last_buf = cq_buf; 4570 cq_new_cqe = false; 4571 } 4572 4573 /* check the current CQ for new cqes */ 4574 while (npolled < num_entries) { 4575 ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); 4576 if (ret == -ENOENT) { 4577 ret = irdma_generated_cmpls(iwcq, cur_cqe); 4578 if (!ret) 4579 irdma_process_cqe(entry + npolled, cur_cqe); 4580 } 4581 if (!ret) { 4582 ++npolled; 4583 cq_new_cqe = true; 4584 continue; 4585 } 4586 4587 if (ret == -ENOENT) 4588 break; 4589 /* QP using the CQ is destroyed. Skip reporting this CQE */ 4590 if (ret == -EFAULT) { 4591 cq_new_cqe = true; 4592 continue; 4593 } 4594 goto error; 4595 } 4596 4597 if (cq_new_cqe) 4598 /* all previous CQ resizes are complete */ 4599 resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); 4600 else if (last_buf) 4601 /* only CQ resizes up to the last_buf are complete */ 4602 resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); 4603 if (resized_bufs) 4604 /* report to the HW the number of complete CQ resizes */ 4605 irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); 4606 4607 return npolled; 4608 error: 4609 ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n", 4610 __func__, ret); 4611 4612 return ret; 4613 } 4614 4615 /** 4616 * irdma_poll_cq - poll cq for completion (kernel apps) 4617 * @ibcq: cq to poll 4618 * @num_entries: number of entries to poll 4619 * @entry: wr of a completed entry 4620 */ 4621 static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, 4622 struct ib_wc *entry) 4623 { 4624 struct irdma_cq *iwcq; 4625 unsigned long flags; 4626 int ret; 4627 4628 iwcq = to_iwcq(ibcq); 4629 4630 spin_lock_irqsave(&iwcq->lock, flags); 4631 ret = __irdma_poll_cq(iwcq, num_entries, entry); 4632 spin_unlock_irqrestore(&iwcq->lock, flags); 4633 4634 return ret; 4635 } 4636 4637 /** 4638 * irdma_req_notify_cq - arm cq kernel application 4639 * @ibcq: cq to arm 4640 * @notify_flags: notofication flags 4641 */ 4642 static int irdma_req_notify_cq(struct ib_cq *ibcq, 4643 enum ib_cq_notify_flags notify_flags) 4644 { 4645 struct irdma_cq *iwcq; 4646 struct irdma_cq_uk *ukcq; 4647 unsigned long flags; 4648 enum irdma_cmpl_notify cq_notify; 4649 bool promo_event = false; 4650 int ret = 0; 4651 4652 cq_notify = notify_flags == IB_CQ_SOLICITED ? 4653 IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT; 4654 iwcq = to_iwcq(ibcq); 4655 ukcq = &iwcq->sc_cq.cq_uk; 4656 4657 spin_lock_irqsave(&iwcq->lock, flags); 4658 /* Only promote to arm the CQ for any event if the last arm event was solicited. */ 4659 if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED && notify_flags != IB_CQ_SOLICITED) 4660 promo_event = true; 4661 4662 if (!atomic_cmpxchg(&iwcq->armed, 0, 1) || promo_event) { 4663 iwcq->last_notify = cq_notify; 4664 irdma_uk_cq_request_notification(ukcq, cq_notify); 4665 } 4666 4667 if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && 4668 (!irdma_uk_cq_empty(ukcq) || !list_empty(&iwcq->cmpl_generated))) 4669 ret = 1; 4670 spin_unlock_irqrestore(&iwcq->lock, flags); 4671 4672 return ret; 4673 } 4674 4675 static const struct rdma_stat_desc irdma_hw_stat_descs[] = { 4676 /* gen1 - 32-bit */ 4677 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards", 4678 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts", 4679 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes", 4680 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards", 4681 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts", 4682 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes", 4683 [IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors", 4684 /* gen1 - 64-bit */ 4685 [IRDMA_HW_STAT_INDEX_IP4RXOCTS].name = "ip4InOctets", 4686 [IRDMA_HW_STAT_INDEX_IP4RXPKTS].name = "ip4InPkts", 4687 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS].name = "ip4InReasmRqd", 4688 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS].name = "ip4InMcastPkts", 4689 [IRDMA_HW_STAT_INDEX_IP4TXOCTS].name = "ip4OutOctets", 4690 [IRDMA_HW_STAT_INDEX_IP4TXPKTS].name = "ip4OutPkts", 4691 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS].name = "ip4OutSegRqd", 4692 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS].name = "ip4OutMcastPkts", 4693 [IRDMA_HW_STAT_INDEX_IP6RXOCTS].name = "ip6InOctets", 4694 [IRDMA_HW_STAT_INDEX_IP6RXPKTS].name = "ip6InPkts", 4695 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS].name = "ip6InReasmRqd", 4696 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS].name = "ip6InMcastPkts", 4697 [IRDMA_HW_STAT_INDEX_IP6TXOCTS].name = "ip6OutOctets", 4698 [IRDMA_HW_STAT_INDEX_IP6TXPKTS].name = "ip6OutPkts", 4699 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS].name = "ip6OutSegRqd", 4700 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS].name = "ip6OutMcastPkts", 4701 [IRDMA_HW_STAT_INDEX_RDMARXRDS].name = "InRdmaReads", 4702 [IRDMA_HW_STAT_INDEX_RDMARXSNDS].name = "InRdmaSends", 4703 [IRDMA_HW_STAT_INDEX_RDMARXWRS].name = "InRdmaWrites", 4704 [IRDMA_HW_STAT_INDEX_RDMATXRDS].name = "OutRdmaReads", 4705 [IRDMA_HW_STAT_INDEX_RDMATXSNDS].name = "OutRdmaSends", 4706 [IRDMA_HW_STAT_INDEX_RDMATXWRS].name = "OutRdmaWrites", 4707 [IRDMA_HW_STAT_INDEX_RDMAVBND].name = "RdmaBnd", 4708 [IRDMA_HW_STAT_INDEX_RDMAVINV].name = "RdmaInv", 4709 4710 /* gen2 - 32-bit */ 4711 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled", 4712 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored", 4713 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent", 4714 /* gen2 - 64-bit */ 4715 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS].name = "ip4InMcastOctets", 4716 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS].name = "ip4OutMcastOctets", 4717 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS].name = "ip6InMcastOctets", 4718 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS].name = "ip6OutMcastOctets", 4719 [IRDMA_HW_STAT_INDEX_UDPRXPKTS].name = "RxUDP", 4720 [IRDMA_HW_STAT_INDEX_UDPTXPKTS].name = "TxUDP", 4721 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS].name = "RxECNMrkd", 4722 [IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "RetransSegs", 4723 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "InOptErrors", 4724 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "InProtoErrors", 4725 [IRDMA_HW_STAT_INDEX_TCPRXSEGS].name = "InSegs", 4726 [IRDMA_HW_STAT_INDEX_TCPTXSEG].name = "OutSegs", 4727 4728 /* gen3 */ 4729 [IRDMA_HW_STAT_INDEX_RNR_SENT].name = "RNR sent", 4730 [IRDMA_HW_STAT_INDEX_RNR_RCVD].name = "RNR received", 4731 [IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT].name = "ord limit count", 4732 [IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT].name = "ird limit count", 4733 [IRDMA_HW_STAT_INDEX_RDMARXATS].name = "Rx atomics", 4734 [IRDMA_HW_STAT_INDEX_RDMATXATS].name = "Tx atomics", 4735 [IRDMA_HW_STAT_INDEX_NAKSEQERR].name = "Nak Sequence Error", 4736 [IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED].name = "Nak Sequence Error Implied", 4737 [IRDMA_HW_STAT_INDEX_RTO].name = "RTO", 4738 [IRDMA_HW_STAT_INDEX_RXOOOPKTS].name = "Rcvd Out of order packets", 4739 [IRDMA_HW_STAT_INDEX_ICRCERR].name = "CRC errors", 4740 }; 4741 4742 static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, 4743 struct ib_port_immutable *immutable) 4744 { 4745 struct ib_port_attr attr; 4746 int err; 4747 4748 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 4749 err = ib_query_port(ibdev, port_num, &attr); 4750 if (err) 4751 return err; 4752 4753 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 4754 immutable->pkey_tbl_len = attr.pkey_tbl_len; 4755 immutable->gid_tbl_len = attr.gid_tbl_len; 4756 4757 return 0; 4758 } 4759 4760 static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num, 4761 struct ib_port_immutable *immutable) 4762 { 4763 struct ib_port_attr attr; 4764 int err; 4765 4766 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 4767 err = ib_query_port(ibdev, port_num, &attr); 4768 if (err) 4769 return err; 4770 immutable->gid_tbl_len = attr.gid_tbl_len; 4771 4772 return 0; 4773 } 4774 4775 static void irdma_get_dev_fw_str(struct ib_device *dev, char *str) 4776 { 4777 struct irdma_device *iwdev = to_iwdev(dev); 4778 4779 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", 4780 irdma_fw_major_ver(&iwdev->rf->sc_dev), 4781 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 4782 } 4783 4784 /** 4785 * irdma_alloc_hw_port_stats - Allocate a hw stats structure 4786 * @ibdev: device pointer from stack 4787 * @port_num: port number 4788 */ 4789 static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev, 4790 u32 port_num) 4791 { 4792 struct irdma_device *iwdev = to_iwdev(ibdev); 4793 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 4794 4795 int num_counters = dev->hw_attrs.max_stat_idx; 4796 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 4797 4798 return rdma_alloc_hw_stats_struct(irdma_hw_stat_descs, num_counters, 4799 lifespan); 4800 } 4801 4802 /** 4803 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 4804 * @ibdev: device pointer from stack 4805 * @stats: stats pointer from stack 4806 * @port_num: port number 4807 * @index: which hw counter the stack is requesting we update 4808 */ 4809 static int irdma_get_hw_stats(struct ib_device *ibdev, 4810 struct rdma_hw_stats *stats, u32 port_num, 4811 int index) 4812 { 4813 struct irdma_device *iwdev = to_iwdev(ibdev); 4814 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 4815 4816 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 4817 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 4818 else 4819 irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat); 4820 4821 memcpy(&stats->value[0], hw_stats, sizeof(u64) * stats->num_counters); 4822 4823 return stats->num_counters; 4824 } 4825 4826 /** 4827 * irdma_query_gid - Query port GID 4828 * @ibdev: device pointer from stack 4829 * @port: port number 4830 * @index: Entry index 4831 * @gid: Global ID 4832 */ 4833 static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index, 4834 union ib_gid *gid) 4835 { 4836 struct irdma_device *iwdev = to_iwdev(ibdev); 4837 4838 memset(gid->raw, 0, sizeof(gid->raw)); 4839 ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); 4840 4841 return 0; 4842 } 4843 4844 /** 4845 * mcast_list_add - Add a new mcast item to list 4846 * @rf: RDMA PCI function 4847 * @new_elem: pointer to element to add 4848 */ 4849 static void mcast_list_add(struct irdma_pci_f *rf, 4850 struct mc_table_list *new_elem) 4851 { 4852 list_add(&new_elem->list, &rf->mc_qht_list.list); 4853 } 4854 4855 /** 4856 * mcast_list_del - Remove an mcast item from list 4857 * @mc_qht_elem: pointer to mcast table list element 4858 */ 4859 static void mcast_list_del(struct mc_table_list *mc_qht_elem) 4860 { 4861 if (mc_qht_elem) 4862 list_del(&mc_qht_elem->list); 4863 } 4864 4865 /** 4866 * mcast_list_lookup_ip - Search mcast list for address 4867 * @rf: RDMA PCI function 4868 * @ip_mcast: pointer to mcast IP address 4869 */ 4870 static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf, 4871 u32 *ip_mcast) 4872 { 4873 struct mc_table_list *mc_qht_el; 4874 struct list_head *pos, *q; 4875 4876 list_for_each_safe (pos, q, &rf->mc_qht_list.list) { 4877 mc_qht_el = list_entry(pos, struct mc_table_list, list); 4878 if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, 4879 sizeof(mc_qht_el->mc_info.dest_ip))) 4880 return mc_qht_el; 4881 } 4882 4883 return NULL; 4884 } 4885 4886 /** 4887 * irdma_mcast_cqp_op - perform a mcast cqp operation 4888 * @iwdev: irdma device 4889 * @mc_grp_ctx: mcast group info 4890 * @op: operation 4891 * 4892 * returns error status 4893 */ 4894 static int irdma_mcast_cqp_op(struct irdma_device *iwdev, 4895 struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) 4896 { 4897 struct cqp_cmds_info *cqp_info; 4898 struct irdma_cqp_request *cqp_request; 4899 int status; 4900 4901 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 4902 if (!cqp_request) 4903 return -ENOMEM; 4904 4905 cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; 4906 cqp_info = &cqp_request->info; 4907 cqp_info->cqp_cmd = op; 4908 cqp_info->post_sq = 1; 4909 cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; 4910 cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; 4911 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 4912 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 4913 4914 return status; 4915 } 4916 4917 /** 4918 * irdma_mcast_mac - Get the multicast MAC for an IP address 4919 * @ip_addr: IPv4 or IPv6 address 4920 * @mac: pointer to result MAC address 4921 * @ipv4: flag indicating IPv4 or IPv6 4922 * 4923 */ 4924 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4) 4925 { 4926 u8 *ip = (u8 *)ip_addr; 4927 4928 if (ipv4) { 4929 unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00, 4930 0x00, 0x00}; 4931 4932 mac4[3] = ip[2] & 0x7F; 4933 mac4[4] = ip[1]; 4934 mac4[5] = ip[0]; 4935 ether_addr_copy(mac, mac4); 4936 } else { 4937 unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00, 4938 0x00, 0x00}; 4939 4940 mac6[2] = ip[3]; 4941 mac6[3] = ip[2]; 4942 mac6[4] = ip[1]; 4943 mac6[5] = ip[0]; 4944 ether_addr_copy(mac, mac6); 4945 } 4946 } 4947 4948 /** 4949 * irdma_attach_mcast - attach a qp to a multicast group 4950 * @ibqp: ptr to qp 4951 * @ibgid: pointer to global ID 4952 * @lid: local ID 4953 * 4954 * returns error status 4955 */ 4956 static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 4957 { 4958 struct irdma_qp *iwqp = to_iwqp(ibqp); 4959 struct irdma_device *iwdev = iwqp->iwdev; 4960 struct irdma_pci_f *rf = iwdev->rf; 4961 struct mc_table_list *mc_qht_elem; 4962 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 4963 unsigned long flags; 4964 u32 ip_addr[4] = {}; 4965 u32 mgn; 4966 u32 no_mgs; 4967 int ret = 0; 4968 bool ipv4; 4969 u16 vlan_id; 4970 union irdma_sockaddr sgid_addr; 4971 unsigned char dmac[ETH_ALEN]; 4972 4973 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 4974 4975 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { 4976 irdma_copy_ip_ntohl(ip_addr, 4977 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4978 irdma_get_vlan_mac_ipv6(ip_addr, &vlan_id, NULL); 4979 ipv4 = false; 4980 ibdev_dbg(&iwdev->ibdev, 4981 "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, 4982 ip_addr); 4983 irdma_mcast_mac(ip_addr, dmac, false); 4984 } else { 4985 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4986 ipv4 = true; 4987 vlan_id = irdma_get_vlan_ipv4(ip_addr); 4988 irdma_mcast_mac(ip_addr, dmac, true); 4989 ibdev_dbg(&iwdev->ibdev, 4990 "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n", 4991 ibqp->qp_num, ip_addr, dmac); 4992 } 4993 4994 spin_lock_irqsave(&rf->qh_list_lock, flags); 4995 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 4996 if (!mc_qht_elem) { 4997 struct irdma_dma_mem *dma_mem_mc; 4998 4999 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5000 mc_qht_elem = kzalloc_obj(*mc_qht_elem); 5001 if (!mc_qht_elem) 5002 return -ENOMEM; 5003 5004 mc_qht_elem->mc_info.ipv4_valid = ipv4; 5005 memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, 5006 sizeof(mc_qht_elem->mc_info.dest_ip)); 5007 ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, 5008 &mgn, &rf->next_mcg); 5009 if (ret) { 5010 kfree(mc_qht_elem); 5011 return -ENOMEM; 5012 } 5013 5014 mc_qht_elem->mc_info.mgn = mgn; 5015 dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; 5016 dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX, 5017 IRDMA_HW_PAGE_SIZE); 5018 dma_mem_mc->va = dma_alloc_coherent(rf->hw.device, 5019 dma_mem_mc->size, 5020 &dma_mem_mc->pa, 5021 GFP_KERNEL); 5022 if (!dma_mem_mc->va) { 5023 irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); 5024 kfree(mc_qht_elem); 5025 return -ENOMEM; 5026 } 5027 5028 mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; 5029 memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, 5030 sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); 5031 mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; 5032 mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; 5033 if (vlan_id < VLAN_N_VID) 5034 mc_qht_elem->mc_grp_ctx.vlan_valid = true; 5035 mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id; 5036 mc_qht_elem->mc_grp_ctx.qs_handle = 5037 iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; 5038 ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); 5039 5040 spin_lock_irqsave(&rf->qh_list_lock, flags); 5041 mcast_list_add(rf, mc_qht_elem); 5042 } else { 5043 if (mc_qht_elem->mc_grp_ctx.no_of_mgs == 5044 IRDMA_MAX_MGS_PER_CTX) { 5045 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5046 return -ENOMEM; 5047 } 5048 } 5049 5050 mcg_info.qp_id = iwqp->ibqp.qp_num; 5051 no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; 5052 irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 5053 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5054 5055 /* Only if there is a change do we need to modify or create */ 5056 if (!no_mgs) { 5057 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 5058 IRDMA_OP_MC_CREATE); 5059 } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { 5060 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 5061 IRDMA_OP_MC_MODIFY); 5062 } else { 5063 return 0; 5064 } 5065 5066 if (ret) 5067 goto error; 5068 5069 return 0; 5070 5071 error: 5072 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 5073 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 5074 mcast_list_del(mc_qht_elem); 5075 dma_free_coherent(rf->hw.device, 5076 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 5077 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 5078 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 5079 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 5080 irdma_free_rsrc(rf, rf->allocated_mcgs, 5081 mc_qht_elem->mc_grp_ctx.mg_id); 5082 kfree(mc_qht_elem); 5083 } 5084 5085 return ret; 5086 } 5087 5088 /** 5089 * irdma_detach_mcast - detach a qp from a multicast group 5090 * @ibqp: ptr to qp 5091 * @ibgid: pointer to global ID 5092 * @lid: local ID 5093 * 5094 * returns error status 5095 */ 5096 static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 5097 { 5098 struct irdma_qp *iwqp = to_iwqp(ibqp); 5099 struct irdma_device *iwdev = iwqp->iwdev; 5100 struct irdma_pci_f *rf = iwdev->rf; 5101 u32 ip_addr[4] = {}; 5102 struct mc_table_list *mc_qht_elem; 5103 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 5104 int ret; 5105 unsigned long flags; 5106 union irdma_sockaddr sgid_addr; 5107 5108 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 5109 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) 5110 irdma_copy_ip_ntohl(ip_addr, 5111 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 5112 else 5113 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 5114 5115 spin_lock_irqsave(&rf->qh_list_lock, flags); 5116 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 5117 if (!mc_qht_elem) { 5118 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5119 ibdev_dbg(&iwdev->ibdev, 5120 "VERBS: address not found MCG\n"); 5121 return 0; 5122 } 5123 5124 mcg_info.qp_id = iwqp->ibqp.qp_num; 5125 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 5126 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 5127 mcast_list_del(mc_qht_elem); 5128 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5129 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 5130 IRDMA_OP_MC_DESTROY); 5131 if (ret) { 5132 ibdev_dbg(&iwdev->ibdev, 5133 "VERBS: failed MC_DESTROY MCG\n"); 5134 spin_lock_irqsave(&rf->qh_list_lock, flags); 5135 mcast_list_add(rf, mc_qht_elem); 5136 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5137 return -EAGAIN; 5138 } 5139 5140 dma_free_coherent(rf->hw.device, 5141 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 5142 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 5143 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 5144 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 5145 irdma_free_rsrc(rf, rf->allocated_mcgs, 5146 mc_qht_elem->mc_grp_ctx.mg_id); 5147 kfree(mc_qht_elem); 5148 } else { 5149 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 5150 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 5151 IRDMA_OP_MC_MODIFY); 5152 if (ret) { 5153 ibdev_dbg(&iwdev->ibdev, 5154 "VERBS: failed Modify MCG\n"); 5155 return ret; 5156 } 5157 } 5158 5159 return 0; 5160 } 5161 5162 static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep) 5163 { 5164 struct irdma_pci_f *rf = iwdev->rf; 5165 int err; 5166 5167 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah->sc_ah.ah_info.ah_idx, 5168 &rf->next_ah); 5169 if (err) 5170 return err; 5171 5172 err = irdma_ah_cqp_op(rf, &ah->sc_ah, IRDMA_OP_AH_CREATE, sleep, 5173 irdma_gsi_ud_qp_ah_cb, &ah->sc_ah); 5174 5175 if (err) { 5176 ibdev_dbg(&iwdev->ibdev, "VERBS: CQP-OP Create AH fail"); 5177 goto err_ah_create; 5178 } 5179 5180 if (!sleep) { 5181 const u64 tmout_ms = irdma_get_timeout_threshold(&rf->sc_dev) * 5182 CQP_COMPL_WAIT_TIME_MS; 5183 5184 if (poll_timeout_us_atomic(irdma_cqp_ce_handler(rf, 5185 &rf->ccq.sc_cq), 5186 ah->sc_ah.ah_info.ah_valid, 1, 5187 tmout_ms * USEC_PER_MSEC, false)) { 5188 ibdev_dbg(&iwdev->ibdev, 5189 "VERBS: CQP create AH timed out"); 5190 err = -ETIMEDOUT; 5191 goto err_ah_create; 5192 } 5193 } 5194 return 0; 5195 5196 err_ah_create: 5197 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx); 5198 5199 return err; 5200 } 5201 5202 static int irdma_setup_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr) 5203 { 5204 struct irdma_pd *pd = to_iwpd(ibah->pd); 5205 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5206 struct rdma_ah_attr *ah_attr = attr->ah_attr; 5207 const struct ib_gid_attr *sgid_attr; 5208 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5209 struct irdma_pci_f *rf = iwdev->rf; 5210 struct irdma_sc_ah *sc_ah; 5211 struct irdma_ah_info *ah_info; 5212 union irdma_sockaddr sgid_addr, dgid_addr; 5213 int err; 5214 u8 dmac[ETH_ALEN]; 5215 5216 ah->pd = pd; 5217 sc_ah = &ah->sc_ah; 5218 sc_ah->ah_info.vsi = &iwdev->vsi; 5219 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 5220 ah->sgid_index = ah_attr->grh.sgid_index; 5221 sgid_attr = ah_attr->grh.sgid_attr; 5222 memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid)); 5223 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid_attr->gid); 5224 rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid); 5225 ah->av.attrs = *ah_attr; 5226 ah->av.net_type = rdma_gid_attr_network_type(sgid_attr); 5227 ah_info = &sc_ah->ah_info; 5228 ah_info->pd_idx = pd->sc_pd.pd_id; 5229 if (ah_attr->ah_flags & IB_AH_GRH) { 5230 ah_info->flow_label = ah_attr->grh.flow_label; 5231 ah_info->hop_ttl = ah_attr->grh.hop_limit; 5232 ah_info->tc_tos = ah_attr->grh.traffic_class; 5233 } 5234 5235 ether_addr_copy(dmac, ah_attr->roce.dmac); 5236 if (ah->av.net_type == RDMA_NETWORK_IPV4) { 5237 ah_info->ipv4_valid = true; 5238 ah_info->dest_ip_addr[0] = 5239 ntohl(dgid_addr.saddr_in.sin_addr.s_addr); 5240 ah_info->src_ip_addr[0] = 5241 ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 5242 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 5243 ah_info->dest_ip_addr[0]); 5244 if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) { 5245 ah_info->do_lpbk = true; 5246 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true); 5247 } 5248 } else { 5249 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 5250 dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 5251 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 5252 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 5253 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 5254 ah_info->dest_ip_addr); 5255 if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr)) { 5256 ah_info->do_lpbk = true; 5257 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false); 5258 } 5259 } 5260 5261 err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag, 5262 ah_info->mac_addr); 5263 if (err) 5264 return err; 5265 5266 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, 5267 ah_info->ipv4_valid, dmac); 5268 5269 if (ah_info->dst_arpindex == -1) 5270 return -EINVAL; 5271 5272 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) 5273 ah_info->vlan_tag = 0; 5274 5275 if (ah_info->vlan_tag < VLAN_N_VID) { 5276 u8 prio = rt_tos2priority(ah_info->tc_tos); 5277 5278 prio = irdma_roce_get_vlan_prio(sgid_attr, prio); 5279 5280 ah_info->vlan_tag |= (u16)prio << VLAN_PRIO_SHIFT; 5281 ah_info->insert_vlan_tag = true; 5282 } 5283 5284 return 0; 5285 } 5286 5287 /** 5288 * irdma_ah_exists - Check for existing identical AH 5289 * @iwdev: irdma device 5290 * @new_ah: AH to check for 5291 * 5292 * returns true if AH is found, false if not found. 5293 */ 5294 static bool irdma_ah_exists(struct irdma_device *iwdev, 5295 struct irdma_ah *new_ah) 5296 { 5297 struct irdma_ah *ah; 5298 u32 key = new_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 5299 new_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 5300 new_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 5301 new_ah->sc_ah.ah_info.dest_ip_addr[3]; 5302 5303 hash_for_each_possible(iwdev->rf->ah_hash_tbl, ah, list, key) { 5304 /* Set ah_valid and ah_id the same so memcmp can work */ 5305 new_ah->sc_ah.ah_info.ah_idx = ah->sc_ah.ah_info.ah_idx; 5306 new_ah->sc_ah.ah_info.ah_valid = ah->sc_ah.ah_info.ah_valid; 5307 if (!memcmp(&ah->sc_ah.ah_info, &new_ah->sc_ah.ah_info, 5308 sizeof(ah->sc_ah.ah_info))) { 5309 refcount_inc(&ah->refcnt); 5310 new_ah->parent_ah = ah; 5311 return true; 5312 } 5313 } 5314 5315 return false; 5316 } 5317 5318 /** 5319 * irdma_destroy_ah - Destroy address handle 5320 * @ibah: pointer to address handle 5321 * @ah_flags: flags for sleepable 5322 */ 5323 static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 5324 { 5325 struct irdma_device *iwdev = to_iwdev(ibah->device); 5326 struct irdma_ah *ah = to_iwah(ibah); 5327 5328 if ((ah_flags & RDMA_DESTROY_AH_SLEEPABLE) && ah->parent_ah) { 5329 mutex_lock(&iwdev->rf->ah_tbl_lock); 5330 if (!refcount_dec_and_test(&ah->parent_ah->refcnt)) { 5331 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5332 return 0; 5333 } 5334 hash_del(&ah->parent_ah->list); 5335 kfree(ah->parent_ah); 5336 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5337 } 5338 5339 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 5340 false, NULL, ah); 5341 5342 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 5343 ah->sc_ah.ah_info.ah_idx); 5344 5345 return 0; 5346 } 5347 5348 /** 5349 * irdma_create_user_ah - create user address handle 5350 * @ibah: address handle 5351 * @attr: address handle attributes 5352 * @udata: User data 5353 * 5354 * returns 0 on success, error otherwise 5355 */ 5356 static int irdma_create_user_ah(struct ib_ah *ibah, 5357 struct rdma_ah_init_attr *attr, 5358 struct ib_udata *udata) 5359 { 5360 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) 5361 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5362 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5363 struct irdma_create_ah_resp uresp = {}; 5364 struct irdma_ah *parent_ah; 5365 int err; 5366 5367 err = ib_is_udata_in_empty(udata); 5368 if (err) 5369 return err; 5370 5371 if (udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) 5372 return -EINVAL; 5373 5374 err = irdma_setup_ah(ibah, attr); 5375 if (err) 5376 return err; 5377 mutex_lock(&iwdev->rf->ah_tbl_lock); 5378 if (!irdma_ah_exists(iwdev, ah)) { 5379 err = irdma_create_hw_ah(iwdev, ah, true); 5380 if (err) { 5381 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5382 return err; 5383 } 5384 /* Add new AH to list */ 5385 parent_ah = kmemdup(ah, sizeof(*ah), GFP_KERNEL); 5386 if (parent_ah) { 5387 u32 key = parent_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 5388 parent_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 5389 parent_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 5390 parent_ah->sc_ah.ah_info.dest_ip_addr[3]; 5391 5392 ah->parent_ah = parent_ah; 5393 hash_add(iwdev->rf->ah_hash_tbl, &parent_ah->list, key); 5394 refcount_set(&parent_ah->refcnt, 1); 5395 } 5396 } 5397 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5398 5399 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 5400 err = ib_respond_udata(udata, uresp); 5401 if (err) 5402 irdma_destroy_ah(ibah, attr->flags); 5403 5404 return err; 5405 } 5406 5407 /** 5408 * irdma_create_ah - create address handle 5409 * @ibah: address handle 5410 * @attr: address handle attributes 5411 * @udata: NULL 5412 * 5413 * returns 0 on success, error otherwise 5414 */ 5415 static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr, 5416 struct ib_udata *udata) 5417 { 5418 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5419 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5420 int err; 5421 5422 err = ib_no_udata_io(udata); 5423 if (err) 5424 return err; 5425 5426 err = irdma_setup_ah(ibah, attr); 5427 if (err) 5428 return err; 5429 err = irdma_create_hw_ah(iwdev, ah, attr->flags & RDMA_CREATE_AH_SLEEPABLE); 5430 5431 return err; 5432 } 5433 5434 /** 5435 * irdma_query_ah - Query address handle 5436 * @ibah: pointer to address handle 5437 * @ah_attr: address handle attributes 5438 */ 5439 static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 5440 { 5441 struct irdma_ah *ah = to_iwah(ibah); 5442 5443 memset(ah_attr, 0, sizeof(*ah_attr)); 5444 if (ah->av.attrs.ah_flags & IB_AH_GRH) { 5445 ah_attr->ah_flags = IB_AH_GRH; 5446 ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; 5447 ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; 5448 ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; 5449 ah_attr->grh.sgid_index = ah->sgid_index; 5450 memcpy(&ah_attr->grh.dgid, &ah->dgid, 5451 sizeof(ah_attr->grh.dgid)); 5452 } 5453 5454 return 0; 5455 } 5456 5457 static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, 5458 u32 port_num) 5459 { 5460 return IB_LINK_LAYER_ETHERNET; 5461 } 5462 5463 static const struct ib_device_ops irdma_gen1_dev_ops = { 5464 .dealloc_driver = irdma_ib_dealloc_device, 5465 }; 5466 5467 static const struct ib_device_ops irdma_gen3_dev_ops = { 5468 .alloc_mw = irdma_alloc_mw, 5469 .create_srq = irdma_create_srq, 5470 .dealloc_mw = irdma_dealloc_mw, 5471 .destroy_srq = irdma_destroy_srq, 5472 .modify_srq = irdma_modify_srq, 5473 .post_srq_recv = irdma_post_srq_recv, 5474 .query_srq = irdma_query_srq, 5475 }; 5476 5477 static const struct ib_device_ops irdma_roce_dev_ops = { 5478 .attach_mcast = irdma_attach_mcast, 5479 .create_ah = irdma_create_ah, 5480 .create_user_ah = irdma_create_user_ah, 5481 .destroy_ah = irdma_destroy_ah, 5482 .detach_mcast = irdma_detach_mcast, 5483 .get_link_layer = irdma_get_link_layer, 5484 .get_port_immutable = irdma_roce_port_immutable, 5485 .modify_qp = irdma_modify_qp_roce, 5486 .query_ah = irdma_query_ah, 5487 .query_pkey = irdma_query_pkey, 5488 }; 5489 5490 static const struct ib_device_ops irdma_iw_dev_ops = { 5491 .get_port_immutable = irdma_iw_port_immutable, 5492 .iw_accept = irdma_accept, 5493 .iw_add_ref = irdma_qp_add_ref, 5494 .iw_connect = irdma_connect, 5495 .iw_create_listen = irdma_create_listen, 5496 .iw_destroy_listen = irdma_destroy_listen, 5497 .iw_get_qp = irdma_get_qp, 5498 .iw_reject = irdma_reject, 5499 .iw_rem_ref = irdma_qp_rem_ref, 5500 .modify_qp = irdma_modify_qp, 5501 .query_gid = irdma_query_gid, 5502 }; 5503 5504 static const struct ib_device_ops irdma_dev_ops = { 5505 .owner = THIS_MODULE, 5506 .driver_id = RDMA_DRIVER_IRDMA, 5507 .uverbs_abi_ver = IRDMA_ABI_VER, 5508 .uverbs_robust_udata = true, 5509 5510 .alloc_hw_port_stats = irdma_alloc_hw_port_stats, 5511 .alloc_mr = irdma_alloc_mr, 5512 .alloc_pd = irdma_alloc_pd, 5513 .alloc_ucontext = irdma_alloc_ucontext, 5514 .create_cq = irdma_create_cq, 5515 .create_qp = irdma_create_qp, 5516 .dealloc_driver = irdma_ib_dealloc_device, 5517 .dealloc_mw = irdma_dealloc_mw, 5518 .dealloc_pd = irdma_dealloc_pd, 5519 .dealloc_ucontext = irdma_dealloc_ucontext, 5520 .dereg_mr = irdma_dereg_mr, 5521 .destroy_cq = irdma_destroy_cq, 5522 .destroy_qp = irdma_destroy_qp, 5523 .disassociate_ucontext = irdma_disassociate_ucontext, 5524 .get_dev_fw_str = irdma_get_dev_fw_str, 5525 .get_dma_mr = irdma_get_dma_mr, 5526 .get_hw_stats = irdma_get_hw_stats, 5527 .map_mr_sg = irdma_map_mr_sg, 5528 .mmap = irdma_mmap, 5529 .mmap_free = irdma_mmap_free, 5530 .poll_cq = irdma_poll_cq, 5531 .post_recv = irdma_post_recv, 5532 .post_send = irdma_post_send, 5533 .query_device = irdma_query_device, 5534 .query_port = irdma_query_port, 5535 .query_qp = irdma_query_qp, 5536 .reg_user_mr = irdma_reg_user_mr, 5537 .reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf, 5538 .rereg_user_mr = irdma_rereg_user_mr, 5539 .req_notify_cq = irdma_req_notify_cq, 5540 .resize_user_cq = irdma_resize_cq, 5541 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd), 5542 INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext), 5543 INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), 5544 INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), 5545 INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), 5546 INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp), 5547 INIT_RDMA_OBJ_SIZE(ib_srq, irdma_srq, ibsrq), 5548 }; 5549 5550 /** 5551 * irdma_init_roce_device - initialization of roce rdma device 5552 * @iwdev: irdma device 5553 */ 5554 static void irdma_init_roce_device(struct irdma_device *iwdev) 5555 { 5556 iwdev->ibdev.node_type = RDMA_NODE_IB_CA; 5557 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 5558 iwdev->netdev->dev_addr); 5559 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); 5560 } 5561 5562 /** 5563 * irdma_init_iw_device - initialization of iwarp rdma device 5564 * @iwdev: irdma device 5565 */ 5566 static void irdma_init_iw_device(struct irdma_device *iwdev) 5567 { 5568 struct net_device *netdev = iwdev->netdev; 5569 5570 iwdev->ibdev.node_type = RDMA_NODE_RNIC; 5571 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 5572 netdev->dev_addr); 5573 memcpy(iwdev->ibdev.iw_ifname, netdev->name, 5574 sizeof(iwdev->ibdev.iw_ifname)); 5575 ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops); 5576 } 5577 5578 /** 5579 * irdma_init_rdma_device - initialization of rdma device 5580 * @iwdev: irdma device 5581 */ 5582 static void irdma_init_rdma_device(struct irdma_device *iwdev) 5583 { 5584 struct pci_dev *pcidev = iwdev->rf->pcidev; 5585 5586 if (iwdev->roce_mode) 5587 irdma_init_roce_device(iwdev); 5588 else 5589 irdma_init_iw_device(iwdev); 5590 5591 iwdev->ibdev.phys_port_cnt = 1; 5592 iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; 5593 iwdev->ibdev.dev.parent = &pcidev->dev; 5594 ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops); 5595 if (iwdev->rf->rdma_ver == IRDMA_GEN_1) 5596 ib_set_device_ops(&iwdev->ibdev, &irdma_gen1_dev_ops); 5597 if (iwdev->rf->rdma_ver >= IRDMA_GEN_3) 5598 ib_set_device_ops(&iwdev->ibdev, &irdma_gen3_dev_ops); 5599 } 5600 5601 /** 5602 * irdma_port_ibevent - indicate port event 5603 * @iwdev: irdma device 5604 */ 5605 void irdma_port_ibevent(struct irdma_device *iwdev) 5606 { 5607 struct ib_event event; 5608 5609 event.device = &iwdev->ibdev; 5610 event.element.port_num = 1; 5611 event.event = 5612 iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; 5613 ib_dispatch_event(&event); 5614 } 5615 5616 /** 5617 * irdma_ib_unregister_device - unregister rdma device from IB 5618 * core 5619 * @iwdev: irdma device 5620 */ 5621 void irdma_ib_unregister_device(struct irdma_device *iwdev) 5622 { 5623 iwdev->iw_status = 0; 5624 irdma_port_ibevent(iwdev); 5625 ib_unregister_device(&iwdev->ibdev); 5626 } 5627 5628 /** 5629 * irdma_ib_register_device - register irdma device to IB core 5630 * @iwdev: irdma device 5631 */ 5632 int irdma_ib_register_device(struct irdma_device *iwdev) 5633 { 5634 int ret; 5635 5636 irdma_init_rdma_device(iwdev); 5637 5638 ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1); 5639 if (ret) 5640 goto error; 5641 dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX); 5642 ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device); 5643 if (ret) 5644 goto error; 5645 5646 iwdev->iw_status = 1; 5647 irdma_port_ibevent(iwdev); 5648 5649 return 0; 5650 5651 error: 5652 if (ret) 5653 ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n"); 5654 5655 return ret; 5656 } 5657 5658 /** 5659 * irdma_ib_dealloc_device 5660 * @ibdev: ib device 5661 * 5662 * callback from ibdev dealloc_driver to deallocate resources 5663 * unber irdma device 5664 */ 5665 void irdma_ib_dealloc_device(struct ib_device *ibdev) 5666 { 5667 struct irdma_device *iwdev = to_iwdev(ibdev); 5668 5669 irdma_rt_deinit_hw(iwdev); 5670 if (!iwdev->is_vport) { 5671 irdma_ctrl_deinit_hw(iwdev->rf); 5672 if (iwdev->rf->vchnl_wq) { 5673 destroy_workqueue(iwdev->rf->vchnl_wq); 5674 mutex_destroy(&iwdev->rf->sc_dev.vchnl_mutex); 5675 } 5676 } 5677 } 5678