1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_query_device - get device attributes 7 * @ibdev: device pointer from stack 8 * @props: returning device attributes 9 * @udata: user data 10 */ 11 static int irdma_query_device(struct ib_device *ibdev, 12 struct ib_device_attr *props, 13 struct ib_udata *udata) 14 { 15 struct irdma_device *iwdev = to_iwdev(ibdev); 16 struct irdma_pci_f *rf = iwdev->rf; 17 struct pci_dev *pcidev = iwdev->rf->pcidev; 18 struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; 19 20 if (udata->inlen || udata->outlen) 21 return -EINVAL; 22 23 memset(props, 0, sizeof(*props)); 24 addrconf_addr_eui48((u8 *)&props->sys_image_guid, 25 iwdev->netdev->dev_addr); 26 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | 27 irdma_fw_minor_ver(&rf->sc_dev); 28 props->device_cap_flags = IB_DEVICE_MEM_WINDOW | 29 IB_DEVICE_MEM_MGT_EXTENSIONS; 30 props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 31 props->vendor_id = pcidev->vendor; 32 props->vendor_part_id = pcidev->device; 33 34 props->hw_ver = rf->pcidev->revision; 35 props->page_size_cap = hw_attrs->page_size_cap; 36 props->max_mr_size = hw_attrs->max_mr_size; 37 props->max_qp = rf->max_qp - rf->used_qps; 38 props->max_qp_wr = hw_attrs->max_qp_wr; 39 props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 40 props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 41 props->max_cq = rf->max_cq - rf->used_cqs; 42 props->max_cqe = rf->max_cqe - 1; 43 props->max_mr = rf->max_mr - rf->used_mrs; 44 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) 45 props->max_mw = props->max_mr; 46 props->max_pd = rf->max_pd - rf->used_pds; 47 props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; 48 props->max_qp_rd_atom = hw_attrs->max_hw_ird; 49 props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; 50 if (rdma_protocol_roce(ibdev, 1)) { 51 props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN; 52 props->max_pkeys = IRDMA_PKEY_TBL_SZ; 53 } 54 55 props->max_ah = rf->max_ah; 56 props->max_mcast_grp = rf->max_mcg; 57 props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; 58 props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; 59 props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; 60 props->max_srq = rf->max_srq - rf->used_srqs; 61 props->max_srq_wr = IRDMA_MAX_SRQ_WRS; 62 props->max_srq_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 63 if (hw_attrs->uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 64 props->atomic_cap = IB_ATOMIC_HCA; 65 else 66 props->atomic_cap = IB_ATOMIC_NONE; 67 props->masked_atomic_cap = props->atomic_cap; 68 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) { 69 #define HCA_CORE_CLOCK_KHZ 1000000UL 70 props->timestamp_mask = GENMASK(31, 0); 71 props->hca_core_clock = HCA_CORE_CLOCK_KHZ; 72 } 73 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) 74 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; 75 76 return 0; 77 } 78 79 /** 80 * irdma_query_port - get port attributes 81 * @ibdev: device pointer from stack 82 * @port: port number for query 83 * @props: returning device attributes 84 */ 85 static int irdma_query_port(struct ib_device *ibdev, u32 port, 86 struct ib_port_attr *props) 87 { 88 struct irdma_device *iwdev = to_iwdev(ibdev); 89 struct net_device *netdev = iwdev->netdev; 90 91 /* no need to zero out pros here. done by caller */ 92 93 props->max_mtu = IB_MTU_4096; 94 props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); 95 props->lid = 1; 96 props->lmc = 0; 97 props->sm_lid = 0; 98 props->sm_sl = 0; 99 if (netif_carrier_ok(netdev) && netif_running(netdev)) { 100 props->state = IB_PORT_ACTIVE; 101 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 102 } else { 103 props->state = IB_PORT_DOWN; 104 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 105 } 106 107 ib_get_eth_speed(ibdev, port, &props->active_speed, 108 &props->active_width); 109 110 if (rdma_protocol_roce(ibdev, 1)) { 111 props->gid_tbl_len = 32; 112 props->ip_gids = true; 113 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 114 } else { 115 props->gid_tbl_len = 1; 116 } 117 props->qkey_viol_cntr = 0; 118 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 119 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 120 121 return 0; 122 } 123 124 /** 125 * irdma_disassociate_ucontext - Disassociate user context 126 * @context: ib user context 127 */ 128 static void irdma_disassociate_ucontext(struct ib_ucontext *context) 129 { 130 } 131 132 static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, 133 struct vm_area_struct *vma) 134 { 135 u64 pfn; 136 137 if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) 138 return -EINVAL; 139 140 vma->vm_private_data = ucontext; 141 pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + 142 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 143 144 return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, 145 pgprot_noncached(vma->vm_page_prot), NULL); 146 } 147 148 static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 149 { 150 struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); 151 152 kfree(entry); 153 } 154 155 static struct rdma_user_mmap_entry* 156 irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, 157 enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) 158 { 159 struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); 160 int ret; 161 162 if (!entry) 163 return NULL; 164 165 entry->bar_offset = bar_offset; 166 entry->mmap_flag = mmap_flag; 167 168 ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, 169 &entry->rdma_entry, PAGE_SIZE); 170 if (ret) { 171 kfree(entry); 172 return NULL; 173 } 174 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 175 176 return &entry->rdma_entry; 177 } 178 179 /** 180 * irdma_mmap - user memory map 181 * @context: context created during alloc 182 * @vma: kernel info for user memory map 183 */ 184 static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 185 { 186 struct rdma_user_mmap_entry *rdma_entry; 187 struct irdma_user_mmap_entry *entry; 188 struct irdma_ucontext *ucontext; 189 u64 pfn; 190 int ret; 191 192 ucontext = to_ucontext(context); 193 194 /* Legacy support for libi40iw with hard-coded mmap key */ 195 if (ucontext->legacy_mode) 196 return irdma_mmap_legacy(ucontext, vma); 197 198 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 199 if (!rdma_entry) { 200 ibdev_dbg(&ucontext->iwdev->ibdev, 201 "VERBS: pgoff[0x%lx] does not have valid entry\n", 202 vma->vm_pgoff); 203 return -EINVAL; 204 } 205 206 entry = to_irdma_mmap_entry(rdma_entry); 207 ibdev_dbg(&ucontext->iwdev->ibdev, 208 "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n", 209 entry->bar_offset, entry->mmap_flag); 210 211 pfn = (entry->bar_offset + 212 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 213 214 switch (entry->mmap_flag) { 215 case IRDMA_MMAP_IO_NC: 216 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 217 pgprot_noncached(vma->vm_page_prot), 218 rdma_entry); 219 break; 220 case IRDMA_MMAP_IO_WC: 221 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 222 pgprot_writecombine(vma->vm_page_prot), 223 rdma_entry); 224 break; 225 default: 226 ret = -EINVAL; 227 } 228 229 if (ret) 230 ibdev_dbg(&ucontext->iwdev->ibdev, 231 "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n", 232 entry->bar_offset, entry->mmap_flag, ret); 233 rdma_user_mmap_entry_put(rdma_entry); 234 235 return ret; 236 } 237 238 /** 239 * irdma_alloc_push_page - allocate a push page for qp 240 * @iwqp: qp pointer 241 */ 242 static void irdma_alloc_push_page(struct irdma_qp *iwqp) 243 { 244 struct irdma_cqp_request *cqp_request; 245 struct cqp_cmds_info *cqp_info; 246 struct irdma_device *iwdev = iwqp->iwdev; 247 struct irdma_sc_qp *qp = &iwqp->sc_qp; 248 int status; 249 250 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 251 if (!cqp_request) 252 return; 253 254 cqp_info = &cqp_request->info; 255 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 256 cqp_info->post_sq = 1; 257 cqp_info->in.u.manage_push_page.info.push_idx = 0; 258 cqp_info->in.u.manage_push_page.info.qs_handle = 259 qp->vsi->qos[qp->user_pri].qs_handle; 260 cqp_info->in.u.manage_push_page.info.free_page = 0; 261 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 262 cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; 263 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 264 265 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 266 if (!status && cqp_request->compl_info.op_ret_val < 267 iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { 268 qp->push_idx = cqp_request->compl_info.op_ret_val; 269 qp->push_offset = 0; 270 } 271 272 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 273 } 274 275 /** 276 * irdma_alloc_ucontext - Allocate the user context data structure 277 * @uctx: uverbs context pointer 278 * @udata: user data 279 * 280 * This keeps track of all objects associated with a particular 281 * user-mode client. 282 */ 283 static int irdma_alloc_ucontext(struct ib_ucontext *uctx, 284 struct ib_udata *udata) 285 { 286 #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8) 287 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) 288 struct ib_device *ibdev = uctx->device; 289 struct irdma_device *iwdev = to_iwdev(ibdev); 290 struct irdma_alloc_ucontext_req req = {}; 291 struct irdma_alloc_ucontext_resp uresp = {}; 292 struct irdma_ucontext *ucontext = to_ucontext(uctx); 293 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 294 295 if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || 296 udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) 297 return -EINVAL; 298 299 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 300 return -EINVAL; 301 302 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 303 goto ver_error; 304 305 ucontext->iwdev = iwdev; 306 ucontext->abi_ver = req.userspace_ver; 307 308 if (!(req.comp_mask & IRDMA_SUPPORT_WQE_FORMAT_V2) && 309 uk_attrs->hw_rev >= IRDMA_GEN_3) 310 return -EOPNOTSUPP; 311 312 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR) 313 ucontext->use_raw_attrs = true; 314 315 /* GEN_1 legacy support with libi40iw */ 316 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { 317 if (uk_attrs->hw_rev != IRDMA_GEN_1) 318 return -EOPNOTSUPP; 319 320 ucontext->legacy_mode = true; 321 uresp.max_qps = iwdev->rf->max_qp; 322 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 323 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 324 uresp.kernel_ver = req.userspace_ver; 325 if (ib_copy_to_udata(udata, &uresp, 326 min(sizeof(uresp), udata->outlen))) 327 return -EFAULT; 328 } else { 329 u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 330 331 ucontext->db_mmap_entry = 332 irdma_user_mmap_entry_insert(ucontext, bar_off, 333 IRDMA_MMAP_IO_NC, 334 &uresp.db_mmap_key); 335 if (!ucontext->db_mmap_entry) 336 return -ENOMEM; 337 338 uresp.kernel_ver = IRDMA_ABI_VER; 339 uresp.feature_flags = uk_attrs->feature_flags; 340 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 341 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 342 uresp.max_hw_inline = uk_attrs->max_hw_inline; 343 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 344 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 345 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 346 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 347 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 348 uresp.hw_rev = uk_attrs->hw_rev; 349 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; 350 uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size; 351 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE; 352 uresp.max_hw_srq_quanta = uk_attrs->max_hw_srq_quanta; 353 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA; 354 if (ib_copy_to_udata(udata, &uresp, 355 min(sizeof(uresp), udata->outlen))) { 356 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 357 return -EFAULT; 358 } 359 } 360 361 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 362 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 363 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 364 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 365 INIT_LIST_HEAD(&ucontext->srq_reg_mem_list); 366 spin_lock_init(&ucontext->srq_reg_mem_list_lock); 367 368 return 0; 369 370 ver_error: 371 ibdev_err(&iwdev->ibdev, 372 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 373 req.userspace_ver, IRDMA_ABI_VER); 374 return -EINVAL; 375 } 376 377 /** 378 * irdma_dealloc_ucontext - deallocate the user context data structure 379 * @context: user context created during alloc 380 */ 381 static void irdma_dealloc_ucontext(struct ib_ucontext *context) 382 { 383 struct irdma_ucontext *ucontext = to_ucontext(context); 384 385 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 386 } 387 388 /** 389 * irdma_alloc_pd - allocate protection domain 390 * @pd: PD pointer 391 * @udata: user data 392 */ 393 static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 394 { 395 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) 396 struct irdma_pd *iwpd = to_iwpd(pd); 397 struct irdma_device *iwdev = to_iwdev(pd->device); 398 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 399 struct irdma_pci_f *rf = iwdev->rf; 400 struct irdma_alloc_pd_resp uresp = {}; 401 struct irdma_sc_pd *sc_pd; 402 u32 pd_id = 0; 403 int err; 404 405 if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) 406 return -EINVAL; 407 408 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 409 &rf->next_pd); 410 if (err) 411 return err; 412 413 sc_pd = &iwpd->sc_pd; 414 if (udata) { 415 struct irdma_ucontext *ucontext = 416 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 417 ibucontext); 418 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 419 uresp.pd_id = pd_id; 420 if (ib_copy_to_udata(udata, &uresp, 421 min(sizeof(uresp), udata->outlen))) { 422 err = -EFAULT; 423 goto error; 424 } 425 } else { 426 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 427 } 428 429 return 0; 430 error: 431 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 432 433 return err; 434 } 435 436 /** 437 * irdma_dealloc_pd - deallocate pd 438 * @ibpd: ptr of pd to be deallocated 439 * @udata: user data 440 */ 441 static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 442 { 443 struct irdma_pd *iwpd = to_iwpd(ibpd); 444 struct irdma_device *iwdev = to_iwdev(ibpd->device); 445 446 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 447 448 return 0; 449 } 450 451 /** 452 * irdma_get_pbl - Retrieve pbl from a list given a virtual 453 * address 454 * @va: user virtual address 455 * @pbl_list: pbl list to search in (QP's or CQ's) 456 */ 457 static struct irdma_pbl *irdma_get_pbl(unsigned long va, 458 struct list_head *pbl_list) 459 { 460 struct irdma_pbl *iwpbl; 461 462 list_for_each_entry (iwpbl, pbl_list, list) { 463 if (iwpbl->user_base == va) { 464 list_del(&iwpbl->list); 465 iwpbl->on_list = false; 466 return iwpbl; 467 } 468 } 469 470 return NULL; 471 } 472 473 /** 474 * irdma_clean_cqes - clean cq entries for qp 475 * @iwqp: qp ptr (user or kernel) 476 * @iwcq: cq ptr 477 */ 478 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) 479 { 480 struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; 481 unsigned long flags; 482 483 spin_lock_irqsave(&iwcq->lock, flags); 484 irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); 485 spin_unlock_irqrestore(&iwcq->lock, flags); 486 } 487 488 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) 489 { 490 if (iwqp->push_db_mmap_entry) { 491 rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); 492 iwqp->push_db_mmap_entry = NULL; 493 } 494 if (iwqp->push_wqe_mmap_entry) { 495 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 496 iwqp->push_wqe_mmap_entry = NULL; 497 } 498 } 499 500 static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, 501 struct irdma_qp *iwqp, 502 u64 *push_wqe_mmap_key, 503 u64 *push_db_mmap_key) 504 { 505 struct irdma_device *iwdev = ucontext->iwdev; 506 u64 rsvd, bar_off; 507 508 rsvd = IRDMA_PF_BAR_RSVD; 509 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 510 /* skip over db page */ 511 bar_off += IRDMA_HW_PAGE_SIZE; 512 /* push wqe page */ 513 bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; 514 iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 515 bar_off, IRDMA_MMAP_IO_WC, 516 push_wqe_mmap_key); 517 if (!iwqp->push_wqe_mmap_entry) 518 return -ENOMEM; 519 520 /* push doorbell page */ 521 bar_off += IRDMA_HW_PAGE_SIZE; 522 iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 523 bar_off, IRDMA_MMAP_IO_NC, 524 push_db_mmap_key); 525 if (!iwqp->push_db_mmap_entry) { 526 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 527 return -ENOMEM; 528 } 529 530 return 0; 531 } 532 533 /** 534 * irdma_destroy_qp - destroy qp 535 * @ibqp: qp's ib pointer also to get to device's qp address 536 * @udata: user data 537 */ 538 static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 539 { 540 struct irdma_qp *iwqp = to_iwqp(ibqp); 541 struct irdma_device *iwdev = iwqp->iwdev; 542 543 iwqp->sc_qp.qp_uk.destroy_pending = true; 544 545 if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE) 546 irdma_modify_qp_to_err(&iwqp->sc_qp); 547 548 if (!iwqp->user_mode) 549 cancel_delayed_work_sync(&iwqp->dwork_flush); 550 551 if (!iwqp->user_mode) { 552 if (iwqp->iwscq) { 553 irdma_clean_cqes(iwqp, iwqp->iwscq); 554 if (iwqp->iwrcq != iwqp->iwscq) 555 irdma_clean_cqes(iwqp, iwqp->iwrcq); 556 } 557 } 558 559 irdma_qp_rem_ref(&iwqp->ibqp); 560 wait_for_completion(&iwqp->free_qp); 561 irdma_free_lsmm_rsrc(iwqp); 562 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); 563 564 irdma_remove_push_mmap_entries(iwqp); 565 566 if (iwqp->sc_qp.qp_uk.qp_id == 1) 567 iwdev->rf->hwqp1_rsvd = false; 568 irdma_free_qp_rsrc(iwqp); 569 570 return 0; 571 } 572 573 /** 574 * irdma_setup_virt_qp - setup for allocation of virtual qp 575 * @iwdev: irdma device 576 * @iwqp: qp ptr 577 * @init_info: initialize info to return 578 */ 579 static void irdma_setup_virt_qp(struct irdma_device *iwdev, 580 struct irdma_qp *iwqp, 581 struct irdma_qp_init_info *init_info) 582 { 583 struct irdma_pbl *iwpbl = iwqp->iwpbl; 584 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 585 586 iwqp->page = qpmr->sq_page; 587 init_info->shadow_area_pa = qpmr->shadow; 588 if (iwpbl->pbl_allocated) { 589 init_info->virtual_map = true; 590 init_info->sq_pa = qpmr->sq_pbl.idx; 591 /* Need to use contiguous buffer for RQ of QP 592 * in case it is associated with SRQ. 593 */ 594 init_info->rq_pa = init_info->qp_uk_init_info.srq_uk ? 595 qpmr->rq_pa : qpmr->rq_pbl.idx; 596 } else { 597 init_info->sq_pa = qpmr->sq_pbl.addr; 598 init_info->rq_pa = qpmr->rq_pbl.addr; 599 } 600 } 601 602 /** 603 * irdma_setup_umode_qp - setup sq and rq size in user mode qp 604 * @udata: udata 605 * @iwdev: iwarp device 606 * @iwqp: qp ptr (user or kernel) 607 * @info: initialize info to return 608 * @init_attr: Initial QP create attributes 609 */ 610 static int irdma_setup_umode_qp(struct ib_udata *udata, 611 struct irdma_device *iwdev, 612 struct irdma_qp *iwqp, 613 struct irdma_qp_init_info *info, 614 struct ib_qp_init_attr *init_attr) 615 { 616 struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, 617 struct irdma_ucontext, ibucontext); 618 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 619 struct irdma_create_qp_req req; 620 unsigned long flags; 621 int ret; 622 623 ret = ib_copy_from_udata(&req, udata, 624 min(sizeof(req), udata->inlen)); 625 if (ret) { 626 ibdev_dbg(&iwdev->ibdev, "VERBS: ib_copy_from_data fail\n"); 627 return ret; 628 } 629 630 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; 631 iwqp->user_mode = 1; 632 if (req.user_wqe_bufs) { 633 info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode; 634 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 635 iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, 636 &ucontext->qp_reg_mem_list); 637 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 638 639 if (!iwqp->iwpbl) { 640 ret = -ENODATA; 641 ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); 642 return ret; 643 } 644 } 645 646 if (!ucontext->use_raw_attrs) { 647 /** 648 * Maintain backward compat with older ABI which passes sq and 649 * rq depth in quanta in cap.max_send_wr and cap.max_recv_wr. 650 * There is no way to compute the correct value of 651 * iwqp->max_send_wr/max_recv_wr in the kernel. 652 */ 653 iwqp->max_send_wr = init_attr->cap.max_send_wr; 654 iwqp->max_recv_wr = init_attr->cap.max_recv_wr; 655 ukinfo->sq_size = init_attr->cap.max_send_wr; 656 ukinfo->rq_size = init_attr->cap.max_recv_wr; 657 irdma_uk_calc_shift_wq(ukinfo, &ukinfo->sq_shift, 658 &ukinfo->rq_shift); 659 } else { 660 ret = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth, 661 &ukinfo->sq_shift); 662 if (ret) 663 return ret; 664 665 ret = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth, 666 &ukinfo->rq_shift); 667 if (ret) 668 return ret; 669 670 iwqp->max_send_wr = 671 (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift; 672 iwqp->max_recv_wr = 673 (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift; 674 ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; 675 ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; 676 } 677 678 irdma_setup_virt_qp(iwdev, iwqp, info); 679 680 return 0; 681 } 682 683 /** 684 * irdma_setup_kmode_qp - setup initialization for kernel mode qp 685 * @iwdev: iwarp device 686 * @iwqp: qp ptr (user or kernel) 687 * @info: initialize info to return 688 * @init_attr: Initial QP create attributes 689 */ 690 static int irdma_setup_kmode_qp(struct irdma_device *iwdev, 691 struct irdma_qp *iwqp, 692 struct irdma_qp_init_info *info, 693 struct ib_qp_init_attr *init_attr) 694 { 695 struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; 696 u32 size; 697 int status; 698 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 699 700 status = irdma_uk_calc_depth_shift_sq(ukinfo, &ukinfo->sq_depth, 701 &ukinfo->sq_shift); 702 if (status) 703 return status; 704 705 status = irdma_uk_calc_depth_shift_rq(ukinfo, &ukinfo->rq_depth, 706 &ukinfo->rq_shift); 707 if (status) 708 return status; 709 710 iwqp->kqp.sq_wrid_mem = 711 kcalloc(ukinfo->sq_depth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL); 712 if (!iwqp->kqp.sq_wrid_mem) 713 return -ENOMEM; 714 715 iwqp->kqp.rq_wrid_mem = 716 kcalloc(ukinfo->rq_depth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL); 717 718 if (!iwqp->kqp.rq_wrid_mem) { 719 kfree(iwqp->kqp.sq_wrid_mem); 720 iwqp->kqp.sq_wrid_mem = NULL; 721 return -ENOMEM; 722 } 723 724 ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; 725 ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; 726 727 size = (ukinfo->sq_depth + ukinfo->rq_depth) * IRDMA_QP_WQE_MIN_SIZE; 728 size += (IRDMA_SHADOW_AREA_SIZE << 3); 729 730 mem->size = ALIGN(size, 256); 731 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 732 &mem->pa, GFP_KERNEL); 733 if (!mem->va) { 734 kfree(iwqp->kqp.sq_wrid_mem); 735 iwqp->kqp.sq_wrid_mem = NULL; 736 kfree(iwqp->kqp.rq_wrid_mem); 737 iwqp->kqp.rq_wrid_mem = NULL; 738 return -ENOMEM; 739 } 740 741 ukinfo->sq = mem->va; 742 info->sq_pa = mem->pa; 743 ukinfo->rq = &ukinfo->sq[ukinfo->sq_depth]; 744 info->rq_pa = info->sq_pa + (ukinfo->sq_depth * IRDMA_QP_WQE_MIN_SIZE); 745 ukinfo->shadow_area = ukinfo->rq[ukinfo->rq_depth].elem; 746 info->shadow_area_pa = 747 info->rq_pa + (ukinfo->rq_depth * IRDMA_QP_WQE_MIN_SIZE); 748 ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; 749 ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; 750 ukinfo->qp_id = info->qp_uk_init_info.qp_id; 751 752 iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift; 753 iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift; 754 init_attr->cap.max_send_wr = iwqp->max_send_wr; 755 init_attr->cap.max_recv_wr = iwqp->max_recv_wr; 756 757 return 0; 758 } 759 760 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) 761 { 762 struct irdma_pci_f *rf = iwqp->iwdev->rf; 763 struct irdma_cqp_request *cqp_request; 764 struct cqp_cmds_info *cqp_info; 765 struct irdma_create_qp_info *qp_info; 766 int status; 767 768 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 769 if (!cqp_request) 770 return -ENOMEM; 771 772 cqp_info = &cqp_request->info; 773 qp_info = &cqp_request->info.in.u.qp_create.info; 774 memset(qp_info, 0, sizeof(*qp_info)); 775 qp_info->mac_valid = true; 776 qp_info->cq_num_valid = true; 777 qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; 778 779 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 780 cqp_info->post_sq = 1; 781 cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; 782 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 783 status = irdma_handle_cqp_op(rf, cqp_request); 784 irdma_put_cqp_request(&rf->cqp, cqp_request); 785 786 return status; 787 } 788 789 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 790 struct irdma_qp_host_ctx_info *ctx_info) 791 { 792 struct irdma_device *iwdev = iwqp->iwdev; 793 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 794 struct irdma_roce_offload_info *roce_info; 795 struct irdma_udp_offload_info *udp_info; 796 797 udp_info = &iwqp->udp_info; 798 udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); 799 udp_info->cwnd = iwdev->roce_cwnd; 800 udp_info->rexmit_thresh = 2; 801 udp_info->rnr_nak_thresh = 2; 802 udp_info->src_port = 0xc000; 803 udp_info->dst_port = ROCE_V2_UDP_DPORT; 804 roce_info = &iwqp->roce_info; 805 ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr); 806 807 if (iwqp->ibqp.qp_type == IB_QPT_GSI && iwqp->ibqp.qp_num != 1) 808 roce_info->is_qp1 = true; 809 roce_info->rd_en = true; 810 roce_info->wr_rdresp_en = true; 811 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) 812 roce_info->bind_en = true; 813 roce_info->dcqcn_en = false; 814 roce_info->rtomin = 5; 815 816 roce_info->ack_credits = iwdev->roce_ackcreds; 817 roce_info->ird_size = dev->hw_attrs.max_hw_ird; 818 roce_info->ord_size = dev->hw_attrs.max_hw_ord; 819 820 if (!iwqp->user_mode) { 821 roce_info->priv_mode_en = true; 822 roce_info->fast_reg_en = true; 823 roce_info->udprivcq_en = true; 824 } 825 roce_info->roce_tver = 0; 826 827 ctx_info->roce_info = &iwqp->roce_info; 828 ctx_info->udp_info = &iwqp->udp_info; 829 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 830 } 831 832 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 833 struct irdma_qp_host_ctx_info *ctx_info) 834 { 835 struct irdma_device *iwdev = iwqp->iwdev; 836 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 837 struct irdma_iwarp_offload_info *iwarp_info; 838 839 iwarp_info = &iwqp->iwarp_info; 840 ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr); 841 iwarp_info->rd_en = true; 842 iwarp_info->wr_rdresp_en = true; 843 iwarp_info->ecn_en = true; 844 iwarp_info->rtomin = 5; 845 846 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 847 iwarp_info->ib_rd_en = true; 848 if (!iwqp->user_mode) { 849 iwarp_info->priv_mode_en = true; 850 iwarp_info->fast_reg_en = true; 851 } 852 iwarp_info->ddp_ver = 1; 853 iwarp_info->rdmap_ver = 1; 854 855 ctx_info->iwarp_info = &iwqp->iwarp_info; 856 ctx_info->iwarp_info_valid = true; 857 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 858 ctx_info->iwarp_info_valid = false; 859 } 860 861 static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, 862 struct irdma_device *iwdev) 863 { 864 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 865 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 866 867 if (init_attr->create_flags) 868 return -EOPNOTSUPP; 869 870 if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || 871 init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || 872 init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags || 873 init_attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta || 874 init_attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta) 875 return -EINVAL; 876 877 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 878 if (init_attr->qp_type != IB_QPT_RC && 879 init_attr->qp_type != IB_QPT_UD && 880 init_attr->qp_type != IB_QPT_GSI) 881 return -EOPNOTSUPP; 882 } else { 883 if (init_attr->qp_type != IB_QPT_RC) 884 return -EOPNOTSUPP; 885 } 886 887 return 0; 888 } 889 890 static void irdma_flush_worker(struct work_struct *work) 891 { 892 struct delayed_work *dwork = to_delayed_work(work); 893 struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush); 894 895 irdma_generate_flush_completions(iwqp); 896 } 897 898 static int irdma_setup_gsi_qp_rsrc(struct irdma_qp *iwqp, u32 *qp_num) 899 { 900 struct irdma_device *iwdev = iwqp->iwdev; 901 struct irdma_pci_f *rf = iwdev->rf; 902 unsigned long flags; 903 int ret; 904 905 if (rf->rdma_ver <= IRDMA_GEN_2) { 906 *qp_num = 1; 907 return 0; 908 } 909 910 spin_lock_irqsave(&rf->rsrc_lock, flags); 911 if (!rf->hwqp1_rsvd) { 912 *qp_num = 1; 913 rf->hwqp1_rsvd = true; 914 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 915 } else { 916 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 917 ret = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 918 qp_num, &rf->next_qp); 919 if (ret) 920 return ret; 921 } 922 923 ret = irdma_vchnl_req_add_vport(&rf->sc_dev, iwdev->vport_id, *qp_num, 924 (&iwdev->vsi)->qos); 925 if (ret) { 926 if (*qp_num != 1) { 927 irdma_free_rsrc(rf, rf->allocated_qps, *qp_num); 928 } else { 929 spin_lock_irqsave(&rf->rsrc_lock, flags); 930 rf->hwqp1_rsvd = false; 931 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 932 } 933 return ret; 934 } 935 936 return 0; 937 } 938 939 /** 940 * irdma_create_qp - create qp 941 * @ibqp: ptr of qp 942 * @init_attr: attributes for qp 943 * @udata: user data for create qp 944 */ 945 static int irdma_create_qp(struct ib_qp *ibqp, 946 struct ib_qp_init_attr *init_attr, 947 struct ib_udata *udata) 948 { 949 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx) 950 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd) 951 struct ib_pd *ibpd = ibqp->pd; 952 struct irdma_pd *iwpd = to_iwpd(ibpd); 953 struct irdma_device *iwdev = to_iwdev(ibpd->device); 954 struct irdma_pci_f *rf = iwdev->rf; 955 struct irdma_qp *iwqp = to_iwqp(ibqp); 956 struct irdma_create_qp_resp uresp = {}; 957 u32 qp_num = 0; 958 int err_code; 959 struct irdma_sc_qp *qp; 960 struct irdma_sc_dev *dev = &rf->sc_dev; 961 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 962 struct irdma_qp_init_info init_info = {}; 963 struct irdma_qp_host_ctx_info *ctx_info; 964 struct irdma_srq *iwsrq; 965 bool srq_valid = false; 966 u32 srq_id = 0; 967 968 if (init_attr->srq) { 969 iwsrq = to_iwsrq(init_attr->srq); 970 srq_valid = true; 971 srq_id = iwsrq->srq_num; 972 init_attr->cap.max_recv_sge = uk_attrs->max_hw_wq_frags; 973 init_attr->cap.max_recv_wr = 4; 974 init_info.qp_uk_init_info.srq_uk = &iwsrq->sc_srq.srq_uk; 975 } 976 977 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 978 if (err_code) 979 return err_code; 980 981 if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN || 982 udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN)) 983 return -EINVAL; 984 985 init_info.vsi = &iwdev->vsi; 986 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 987 init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr; 988 init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr; 989 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 990 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 991 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 992 993 qp = &iwqp->sc_qp; 994 qp->qp_uk.back_qp = iwqp; 995 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 996 997 iwqp->iwdev = iwdev; 998 iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, 999 256); 1000 iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, 1001 iwqp->q2_ctx_mem.size, 1002 &iwqp->q2_ctx_mem.pa, 1003 GFP_KERNEL); 1004 if (!iwqp->q2_ctx_mem.va) 1005 return -ENOMEM; 1006 1007 init_info.q2 = iwqp->q2_ctx_mem.va; 1008 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 1009 init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE); 1010 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 1011 1012 if (init_attr->qp_type == IB_QPT_GSI) { 1013 err_code = irdma_setup_gsi_qp_rsrc(iwqp, &qp_num); 1014 if (err_code) 1015 goto error; 1016 iwqp->ibqp.qp_num = 1; 1017 } else { 1018 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 1019 &qp_num, &rf->next_qp); 1020 if (err_code) 1021 goto error; 1022 iwqp->ibqp.qp_num = qp_num; 1023 } 1024 1025 iwqp->iwpd = iwpd; 1026 qp = &iwqp->sc_qp; 1027 iwqp->iwscq = to_iwcq(init_attr->send_cq); 1028 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 1029 iwqp->host_ctx.va = init_info.host_ctx; 1030 iwqp->host_ctx.pa = init_info.host_ctx_pa; 1031 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 1032 1033 init_info.pd = &iwpd->sc_pd; 1034 init_info.qp_uk_init_info.qp_id = qp_num; 1035 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 1036 init_info.qp_uk_init_info.first_sq_wq = 1; 1037 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 1038 init_waitqueue_head(&iwqp->waitq); 1039 init_waitqueue_head(&iwqp->mod_qp_waitq); 1040 1041 if (udata) { 1042 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 1043 err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, 1044 init_attr); 1045 } else { 1046 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); 1047 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 1048 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 1049 } 1050 1051 if (err_code) { 1052 ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n"); 1053 goto error; 1054 } 1055 1056 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 1057 if (init_attr->qp_type == IB_QPT_RC) { 1058 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 1059 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 1060 IRDMA_WRITE_WITH_IMM | 1061 IRDMA_ROCE; 1062 } else { 1063 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 1064 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 1065 IRDMA_ROCE; 1066 } 1067 } else { 1068 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 1069 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 1070 } 1071 1072 if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) 1073 init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE; 1074 1075 err_code = irdma_sc_qp_init(qp, &init_info); 1076 if (err_code) { 1077 ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n"); 1078 goto error; 1079 } 1080 1081 ctx_info = &iwqp->ctx_info; 1082 ctx_info->srq_valid = srq_valid; 1083 ctx_info->srq_id = srq_id; 1084 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1085 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1086 1087 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 1088 if (dev->ws_add(&iwdev->vsi, 0)) { 1089 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 1090 err_code = -EINVAL; 1091 goto error; 1092 } 1093 irdma_qp_add_qos(&iwqp->sc_qp); 1094 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 1095 } else { 1096 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 1097 } 1098 1099 err_code = irdma_cqp_create_qp_cmd(iwqp); 1100 if (err_code) 1101 goto error; 1102 1103 refcount_set(&iwqp->refcnt, 1); 1104 spin_lock_init(&iwqp->lock); 1105 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 1106 iwqp->sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1107 rf->qp_table[qp_num] = iwqp; 1108 1109 if (udata) { 1110 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 1111 if (udata->outlen < sizeof(uresp)) { 1112 uresp.lsmm = 1; 1113 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 1114 } else { 1115 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 1116 uresp.lsmm = 1; 1117 } 1118 uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size; 1119 uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size; 1120 uresp.qp_id = qp_num; 1121 uresp.qp_caps = qp->qp_uk.qp_caps; 1122 1123 err_code = ib_copy_to_udata(udata, &uresp, 1124 min(sizeof(uresp), udata->outlen)); 1125 if (err_code) { 1126 ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); 1127 irdma_destroy_qp(&iwqp->ibqp, udata); 1128 return err_code; 1129 } 1130 } 1131 1132 init_completion(&iwqp->free_qp); 1133 return 0; 1134 1135 error: 1136 irdma_free_qp_rsrc(iwqp); 1137 return err_code; 1138 } 1139 1140 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) 1141 { 1142 int acc_flags = 0; 1143 1144 if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { 1145 if (iwqp->roce_info.wr_rdresp_en) { 1146 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1147 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1148 } 1149 if (iwqp->roce_info.rd_en) 1150 acc_flags |= IB_ACCESS_REMOTE_READ; 1151 if (iwqp->roce_info.bind_en) 1152 acc_flags |= IB_ACCESS_MW_BIND; 1153 if (iwqp->ctx_info.remote_atomics_en) 1154 acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 1155 } else { 1156 if (iwqp->iwarp_info.wr_rdresp_en) { 1157 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1158 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1159 } 1160 if (iwqp->iwarp_info.rd_en) 1161 acc_flags |= IB_ACCESS_REMOTE_READ; 1162 if (iwqp->ctx_info.remote_atomics_en) 1163 acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 1164 } 1165 return acc_flags; 1166 } 1167 1168 /** 1169 * irdma_query_qp - query qp attributes 1170 * @ibqp: qp pointer 1171 * @attr: attributes pointer 1172 * @attr_mask: Not used 1173 * @init_attr: qp attributes to return 1174 */ 1175 static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1176 int attr_mask, struct ib_qp_init_attr *init_attr) 1177 { 1178 struct irdma_qp *iwqp = to_iwqp(ibqp); 1179 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1180 1181 memset(attr, 0, sizeof(*attr)); 1182 memset(init_attr, 0, sizeof(*init_attr)); 1183 1184 attr->qp_state = iwqp->ibqp_state; 1185 attr->cur_qp_state = iwqp->ibqp_state; 1186 attr->cap.max_send_wr = iwqp->max_send_wr; 1187 attr->cap.max_recv_wr = iwqp->max_recv_wr; 1188 attr->cap.max_inline_data = qp->qp_uk.max_inline_data; 1189 attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; 1190 attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; 1191 attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); 1192 attr->port_num = 1; 1193 if (rdma_protocol_roce(ibqp->device, 1)) { 1194 attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); 1195 attr->qkey = iwqp->roce_info.qkey; 1196 attr->rq_psn = iwqp->udp_info.epsn; 1197 attr->sq_psn = iwqp->udp_info.psn_nxt; 1198 attr->dest_qp_num = iwqp->roce_info.dest_qp; 1199 attr->pkey_index = iwqp->roce_info.p_key; 1200 attr->retry_cnt = iwqp->udp_info.rexmit_thresh; 1201 attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; 1202 attr->min_rnr_timer = iwqp->udp_info.min_rnr_timer; 1203 attr->max_rd_atomic = iwqp->roce_info.ord_size; 1204 attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; 1205 } 1206 1207 init_attr->event_handler = iwqp->ibqp.event_handler; 1208 init_attr->qp_context = iwqp->ibqp.qp_context; 1209 init_attr->send_cq = iwqp->ibqp.send_cq; 1210 init_attr->recv_cq = iwqp->ibqp.recv_cq; 1211 init_attr->srq = iwqp->ibqp.srq; 1212 init_attr->cap = attr->cap; 1213 1214 return 0; 1215 } 1216 1217 /** 1218 * irdma_query_pkey - Query partition key 1219 * @ibdev: device pointer from stack 1220 * @port: port number 1221 * @index: index of pkey 1222 * @pkey: pointer to store the pkey 1223 */ 1224 static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 1225 u16 *pkey) 1226 { 1227 if (index >= IRDMA_PKEY_TBL_SZ) 1228 return -EINVAL; 1229 1230 *pkey = IRDMA_DEFAULT_PKEY; 1231 return 0; 1232 } 1233 1234 static u8 irdma_roce_get_vlan_prio(const struct ib_gid_attr *attr, u8 prio) 1235 { 1236 struct net_device *ndev; 1237 1238 rcu_read_lock(); 1239 ndev = rcu_dereference(attr->ndev); 1240 if (!ndev) 1241 goto exit; 1242 if (is_vlan_dev(ndev)) { 1243 u16 vlan_qos = vlan_dev_get_egress_qos_mask(ndev, prio); 1244 1245 prio = (vlan_qos & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1246 } 1247 exit: 1248 rcu_read_unlock(); 1249 return prio; 1250 } 1251 1252 static int irdma_wait_for_suspend(struct irdma_qp *iwqp) 1253 { 1254 if (!wait_event_timeout(iwqp->iwdev->suspend_wq, 1255 !iwqp->suspend_pending, 1256 msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) { 1257 iwqp->suspend_pending = false; 1258 ibdev_warn(&iwqp->iwdev->ibdev, 1259 "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n", 1260 iwqp->ibqp.qp_num, iwqp->last_aeq); 1261 return -EBUSY; 1262 } 1263 1264 return 0; 1265 } 1266 1267 /** 1268 * irdma_modify_qp_roce - modify qp request 1269 * @ibqp: qp's pointer for modify 1270 * @attr: access attributes 1271 * @attr_mask: state mask 1272 * @udata: user data 1273 */ 1274 int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1275 int attr_mask, struct ib_udata *udata) 1276 { 1277 #define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) 1278 #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) 1279 struct irdma_pd *iwpd = to_iwpd(ibqp->pd); 1280 struct irdma_qp *iwqp = to_iwqp(ibqp); 1281 struct irdma_device *iwdev = iwqp->iwdev; 1282 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1283 struct irdma_qp_host_ctx_info *ctx_info; 1284 struct irdma_roce_offload_info *roce_info; 1285 struct irdma_udp_offload_info *udp_info; 1286 struct irdma_modify_qp_info info = {}; 1287 struct irdma_modify_qp_resp uresp = {}; 1288 struct irdma_modify_qp_req ureq = {}; 1289 unsigned long flags; 1290 u8 issue_modify_qp = 0; 1291 int ret = 0; 1292 1293 ctx_info = &iwqp->ctx_info; 1294 roce_info = &iwqp->roce_info; 1295 udp_info = &iwqp->udp_info; 1296 1297 if (udata) { 1298 /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ 1299 if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || 1300 (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) 1301 return -EINVAL; 1302 } 1303 1304 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1305 return -EOPNOTSUPP; 1306 1307 if (attr_mask & IB_QP_DEST_QPN) 1308 roce_info->dest_qp = attr->dest_qp_num; 1309 1310 if (attr_mask & IB_QP_PKEY_INDEX) { 1311 ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, 1312 &roce_info->p_key); 1313 if (ret) 1314 return ret; 1315 } 1316 1317 if (attr_mask & IB_QP_QKEY) 1318 roce_info->qkey = attr->qkey; 1319 1320 if (attr_mask & IB_QP_PATH_MTU) 1321 udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); 1322 1323 if (attr_mask & IB_QP_SQ_PSN) { 1324 udp_info->psn_nxt = attr->sq_psn; 1325 udp_info->lsn = 0xffff; 1326 udp_info->psn_una = attr->sq_psn; 1327 udp_info->psn_max = attr->sq_psn; 1328 } 1329 1330 if (attr_mask & IB_QP_RQ_PSN) 1331 udp_info->epsn = attr->rq_psn; 1332 1333 if (attr_mask & IB_QP_RNR_RETRY) 1334 udp_info->rnr_nak_thresh = attr->rnr_retry; 1335 1336 if (attr_mask & IB_QP_MIN_RNR_TIMER && 1337 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) 1338 udp_info->min_rnr_timer = attr->min_rnr_timer; 1339 1340 if (attr_mask & IB_QP_RETRY_CNT) 1341 udp_info->rexmit_thresh = attr->retry_cnt; 1342 1343 ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; 1344 1345 if (attr_mask & IB_QP_AV) { 1346 struct irdma_av *av = &iwqp->roce_ah.av; 1347 const struct ib_gid_attr *sgid_attr = 1348 attr->ah_attr.grh.sgid_attr; 1349 u16 vlan_id = VLAN_N_VID; 1350 u32 local_ip[4]; 1351 1352 memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); 1353 if (attr->ah_attr.ah_flags & IB_AH_GRH) { 1354 udp_info->ttl = attr->ah_attr.grh.hop_limit; 1355 udp_info->flow_label = attr->ah_attr.grh.flow_label; 1356 udp_info->tos = attr->ah_attr.grh.traffic_class; 1357 udp_info->src_port = 1358 rdma_get_udp_sport(udp_info->flow_label, 1359 ibqp->qp_num, 1360 roce_info->dest_qp); 1361 irdma_qp_rem_qos(&iwqp->sc_qp); 1362 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); 1363 if (iwqp->sc_qp.vsi->dscp_mode) 1364 ctx_info->user_pri = 1365 iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(udp_info->tos)]; 1366 else 1367 ctx_info->user_pri = rt_tos2priority(udp_info->tos); 1368 } 1369 ret = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, 1370 ctx_info->roce_info->mac_addr); 1371 if (ret) 1372 return ret; 1373 ctx_info->user_pri = irdma_roce_get_vlan_prio(sgid_attr, 1374 ctx_info->user_pri); 1375 if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) 1376 return -ENOMEM; 1377 iwqp->sc_qp.user_pri = ctx_info->user_pri; 1378 irdma_qp_add_qos(&iwqp->sc_qp); 1379 1380 if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 1381 vlan_id = 0; 1382 if (vlan_id < VLAN_N_VID) { 1383 udp_info->insert_vlan_tag = true; 1384 udp_info->vlan_tag = vlan_id | 1385 ctx_info->user_pri << VLAN_PRIO_SHIFT; 1386 } else { 1387 udp_info->insert_vlan_tag = false; 1388 } 1389 1390 av->attrs = attr->ah_attr; 1391 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid); 1392 rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); 1393 av->net_type = rdma_gid_attr_network_type(sgid_attr); 1394 if (av->net_type == RDMA_NETWORK_IPV6) { 1395 __be32 *daddr = 1396 av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1397 __be32 *saddr = 1398 av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1399 1400 irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); 1401 irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); 1402 1403 udp_info->ipv4 = false; 1404 irdma_copy_ip_ntohl(local_ip, daddr); 1405 1406 } else if (av->net_type == RDMA_NETWORK_IPV4) { 1407 __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; 1408 __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; 1409 1410 local_ip[0] = ntohl(daddr); 1411 1412 udp_info->ipv4 = true; 1413 udp_info->dest_ip_addr[0] = 0; 1414 udp_info->dest_ip_addr[1] = 0; 1415 udp_info->dest_ip_addr[2] = 0; 1416 udp_info->dest_ip_addr[3] = local_ip[0]; 1417 1418 udp_info->local_ipaddr[0] = 0; 1419 udp_info->local_ipaddr[1] = 0; 1420 udp_info->local_ipaddr[2] = 0; 1421 udp_info->local_ipaddr[3] = ntohl(saddr); 1422 } 1423 udp_info->arp_idx = 1424 irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4, 1425 attr->ah_attr.roce.dmac); 1426 } 1427 1428 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1429 if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { 1430 ibdev_err(&iwdev->ibdev, 1431 "rd_atomic = %d, above max_hw_ord=%d\n", 1432 attr->max_rd_atomic, 1433 dev->hw_attrs.max_hw_ord); 1434 return -EINVAL; 1435 } 1436 if (attr->max_rd_atomic) 1437 roce_info->ord_size = attr->max_rd_atomic; 1438 info.ord_valid = true; 1439 } 1440 1441 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1442 if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { 1443 ibdev_err(&iwdev->ibdev, 1444 "rd_atomic = %d, above max_hw_ird=%d\n", 1445 attr->max_dest_rd_atomic, 1446 dev->hw_attrs.max_hw_ird); 1447 return -EINVAL; 1448 } 1449 if (attr->max_dest_rd_atomic) 1450 roce_info->ird_size = attr->max_dest_rd_atomic; 1451 } 1452 1453 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1454 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1455 roce_info->wr_rdresp_en = true; 1456 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1457 roce_info->wr_rdresp_en = true; 1458 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1459 roce_info->rd_en = true; 1460 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 1461 if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) 1462 ctx_info->remote_atomics_en = true; 1463 } 1464 1465 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1466 1467 ibdev_dbg(&iwdev->ibdev, 1468 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", 1469 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1470 iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); 1471 1472 spin_lock_irqsave(&iwqp->lock, flags); 1473 if (attr_mask & IB_QP_STATE) { 1474 if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, 1475 iwqp->ibqp.qp_type, attr_mask)) { 1476 ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", 1477 iwqp->ibqp.qp_num, iwqp->ibqp_state, 1478 attr->qp_state); 1479 ret = -EINVAL; 1480 goto exit; 1481 } 1482 info.curr_iwarp_state = iwqp->iwarp_state; 1483 1484 switch (attr->qp_state) { 1485 case IB_QPS_INIT: 1486 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1487 ret = -EINVAL; 1488 goto exit; 1489 } 1490 1491 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1492 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1493 issue_modify_qp = 1; 1494 } 1495 break; 1496 case IB_QPS_RTR: 1497 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1498 ret = -EINVAL; 1499 goto exit; 1500 } 1501 info.arp_cache_idx_valid = true; 1502 info.cq_num_valid = true; 1503 info.next_iwarp_state = IRDMA_QP_STATE_RTR; 1504 issue_modify_qp = 1; 1505 break; 1506 case IB_QPS_RTS: 1507 if (iwqp->ibqp_state < IB_QPS_RTR || 1508 iwqp->ibqp_state == IB_QPS_ERR) { 1509 ret = -EINVAL; 1510 goto exit; 1511 } 1512 1513 info.arp_cache_idx_valid = true; 1514 info.cq_num_valid = true; 1515 info.ord_valid = true; 1516 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1517 issue_modify_qp = 1; 1518 if (iwdev->push_mode && udata && 1519 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1520 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1521 spin_unlock_irqrestore(&iwqp->lock, flags); 1522 irdma_alloc_push_page(iwqp); 1523 spin_lock_irqsave(&iwqp->lock, flags); 1524 } 1525 break; 1526 case IB_QPS_SQD: 1527 if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) 1528 goto exit; 1529 1530 if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { 1531 ret = -EINVAL; 1532 goto exit; 1533 } 1534 1535 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1536 issue_modify_qp = 1; 1537 iwqp->suspend_pending = true; 1538 break; 1539 case IB_QPS_SQE: 1540 case IB_QPS_ERR: 1541 case IB_QPS_RESET: 1542 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1543 spin_unlock_irqrestore(&iwqp->lock, flags); 1544 if (udata && udata->inlen) { 1545 if (ib_copy_from_udata(&ureq, udata, 1546 min(sizeof(ureq), udata->inlen))) 1547 return -EINVAL; 1548 1549 irdma_flush_wqes(iwqp, 1550 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1551 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1552 IRDMA_REFLUSH); 1553 } 1554 return 0; 1555 } 1556 1557 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1558 issue_modify_qp = 1; 1559 break; 1560 default: 1561 ret = -EINVAL; 1562 goto exit; 1563 } 1564 1565 iwqp->ibqp_state = attr->qp_state; 1566 } 1567 1568 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1569 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1570 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1571 spin_unlock_irqrestore(&iwqp->lock, flags); 1572 1573 if (attr_mask & IB_QP_STATE) { 1574 if (issue_modify_qp) { 1575 ctx_info->rem_endpoint_idx = udp_info->arp_idx; 1576 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1577 return -EINVAL; 1578 if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) { 1579 ret = irdma_wait_for_suspend(iwqp); 1580 if (ret) 1581 return ret; 1582 } 1583 spin_lock_irqsave(&iwqp->lock, flags); 1584 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1585 iwqp->iwarp_state = info.next_iwarp_state; 1586 iwqp->ibqp_state = attr->qp_state; 1587 } 1588 if (iwqp->ibqp_state > IB_QPS_RTS && 1589 !iwqp->flush_issued) { 1590 spin_unlock_irqrestore(&iwqp->lock, flags); 1591 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | 1592 IRDMA_FLUSH_RQ | 1593 IRDMA_FLUSH_WAIT); 1594 iwqp->flush_issued = 1; 1595 } else { 1596 spin_unlock_irqrestore(&iwqp->lock, flags); 1597 } 1598 } else { 1599 iwqp->ibqp_state = attr->qp_state; 1600 } 1601 if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1602 struct irdma_ucontext *ucontext; 1603 1604 ucontext = rdma_udata_to_drv_context(udata, 1605 struct irdma_ucontext, ibucontext); 1606 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1607 !iwqp->push_wqe_mmap_entry && 1608 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1609 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1610 uresp.push_valid = 1; 1611 uresp.push_offset = iwqp->sc_qp.push_offset; 1612 } 1613 ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1614 udata->outlen)); 1615 if (ret) { 1616 irdma_remove_push_mmap_entries(iwqp); 1617 ibdev_dbg(&iwdev->ibdev, 1618 "VERBS: copy_to_udata failed\n"); 1619 return ret; 1620 } 1621 } 1622 } 1623 1624 return 0; 1625 exit: 1626 spin_unlock_irqrestore(&iwqp->lock, flags); 1627 1628 return ret; 1629 } 1630 1631 /** 1632 * irdma_modify_qp - modify qp request 1633 * @ibqp: qp's pointer for modify 1634 * @attr: access attributes 1635 * @attr_mask: state mask 1636 * @udata: user data 1637 */ 1638 int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1639 struct ib_udata *udata) 1640 { 1641 #define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) 1642 #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) 1643 struct irdma_qp *iwqp = to_iwqp(ibqp); 1644 struct irdma_device *iwdev = iwqp->iwdev; 1645 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1646 struct irdma_qp_host_ctx_info *ctx_info; 1647 struct irdma_tcp_offload_info *tcp_info; 1648 struct irdma_iwarp_offload_info *offload_info; 1649 struct irdma_modify_qp_info info = {}; 1650 struct irdma_modify_qp_resp uresp = {}; 1651 struct irdma_modify_qp_req ureq = {}; 1652 u8 issue_modify_qp = 0; 1653 u8 dont_wait = 0; 1654 int err; 1655 unsigned long flags; 1656 1657 if (udata) { 1658 /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ 1659 if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || 1660 (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) 1661 return -EINVAL; 1662 } 1663 1664 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1665 return -EOPNOTSUPP; 1666 1667 ctx_info = &iwqp->ctx_info; 1668 offload_info = &iwqp->iwarp_info; 1669 tcp_info = &iwqp->tcp_info; 1670 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1671 ibdev_dbg(&iwdev->ibdev, 1672 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", 1673 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1674 iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, 1675 iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); 1676 1677 spin_lock_irqsave(&iwqp->lock, flags); 1678 if (attr_mask & IB_QP_STATE) { 1679 info.curr_iwarp_state = iwqp->iwarp_state; 1680 switch (attr->qp_state) { 1681 case IB_QPS_INIT: 1682 case IB_QPS_RTR: 1683 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1684 err = -EINVAL; 1685 goto exit; 1686 } 1687 1688 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1689 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1690 issue_modify_qp = 1; 1691 } 1692 if (iwdev->push_mode && udata && 1693 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1694 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1695 spin_unlock_irqrestore(&iwqp->lock, flags); 1696 irdma_alloc_push_page(iwqp); 1697 spin_lock_irqsave(&iwqp->lock, flags); 1698 } 1699 break; 1700 case IB_QPS_RTS: 1701 if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || 1702 !iwqp->cm_id) { 1703 err = -EINVAL; 1704 goto exit; 1705 } 1706 1707 issue_modify_qp = 1; 1708 iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 1709 iwqp->hte_added = 1; 1710 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1711 info.tcp_ctx_valid = true; 1712 info.ord_valid = true; 1713 info.arp_cache_idx_valid = true; 1714 info.cq_num_valid = true; 1715 break; 1716 case IB_QPS_SQD: 1717 if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { 1718 err = 0; 1719 goto exit; 1720 } 1721 1722 if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || 1723 iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { 1724 err = 0; 1725 goto exit; 1726 } 1727 1728 if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { 1729 err = -EINVAL; 1730 goto exit; 1731 } 1732 1733 info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; 1734 issue_modify_qp = 1; 1735 break; 1736 case IB_QPS_SQE: 1737 if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { 1738 err = -EINVAL; 1739 goto exit; 1740 } 1741 1742 info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; 1743 issue_modify_qp = 1; 1744 break; 1745 case IB_QPS_ERR: 1746 case IB_QPS_RESET: 1747 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1748 spin_unlock_irqrestore(&iwqp->lock, flags); 1749 if (udata && udata->inlen) { 1750 if (ib_copy_from_udata(&ureq, udata, 1751 min(sizeof(ureq), udata->inlen))) 1752 return -EINVAL; 1753 1754 irdma_flush_wqes(iwqp, 1755 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1756 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1757 IRDMA_REFLUSH); 1758 } 1759 return 0; 1760 } 1761 1762 if (iwqp->sc_qp.term_flags) { 1763 spin_unlock_irqrestore(&iwqp->lock, flags); 1764 irdma_terminate_del_timer(&iwqp->sc_qp); 1765 spin_lock_irqsave(&iwqp->lock, flags); 1766 } 1767 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1768 if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && 1769 iwdev->iw_status && 1770 iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) 1771 info.reset_tcp_conn = true; 1772 else 1773 dont_wait = 1; 1774 1775 issue_modify_qp = 1; 1776 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1777 break; 1778 default: 1779 err = -EINVAL; 1780 goto exit; 1781 } 1782 1783 iwqp->ibqp_state = attr->qp_state; 1784 } 1785 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1786 ctx_info->iwarp_info_valid = true; 1787 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1788 offload_info->wr_rdresp_en = true; 1789 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1790 offload_info->wr_rdresp_en = true; 1791 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1792 offload_info->rd_en = true; 1793 } 1794 1795 if (ctx_info->iwarp_info_valid) { 1796 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1797 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1798 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1799 } 1800 spin_unlock_irqrestore(&iwqp->lock, flags); 1801 1802 if (attr_mask & IB_QP_STATE) { 1803 if (issue_modify_qp) { 1804 ctx_info->rem_endpoint_idx = tcp_info->arp_idx; 1805 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1806 return -EINVAL; 1807 } 1808 1809 spin_lock_irqsave(&iwqp->lock, flags); 1810 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1811 iwqp->iwarp_state = info.next_iwarp_state; 1812 iwqp->ibqp_state = attr->qp_state; 1813 } 1814 spin_unlock_irqrestore(&iwqp->lock, flags); 1815 } 1816 1817 if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { 1818 if (dont_wait) { 1819 if (iwqp->hw_tcp_state) { 1820 spin_lock_irqsave(&iwqp->lock, flags); 1821 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1822 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1823 spin_unlock_irqrestore(&iwqp->lock, flags); 1824 } 1825 irdma_cm_disconn(iwqp); 1826 } else { 1827 int close_timer_started; 1828 1829 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 1830 1831 if (iwqp->cm_node) { 1832 refcount_inc(&iwqp->cm_node->refcnt); 1833 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1834 close_timer_started = atomic_inc_return(&iwqp->close_timer_started); 1835 if (iwqp->cm_id && close_timer_started == 1) 1836 irdma_schedule_cm_timer(iwqp->cm_node, 1837 (struct irdma_puda_buf *)iwqp, 1838 IRDMA_TIMER_TYPE_CLOSE, 1, 0); 1839 1840 irdma_rem_ref_cm_node(iwqp->cm_node); 1841 } else { 1842 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1843 } 1844 } 1845 } 1846 if (attr_mask & IB_QP_STATE && udata && udata->outlen && 1847 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1848 struct irdma_ucontext *ucontext; 1849 1850 ucontext = rdma_udata_to_drv_context(udata, 1851 struct irdma_ucontext, ibucontext); 1852 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1853 !iwqp->push_wqe_mmap_entry && 1854 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1855 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1856 uresp.push_valid = 1; 1857 uresp.push_offset = iwqp->sc_qp.push_offset; 1858 } 1859 1860 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1861 udata->outlen)); 1862 if (err) { 1863 irdma_remove_push_mmap_entries(iwqp); 1864 ibdev_dbg(&iwdev->ibdev, 1865 "VERBS: copy_to_udata failed\n"); 1866 return err; 1867 } 1868 } 1869 1870 return 0; 1871 exit: 1872 spin_unlock_irqrestore(&iwqp->lock, flags); 1873 1874 return err; 1875 } 1876 1877 /** 1878 * irdma_srq_free_rsrc - free up resources for srq 1879 * @rf: RDMA PCI function 1880 * @iwsrq: srq ptr 1881 */ 1882 static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq) 1883 { 1884 struct irdma_sc_srq *srq = &iwsrq->sc_srq; 1885 1886 if (!iwsrq->user_mode) { 1887 dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size, 1888 iwsrq->kmem.va, iwsrq->kmem.pa); 1889 iwsrq->kmem.va = NULL; 1890 } 1891 1892 irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id); 1893 } 1894 1895 /** 1896 * irdma_cq_free_rsrc - free up resources for cq 1897 * @rf: RDMA PCI function 1898 * @iwcq: cq ptr 1899 */ 1900 static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) 1901 { 1902 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1903 1904 if (!iwcq->user_mode) { 1905 dma_free_coherent(rf->sc_dev.hw->device, iwcq->kmem.size, 1906 iwcq->kmem.va, iwcq->kmem.pa); 1907 iwcq->kmem.va = NULL; 1908 dma_free_coherent(rf->sc_dev.hw->device, 1909 iwcq->kmem_shadow.size, 1910 iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); 1911 iwcq->kmem_shadow.va = NULL; 1912 } 1913 1914 irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); 1915 } 1916 1917 /** 1918 * irdma_free_cqbuf - worker to free a cq buffer 1919 * @work: provides access to the cq buffer to free 1920 */ 1921 static void irdma_free_cqbuf(struct work_struct *work) 1922 { 1923 struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); 1924 1925 dma_free_coherent(cq_buf->hw->device, cq_buf->kmem_buf.size, 1926 cq_buf->kmem_buf.va, cq_buf->kmem_buf.pa); 1927 cq_buf->kmem_buf.va = NULL; 1928 kfree(cq_buf); 1929 } 1930 1931 /** 1932 * irdma_process_resize_list - remove resized cq buffers from the resize_list 1933 * @iwcq: cq which owns the resize_list 1934 * @iwdev: irdma device 1935 * @lcqe_buf: the buffer where the last cqe is received 1936 */ 1937 static int irdma_process_resize_list(struct irdma_cq *iwcq, 1938 struct irdma_device *iwdev, 1939 struct irdma_cq_buf *lcqe_buf) 1940 { 1941 struct list_head *tmp_node, *list_node; 1942 struct irdma_cq_buf *cq_buf; 1943 int cnt = 0; 1944 1945 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 1946 cq_buf = list_entry(list_node, struct irdma_cq_buf, list); 1947 if (cq_buf == lcqe_buf) 1948 return cnt; 1949 1950 list_del(&cq_buf->list); 1951 queue_work(iwdev->cleanup_wq, &cq_buf->work); 1952 cnt++; 1953 } 1954 1955 return cnt; 1956 } 1957 1958 /** 1959 * irdma_destroy_srq - destroy srq 1960 * @ibsrq: srq pointer 1961 * @udata: user data 1962 */ 1963 static int irdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1964 { 1965 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 1966 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 1967 struct irdma_sc_srq *srq = &iwsrq->sc_srq; 1968 1969 irdma_srq_wq_destroy(iwdev->rf, srq); 1970 irdma_srq_free_rsrc(iwdev->rf, iwsrq); 1971 return 0; 1972 } 1973 1974 /** 1975 * irdma_destroy_cq - destroy cq 1976 * @ib_cq: cq pointer 1977 * @udata: user data 1978 */ 1979 static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1980 { 1981 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1982 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1983 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1984 struct irdma_sc_dev *dev = cq->dev; 1985 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1986 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1987 unsigned long flags; 1988 1989 spin_lock_irqsave(&iwcq->lock, flags); 1990 if (!list_empty(&iwcq->cmpl_generated)) 1991 irdma_remove_cmpls_list(iwcq); 1992 if (!list_empty(&iwcq->resize_list)) 1993 irdma_process_resize_list(iwcq, iwdev, NULL); 1994 spin_unlock_irqrestore(&iwcq->lock, flags); 1995 1996 irdma_cq_rem_ref(ib_cq); 1997 wait_for_completion(&iwcq->free_cq); 1998 1999 irdma_cq_wq_destroy(iwdev->rf, cq); 2000 2001 spin_lock_irqsave(&iwceq->ce_lock, flags); 2002 irdma_sc_cleanup_ceqes(cq, ceq); 2003 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 2004 irdma_cq_free_rsrc(iwdev->rf, iwcq); 2005 2006 return 0; 2007 } 2008 2009 /** 2010 * irdma_resize_cq - resize cq 2011 * @ibcq: cq to be resized 2012 * @entries: desired cq size 2013 * @udata: user data 2014 */ 2015 static int irdma_resize_cq(struct ib_cq *ibcq, int entries, 2016 struct ib_udata *udata) 2017 { 2018 #define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer) 2019 struct irdma_cq *iwcq = to_iwcq(ibcq); 2020 struct irdma_sc_dev *dev = iwcq->sc_cq.dev; 2021 struct irdma_cqp_request *cqp_request; 2022 struct cqp_cmds_info *cqp_info; 2023 struct irdma_modify_cq_info *m_info; 2024 struct irdma_modify_cq_info info = {}; 2025 struct irdma_dma_mem kmem_buf; 2026 struct irdma_cq_mr *cqmr_buf; 2027 struct irdma_pbl *iwpbl_buf; 2028 struct irdma_device *iwdev; 2029 struct irdma_pci_f *rf; 2030 struct irdma_cq_buf *cq_buf = NULL; 2031 unsigned long flags; 2032 int ret; 2033 2034 iwdev = to_iwdev(ibcq->device); 2035 rf = iwdev->rf; 2036 2037 if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2038 IRDMA_FEATURE_CQ_RESIZE)) 2039 return -EOPNOTSUPP; 2040 2041 if (udata && udata->inlen < IRDMA_RESIZE_CQ_MIN_REQ_LEN) 2042 return -EINVAL; 2043 2044 if (entries > rf->max_cqe) 2045 return -EINVAL; 2046 2047 if (!iwcq->user_mode) { 2048 entries++; 2049 2050 if (!iwcq->sc_cq.cq_uk.avoid_mem_cflct && 2051 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2052 entries *= 2; 2053 2054 if (entries & 1) 2055 entries += 1; /* cq size must be an even number */ 2056 } 2057 2058 info.cq_size = max(entries, 4); 2059 2060 if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) 2061 return 0; 2062 2063 if (udata) { 2064 struct irdma_resize_cq_req req = {}; 2065 struct irdma_ucontext *ucontext = 2066 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2067 ibucontext); 2068 2069 /* CQ resize not supported with legacy GEN_1 libi40iw */ 2070 if (ucontext->legacy_mode) 2071 return -EOPNOTSUPP; 2072 2073 if (ib_copy_from_udata(&req, udata, 2074 min(sizeof(req), udata->inlen))) 2075 return -EINVAL; 2076 2077 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2078 iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, 2079 &ucontext->cq_reg_mem_list); 2080 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2081 2082 if (!iwpbl_buf) 2083 return -ENOMEM; 2084 2085 cqmr_buf = &iwpbl_buf->cq_mr; 2086 if (iwpbl_buf->pbl_allocated) { 2087 info.virtual_map = true; 2088 info.pbl_chunk_size = 1; 2089 info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; 2090 } else { 2091 info.cq_pa = cqmr_buf->cq_pbl.addr; 2092 } 2093 } else { 2094 /* Kmode CQ resize */ 2095 int rsize; 2096 2097 rsize = info.cq_size * sizeof(struct irdma_cqe); 2098 kmem_buf.size = ALIGN(round_up(rsize, 256), 256); 2099 kmem_buf.va = dma_alloc_coherent(dev->hw->device, 2100 kmem_buf.size, &kmem_buf.pa, 2101 GFP_KERNEL); 2102 if (!kmem_buf.va) 2103 return -ENOMEM; 2104 2105 info.cq_base = kmem_buf.va; 2106 info.cq_pa = kmem_buf.pa; 2107 cq_buf = kzalloc(sizeof(*cq_buf), GFP_KERNEL); 2108 if (!cq_buf) { 2109 ret = -ENOMEM; 2110 goto error; 2111 } 2112 } 2113 2114 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2115 if (!cqp_request) { 2116 ret = -ENOMEM; 2117 goto error; 2118 } 2119 2120 info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; 2121 info.cq_resize = true; 2122 2123 cqp_info = &cqp_request->info; 2124 m_info = &cqp_info->in.u.cq_modify.info; 2125 memcpy(m_info, &info, sizeof(*m_info)); 2126 2127 cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; 2128 cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; 2129 cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; 2130 cqp_info->post_sq = 1; 2131 ret = irdma_handle_cqp_op(rf, cqp_request); 2132 irdma_put_cqp_request(&rf->cqp, cqp_request); 2133 if (ret) 2134 goto error; 2135 2136 spin_lock_irqsave(&iwcq->lock, flags); 2137 if (cq_buf) { 2138 cq_buf->kmem_buf = iwcq->kmem; 2139 cq_buf->hw = dev->hw; 2140 memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); 2141 INIT_WORK(&cq_buf->work, irdma_free_cqbuf); 2142 list_add_tail(&cq_buf->list, &iwcq->resize_list); 2143 iwcq->kmem = kmem_buf; 2144 } 2145 2146 irdma_sc_cq_resize(&iwcq->sc_cq, &info); 2147 ibcq->cqe = info.cq_size - 1; 2148 spin_unlock_irqrestore(&iwcq->lock, flags); 2149 2150 return 0; 2151 error: 2152 if (!udata) { 2153 dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, 2154 kmem_buf.pa); 2155 kmem_buf.va = NULL; 2156 } 2157 kfree(cq_buf); 2158 2159 return ret; 2160 } 2161 2162 /** 2163 * irdma_srq_event - event notification for srq limit 2164 * @srq: shared srq struct 2165 */ 2166 void irdma_srq_event(struct irdma_sc_srq *srq) 2167 { 2168 struct irdma_srq *iwsrq = container_of(srq, struct irdma_srq, sc_srq); 2169 struct ib_srq *ibsrq = &iwsrq->ibsrq; 2170 struct ib_event event; 2171 2172 srq->srq_limit = 0; 2173 2174 if (!ibsrq->event_handler) 2175 return; 2176 2177 event.device = ibsrq->device; 2178 event.element.port_num = 1; 2179 event.element.srq = ibsrq; 2180 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 2181 ibsrq->event_handler(&event, ibsrq->srq_context); 2182 } 2183 2184 /** 2185 * irdma_modify_srq - modify srq request 2186 * @ibsrq: srq's pointer for modify 2187 * @attr: access attributes 2188 * @attr_mask: state mask 2189 * @udata: user data 2190 */ 2191 static int irdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 2192 enum ib_srq_attr_mask attr_mask, 2193 struct ib_udata *udata) 2194 { 2195 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 2196 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2197 struct irdma_cqp_request *cqp_request; 2198 struct irdma_pci_f *rf = iwdev->rf; 2199 struct irdma_modify_srq_info *info; 2200 struct cqp_cmds_info *cqp_info; 2201 int status; 2202 2203 if (attr_mask & IB_SRQ_MAX_WR) 2204 return -EINVAL; 2205 2206 if (!(attr_mask & IB_SRQ_LIMIT)) 2207 return 0; 2208 2209 if (attr->srq_limit > iwsrq->sc_srq.srq_uk.srq_size) 2210 return -EINVAL; 2211 2212 /* Execute this cqp op synchronously, so we can update srq_limit 2213 * upon successful completion. 2214 */ 2215 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2216 if (!cqp_request) 2217 return -ENOMEM; 2218 2219 cqp_info = &cqp_request->info; 2220 info = &cqp_info->in.u.srq_modify.info; 2221 info->srq_limit = attr->srq_limit; 2222 if (info->srq_limit > 0xFFF) 2223 info->srq_limit = 0xFFF; 2224 info->arm_limit_event = 1; 2225 2226 cqp_info->cqp_cmd = IRDMA_OP_SRQ_MODIFY; 2227 cqp_info->post_sq = 1; 2228 cqp_info->in.u.srq_modify.srq = &iwsrq->sc_srq; 2229 cqp_info->in.u.srq_modify.scratch = (uintptr_t)cqp_request; 2230 status = irdma_handle_cqp_op(rf, cqp_request); 2231 irdma_put_cqp_request(&rf->cqp, cqp_request); 2232 if (status) 2233 return status; 2234 2235 iwsrq->sc_srq.srq_limit = info->srq_limit; 2236 2237 return 0; 2238 } 2239 2240 static int irdma_setup_umode_srq(struct irdma_device *iwdev, 2241 struct irdma_srq *iwsrq, 2242 struct irdma_srq_init_info *info, 2243 struct ib_udata *udata) 2244 { 2245 #define IRDMA_CREATE_SRQ_MIN_REQ_LEN \ 2246 offsetofend(struct irdma_create_srq_req, user_shadow_area) 2247 struct irdma_create_srq_req req = {}; 2248 struct irdma_ucontext *ucontext; 2249 struct irdma_srq_mr *srqmr; 2250 struct irdma_pbl *iwpbl; 2251 unsigned long flags; 2252 2253 iwsrq->user_mode = true; 2254 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2255 ibucontext); 2256 2257 if (udata->inlen < IRDMA_CREATE_SRQ_MIN_REQ_LEN) 2258 return -EINVAL; 2259 2260 if (ib_copy_from_udata(&req, udata, 2261 min(sizeof(req), udata->inlen))) 2262 return -EFAULT; 2263 2264 spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); 2265 iwpbl = irdma_get_pbl((unsigned long)req.user_srq_buf, 2266 &ucontext->srq_reg_mem_list); 2267 spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); 2268 if (!iwpbl) 2269 return -EPROTO; 2270 2271 iwsrq->iwpbl = iwpbl; 2272 srqmr = &iwpbl->srq_mr; 2273 2274 if (iwpbl->pbl_allocated) { 2275 info->virtual_map = true; 2276 info->pbl_chunk_size = 1; 2277 info->first_pm_pbl_idx = srqmr->srq_pbl.idx; 2278 info->leaf_pbl_size = 1; 2279 } else { 2280 info->srq_pa = srqmr->srq_pbl.addr; 2281 } 2282 info->shadow_area_pa = srqmr->shadow; 2283 2284 return 0; 2285 } 2286 2287 static int irdma_setup_kmode_srq(struct irdma_device *iwdev, 2288 struct irdma_srq *iwsrq, 2289 struct irdma_srq_init_info *info, u32 depth, 2290 u8 shift) 2291 { 2292 struct irdma_srq_uk_init_info *ukinfo = &info->srq_uk_init_info; 2293 struct irdma_dma_mem *mem = &iwsrq->kmem; 2294 u32 size, ring_size; 2295 2296 ring_size = depth * IRDMA_QP_WQE_MIN_SIZE; 2297 size = ring_size + (IRDMA_SHADOW_AREA_SIZE << 3); 2298 2299 mem->size = ALIGN(size, 256); 2300 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 2301 &mem->pa, GFP_KERNEL); 2302 if (!mem->va) 2303 return -ENOMEM; 2304 2305 ukinfo->srq = mem->va; 2306 ukinfo->srq_size = depth >> shift; 2307 ukinfo->shadow_area = mem->va + ring_size; 2308 2309 info->shadow_area_pa = info->srq_pa + ring_size; 2310 info->srq_pa = mem->pa; 2311 2312 return 0; 2313 } 2314 2315 /** 2316 * irdma_create_srq - create srq 2317 * @ibsrq: ib's srq pointer 2318 * @initattrs: attributes for srq 2319 * @udata: user data for create srq 2320 */ 2321 static int irdma_create_srq(struct ib_srq *ibsrq, 2322 struct ib_srq_init_attr *initattrs, 2323 struct ib_udata *udata) 2324 { 2325 struct irdma_device *iwdev = to_iwdev(ibsrq->device); 2326 struct ib_srq_attr *attr = &initattrs->attr; 2327 struct irdma_pd *iwpd = to_iwpd(ibsrq->pd); 2328 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2329 struct irdma_srq_uk_init_info *ukinfo; 2330 struct irdma_cqp_request *cqp_request; 2331 struct irdma_srq_init_info info = {}; 2332 struct irdma_pci_f *rf = iwdev->rf; 2333 struct irdma_uk_attrs *uk_attrs; 2334 struct cqp_cmds_info *cqp_info; 2335 int err_code = 0; 2336 u32 depth; 2337 u8 shift; 2338 2339 uk_attrs = &rf->sc_dev.hw_attrs.uk_attrs; 2340 ukinfo = &info.srq_uk_init_info; 2341 2342 if (initattrs->srq_type != IB_SRQT_BASIC) 2343 return -EOPNOTSUPP; 2344 2345 if (!(uk_attrs->feature_flags & IRDMA_FEATURE_SRQ) || 2346 attr->max_sge > uk_attrs->max_hw_wq_frags) 2347 return -EINVAL; 2348 2349 refcount_set(&iwsrq->refcnt, 1); 2350 spin_lock_init(&iwsrq->lock); 2351 err_code = irdma_alloc_rsrc(rf, rf->allocated_srqs, rf->max_srq, 2352 &iwsrq->srq_num, &rf->next_srq); 2353 if (err_code) 2354 return err_code; 2355 2356 ukinfo->max_srq_frag_cnt = attr->max_sge; 2357 ukinfo->uk_attrs = uk_attrs; 2358 ukinfo->srq_id = iwsrq->srq_num; 2359 2360 irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_srq_frag_cnt, 0, 2361 &shift); 2362 2363 err_code = irdma_get_srqdepth(ukinfo->uk_attrs, attr->max_wr, 2364 shift, &depth); 2365 if (err_code) 2366 return err_code; 2367 2368 /* Actual SRQ size in WRs for ring and HW */ 2369 ukinfo->srq_size = depth >> shift; 2370 2371 /* Max postable WRs to SRQ */ 2372 iwsrq->max_wr = (depth - IRDMA_RQ_RSVD) >> shift; 2373 attr->max_wr = iwsrq->max_wr; 2374 2375 if (udata) 2376 err_code = irdma_setup_umode_srq(iwdev, iwsrq, &info, udata); 2377 else 2378 err_code = irdma_setup_kmode_srq(iwdev, iwsrq, &info, depth, 2379 shift); 2380 2381 if (err_code) 2382 goto free_rsrc; 2383 2384 info.vsi = &iwdev->vsi; 2385 info.pd = &iwpd->sc_pd; 2386 2387 err_code = irdma_sc_srq_init(&iwsrq->sc_srq, &info); 2388 if (err_code) 2389 goto free_dmem; 2390 2391 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2392 if (!cqp_request) { 2393 err_code = -ENOMEM; 2394 goto free_dmem; 2395 } 2396 2397 cqp_info = &cqp_request->info; 2398 cqp_info->cqp_cmd = IRDMA_OP_SRQ_CREATE; 2399 cqp_info->post_sq = 1; 2400 cqp_info->in.u.srq_create.srq = &iwsrq->sc_srq; 2401 cqp_info->in.u.srq_create.scratch = (uintptr_t)cqp_request; 2402 err_code = irdma_handle_cqp_op(rf, cqp_request); 2403 irdma_put_cqp_request(&rf->cqp, cqp_request); 2404 if (err_code) 2405 goto free_dmem; 2406 2407 if (udata) { 2408 struct irdma_create_srq_resp resp = {}; 2409 2410 resp.srq_id = iwsrq->srq_num; 2411 resp.srq_size = ukinfo->srq_size; 2412 if (ib_copy_to_udata(udata, &resp, 2413 min(sizeof(resp), udata->outlen))) { 2414 err_code = -EPROTO; 2415 goto srq_destroy; 2416 } 2417 } 2418 2419 return 0; 2420 2421 srq_destroy: 2422 irdma_srq_wq_destroy(rf, &iwsrq->sc_srq); 2423 2424 free_dmem: 2425 if (!iwsrq->user_mode) 2426 dma_free_coherent(rf->hw.device, iwsrq->kmem.size, 2427 iwsrq->kmem.va, iwsrq->kmem.pa); 2428 free_rsrc: 2429 irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num); 2430 return err_code; 2431 } 2432 2433 /** 2434 * irdma_query_srq - get SRQ attributes 2435 * @ibsrq: the SRQ to query 2436 * @attr: the attributes of the SRQ 2437 */ 2438 static int irdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 2439 { 2440 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 2441 2442 attr->max_wr = iwsrq->max_wr; 2443 attr->max_sge = iwsrq->sc_srq.srq_uk.max_srq_frag_cnt; 2444 attr->srq_limit = iwsrq->sc_srq.srq_limit; 2445 2446 return 0; 2447 } 2448 2449 static inline int cq_validate_flags(u32 flags, u8 hw_rev) 2450 { 2451 /* GEN1/2 does not support CQ create flags */ 2452 if (hw_rev <= IRDMA_GEN_2) 2453 return flags ? -EOPNOTSUPP : 0; 2454 2455 return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; 2456 } 2457 2458 /** 2459 * irdma_create_cq - create cq 2460 * @ibcq: CQ allocated 2461 * @attr: attributes for cq 2462 * @attrs: uverbs attribute bundle 2463 */ 2464 static int irdma_create_cq(struct ib_cq *ibcq, 2465 const struct ib_cq_init_attr *attr, 2466 struct uverbs_attr_bundle *attrs) 2467 { 2468 #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf) 2469 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) 2470 struct ib_udata *udata = &attrs->driver_udata; 2471 struct ib_device *ibdev = ibcq->device; 2472 struct irdma_device *iwdev = to_iwdev(ibdev); 2473 struct irdma_pci_f *rf = iwdev->rf; 2474 struct irdma_cq *iwcq = to_iwcq(ibcq); 2475 u32 cq_num = 0; 2476 struct irdma_sc_cq *cq; 2477 struct irdma_sc_dev *dev = &rf->sc_dev; 2478 struct irdma_cq_init_info info = {}; 2479 struct irdma_cqp_request *cqp_request; 2480 struct cqp_cmds_info *cqp_info; 2481 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 2482 unsigned long flags; 2483 int err_code; 2484 int entries = attr->cqe; 2485 bool cqe_64byte_ena; 2486 2487 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 2488 if (err_code) 2489 return err_code; 2490 2491 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || 2492 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) 2493 return -EINVAL; 2494 2495 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 2496 &rf->next_cq); 2497 if (err_code) 2498 return err_code; 2499 2500 cq = &iwcq->sc_cq; 2501 cq->back_cq = iwcq; 2502 refcount_set(&iwcq->refcnt, 1); 2503 spin_lock_init(&iwcq->lock); 2504 INIT_LIST_HEAD(&iwcq->resize_list); 2505 INIT_LIST_HEAD(&iwcq->cmpl_generated); 2506 info.dev = dev; 2507 ukinfo->cq_size = max(entries, 4); 2508 ukinfo->cq_id = cq_num; 2509 cqe_64byte_ena = dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? 2510 true : false; 2511 ukinfo->avoid_mem_cflct = cqe_64byte_ena; 2512 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 2513 if (attr->comp_vector < rf->ceqs_count) 2514 info.ceq_id = attr->comp_vector; 2515 info.ceq_id_valid = true; 2516 info.ceqe_mask = 1; 2517 info.type = IRDMA_CQ_TYPE_IWARP; 2518 info.vsi = &iwdev->vsi; 2519 2520 if (udata) { 2521 struct irdma_ucontext *ucontext; 2522 struct irdma_create_cq_req req = {}; 2523 struct irdma_cq_mr *cqmr; 2524 struct irdma_pbl *iwpbl; 2525 struct irdma_pbl *iwpbl_shadow; 2526 struct irdma_cq_mr *cqmr_shadow; 2527 2528 iwcq->user_mode = true; 2529 ucontext = 2530 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2531 ibucontext); 2532 if (ib_copy_from_udata(&req, udata, 2533 min(sizeof(req), udata->inlen))) { 2534 err_code = -EFAULT; 2535 goto cq_free_rsrc; 2536 } 2537 2538 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2539 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 2540 &ucontext->cq_reg_mem_list); 2541 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2542 if (!iwpbl) { 2543 err_code = -EPROTO; 2544 goto cq_free_rsrc; 2545 } 2546 2547 iwcq->iwpbl = iwpbl; 2548 iwcq->cq_mem_size = 0; 2549 cqmr = &iwpbl->cq_mr; 2550 2551 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2552 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { 2553 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2554 iwpbl_shadow = irdma_get_pbl( 2555 (unsigned long)req.user_shadow_area, 2556 &ucontext->cq_reg_mem_list); 2557 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2558 2559 if (!iwpbl_shadow) { 2560 err_code = -EPROTO; 2561 goto cq_free_rsrc; 2562 } 2563 iwcq->iwpbl_shadow = iwpbl_shadow; 2564 cqmr_shadow = &iwpbl_shadow->cq_mr; 2565 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 2566 cqmr->split = true; 2567 } else { 2568 info.shadow_area_pa = cqmr->shadow; 2569 } 2570 if (iwpbl->pbl_allocated) { 2571 info.virtual_map = true; 2572 info.pbl_chunk_size = 1; 2573 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 2574 } else { 2575 info.cq_base_pa = cqmr->cq_pbl.addr; 2576 } 2577 } else { 2578 /* Kmode allocations */ 2579 int rsize; 2580 2581 if (entries < 1 || entries > rf->max_cqe) { 2582 err_code = -EINVAL; 2583 goto cq_free_rsrc; 2584 } 2585 2586 entries++; 2587 if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2588 entries *= 2; 2589 2590 if (entries & 1) 2591 entries += 1; /* cq size must be an even number */ 2592 2593 ukinfo->cq_size = entries; 2594 2595 if (cqe_64byte_ena) 2596 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe); 2597 else 2598 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 2599 iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256); 2600 iwcq->kmem.va = dma_alloc_coherent(dev->hw->device, 2601 iwcq->kmem.size, 2602 &iwcq->kmem.pa, GFP_KERNEL); 2603 if (!iwcq->kmem.va) { 2604 err_code = -ENOMEM; 2605 goto cq_free_rsrc; 2606 } 2607 2608 iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, 2609 64); 2610 iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device, 2611 iwcq->kmem_shadow.size, 2612 &iwcq->kmem_shadow.pa, 2613 GFP_KERNEL); 2614 if (!iwcq->kmem_shadow.va) { 2615 err_code = -ENOMEM; 2616 goto cq_free_rsrc; 2617 } 2618 info.shadow_area_pa = iwcq->kmem_shadow.pa; 2619 ukinfo->shadow_area = iwcq->kmem_shadow.va; 2620 ukinfo->cq_base = iwcq->kmem.va; 2621 info.cq_base_pa = iwcq->kmem.pa; 2622 } 2623 2624 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 2625 (u32)IRDMA_MAX_CQ_READ_THRESH); 2626 2627 if (irdma_sc_cq_init(cq, &info)) { 2628 ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n"); 2629 err_code = -EPROTO; 2630 goto cq_free_rsrc; 2631 } 2632 2633 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2634 if (!cqp_request) { 2635 err_code = -ENOMEM; 2636 goto cq_free_rsrc; 2637 } 2638 2639 cqp_info = &cqp_request->info; 2640 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 2641 cqp_info->post_sq = 1; 2642 cqp_info->in.u.cq_create.cq = cq; 2643 cqp_info->in.u.cq_create.check_overflow = true; 2644 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 2645 err_code = irdma_handle_cqp_op(rf, cqp_request); 2646 irdma_put_cqp_request(&rf->cqp, cqp_request); 2647 if (err_code) 2648 goto cq_free_rsrc; 2649 2650 if (udata) { 2651 struct irdma_create_cq_resp resp = {}; 2652 2653 resp.cq_id = info.cq_uk_init_info.cq_id; 2654 resp.cq_size = info.cq_uk_init_info.cq_size; 2655 if (ib_copy_to_udata(udata, &resp, 2656 min(sizeof(resp), udata->outlen))) { 2657 ibdev_dbg(&iwdev->ibdev, 2658 "VERBS: copy to user data\n"); 2659 err_code = -EPROTO; 2660 goto cq_destroy; 2661 } 2662 } 2663 rf->cq_table[cq_num] = iwcq; 2664 init_completion(&iwcq->free_cq); 2665 2666 return 0; 2667 cq_destroy: 2668 irdma_cq_wq_destroy(rf, cq); 2669 cq_free_rsrc: 2670 irdma_cq_free_rsrc(rf, iwcq); 2671 2672 return err_code; 2673 } 2674 2675 /** 2676 * irdma_get_mr_access - get hw MR access permissions from IB access flags 2677 * @access: IB access flags 2678 * @hw_rev: Hardware version 2679 */ 2680 static inline u16 irdma_get_mr_access(int access, u8 hw_rev) 2681 { 2682 u16 hw_access = 0; 2683 2684 hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? 2685 IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; 2686 hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? 2687 IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; 2688 hw_access |= (access & IB_ACCESS_REMOTE_READ) ? 2689 IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; 2690 if (hw_rev >= IRDMA_GEN_3) { 2691 hw_access |= (access & IB_ACCESS_MW_BIND) ? 2692 IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; 2693 } 2694 hw_access |= (access & IB_ZERO_BASED) ? 2695 IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; 2696 hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; 2697 2698 return hw_access; 2699 } 2700 2701 /** 2702 * irdma_free_stag - free stag resource 2703 * @iwdev: irdma device 2704 * @stag: stag to free 2705 */ 2706 static void irdma_free_stag(struct irdma_device *iwdev, u32 stag) 2707 { 2708 u32 stag_idx; 2709 2710 stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; 2711 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); 2712 } 2713 2714 /** 2715 * irdma_create_stag - create random stag 2716 * @iwdev: irdma device 2717 */ 2718 static u32 irdma_create_stag(struct irdma_device *iwdev) 2719 { 2720 u32 stag = 0; 2721 u32 stag_index = 0; 2722 u32 next_stag_index; 2723 u32 driver_key; 2724 u32 random; 2725 u8 consumer_key; 2726 int ret; 2727 2728 get_random_bytes(&random, sizeof(random)); 2729 consumer_key = (u8)random; 2730 2731 driver_key = random & ~iwdev->rf->mr_stagmask; 2732 next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; 2733 next_stag_index %= iwdev->rf->max_mr; 2734 2735 ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, 2736 iwdev->rf->max_mr, &stag_index, 2737 &next_stag_index); 2738 if (ret) 2739 return stag; 2740 stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; 2741 stag |= driver_key; 2742 stag += (u32)consumer_key; 2743 2744 return stag; 2745 } 2746 2747 /** 2748 * irdma_next_pbl_addr - Get next pbl address 2749 * @pbl: pointer to a pble 2750 * @pinfo: info pointer 2751 * @idx: index 2752 */ 2753 static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, 2754 u32 *idx) 2755 { 2756 *idx += 1; 2757 if (!(*pinfo) || *idx != (*pinfo)->cnt) 2758 return ++pbl; 2759 *idx = 0; 2760 (*pinfo)++; 2761 2762 return (*pinfo)->addr; 2763 } 2764 2765 /** 2766 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 2767 * @iwmr: iwmr for IB's user page addresses 2768 * @pbl: ple pointer to save 1 level or 0 level pble 2769 * @level: indicated level 0, 1 or 2 2770 */ 2771 static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 2772 enum irdma_pble_level level) 2773 { 2774 struct ib_umem *region = iwmr->region; 2775 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2776 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2777 struct irdma_pble_info *pinfo; 2778 struct ib_block_iter biter; 2779 u32 idx = 0; 2780 u32 pbl_cnt = 0; 2781 2782 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 2783 2784 if (iwmr->type == IRDMA_MEMREG_TYPE_QP) 2785 iwpbl->qp_mr.sq_page = sg_page(region->sgt_append.sgt.sgl); 2786 2787 rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { 2788 *pbl = rdma_block_iter_dma_address(&biter); 2789 if (++pbl_cnt == palloc->total_cnt) 2790 break; 2791 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 2792 } 2793 } 2794 2795 /** 2796 * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous 2797 * @arr: lvl1 pbl array 2798 * @npages: page count 2799 * @pg_size: page size 2800 * 2801 */ 2802 static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) 2803 { 2804 u32 pg_idx; 2805 2806 for (pg_idx = 0; pg_idx < npages; pg_idx++) { 2807 if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) 2808 return false; 2809 } 2810 2811 return true; 2812 } 2813 2814 /** 2815 * irdma_check_mr_contiguous - check if MR is physically contiguous 2816 * @palloc: pbl allocation struct 2817 * @pg_size: page size 2818 */ 2819 static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, 2820 u32 pg_size) 2821 { 2822 struct irdma_pble_level2 *lvl2 = &palloc->level2; 2823 struct irdma_pble_info *leaf = lvl2->leaf; 2824 u64 *arr = NULL; 2825 u64 *start_addr = NULL; 2826 int i; 2827 bool ret; 2828 2829 if (palloc->level == PBLE_LEVEL_1) { 2830 arr = palloc->level1.addr; 2831 ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, 2832 pg_size); 2833 return ret; 2834 } 2835 2836 start_addr = leaf->addr; 2837 2838 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { 2839 arr = leaf->addr; 2840 if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) 2841 return false; 2842 ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); 2843 if (!ret) 2844 return false; 2845 } 2846 2847 return true; 2848 } 2849 2850 /** 2851 * irdma_setup_pbles - copy user pg address to pble's 2852 * @rf: RDMA PCI function 2853 * @iwmr: mr pointer for this memory registration 2854 * @lvl: requested pble levels 2855 */ 2856 static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, 2857 u8 lvl) 2858 { 2859 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2860 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2861 struct irdma_pble_info *pinfo; 2862 u64 *pbl; 2863 int status; 2864 enum irdma_pble_level level = PBLE_LEVEL_1; 2865 2866 if (lvl) { 2867 status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, 2868 lvl); 2869 if (status) 2870 return status; 2871 2872 iwpbl->pbl_allocated = true; 2873 level = palloc->level; 2874 pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : 2875 palloc->level2.leaf; 2876 pbl = pinfo->addr; 2877 } else { 2878 pbl = iwmr->pgaddrmem; 2879 } 2880 2881 irdma_copy_user_pgaddrs(iwmr, pbl, level); 2882 2883 if (lvl) 2884 iwmr->pgaddrmem[0] = *pbl; 2885 2886 return 0; 2887 } 2888 2889 /** 2890 * irdma_handle_q_mem - handle memory for qp and cq 2891 * @iwdev: irdma device 2892 * @req: information for q memory management 2893 * @iwpbl: pble struct 2894 * @lvl: pble level mask 2895 */ 2896 static int irdma_handle_q_mem(struct irdma_device *iwdev, 2897 struct irdma_mem_reg_req *req, 2898 struct irdma_pbl *iwpbl, u8 lvl) 2899 { 2900 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2901 struct irdma_mr *iwmr = iwpbl->iwmr; 2902 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 2903 struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; 2904 struct irdma_srq_mr *srqmr = &iwpbl->srq_mr; 2905 struct irdma_hmc_pble *hmc_p; 2906 u64 *arr = iwmr->pgaddrmem; 2907 u32 pg_size, total; 2908 int err = 0; 2909 bool ret = true; 2910 2911 pg_size = iwmr->page_size; 2912 err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); 2913 if (err) 2914 return err; 2915 2916 if (lvl) 2917 arr = palloc->level1.addr; 2918 2919 switch (iwmr->type) { 2920 case IRDMA_MEMREG_TYPE_QP: 2921 total = req->sq_pages + req->rq_pages; 2922 hmc_p = &qpmr->sq_pbl; 2923 qpmr->shadow = (dma_addr_t)arr[total]; 2924 /* Need to use physical address for RQ of QP 2925 * in case it is associated with SRQ. 2926 */ 2927 qpmr->rq_pa = (dma_addr_t)arr[req->sq_pages]; 2928 if (lvl) { 2929 ret = irdma_check_mem_contiguous(arr, req->sq_pages, 2930 pg_size); 2931 if (ret) 2932 ret = irdma_check_mem_contiguous(&arr[req->sq_pages], 2933 req->rq_pages, 2934 pg_size); 2935 } 2936 2937 if (!ret) { 2938 hmc_p->idx = palloc->level1.idx; 2939 hmc_p = &qpmr->rq_pbl; 2940 hmc_p->idx = palloc->level1.idx + req->sq_pages; 2941 } else { 2942 hmc_p->addr = arr[0]; 2943 hmc_p = &qpmr->rq_pbl; 2944 hmc_p->addr = arr[req->sq_pages]; 2945 } 2946 break; 2947 case IRDMA_MEMREG_TYPE_SRQ: 2948 hmc_p = &srqmr->srq_pbl; 2949 srqmr->shadow = (dma_addr_t)arr[req->rq_pages]; 2950 if (lvl) 2951 ret = irdma_check_mem_contiguous(arr, req->rq_pages, 2952 pg_size); 2953 2954 if (!ret) 2955 hmc_p->idx = palloc->level1.idx; 2956 else 2957 hmc_p->addr = arr[0]; 2958 break; 2959 case IRDMA_MEMREG_TYPE_CQ: 2960 hmc_p = &cqmr->cq_pbl; 2961 2962 if (!cqmr->split) 2963 cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; 2964 2965 if (lvl) 2966 ret = irdma_check_mem_contiguous(arr, req->cq_pages, 2967 pg_size); 2968 2969 if (!ret) 2970 hmc_p->idx = palloc->level1.idx; 2971 else 2972 hmc_p->addr = arr[0]; 2973 break; 2974 default: 2975 ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n"); 2976 err = -EINVAL; 2977 } 2978 2979 if (lvl && ret) { 2980 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2981 iwpbl->pbl_allocated = false; 2982 } 2983 2984 return err; 2985 } 2986 2987 /** 2988 * irdma_hw_alloc_mw - create the hw memory window 2989 * @iwdev: irdma device 2990 * @iwmr: pointer to memory window info 2991 */ 2992 static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) 2993 { 2994 struct irdma_mw_alloc_info *info; 2995 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2996 struct irdma_cqp_request *cqp_request; 2997 struct cqp_cmds_info *cqp_info; 2998 int status; 2999 3000 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3001 if (!cqp_request) 3002 return -ENOMEM; 3003 3004 cqp_info = &cqp_request->info; 3005 info = &cqp_info->in.u.mw_alloc.info; 3006 memset(info, 0, sizeof(*info)); 3007 if (iwmr->ibmw.type == IB_MW_TYPE_1) 3008 info->mw_wide = true; 3009 3010 info->page_size = PAGE_SIZE; 3011 info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3012 info->pd_id = iwpd->sc_pd.pd_id; 3013 info->remote_access = true; 3014 cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; 3015 cqp_info->post_sq = 1; 3016 cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; 3017 cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; 3018 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3019 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3020 3021 return status; 3022 } 3023 3024 /** 3025 * irdma_alloc_mw - Allocate memory window 3026 * @ibmw: Memory Window 3027 * @udata: user data pointer 3028 */ 3029 static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 3030 { 3031 struct irdma_device *iwdev = to_iwdev(ibmw->device); 3032 struct irdma_mr *iwmr = to_iwmw(ibmw); 3033 int err_code; 3034 u32 stag; 3035 3036 stag = irdma_create_stag(iwdev); 3037 if (!stag) 3038 return -ENOMEM; 3039 3040 iwmr->stag = stag; 3041 ibmw->rkey = stag; 3042 3043 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 3044 if (err_code) { 3045 irdma_free_stag(iwdev, stag); 3046 return err_code; 3047 } 3048 3049 return 0; 3050 } 3051 3052 /** 3053 * irdma_dealloc_mw - Dealloc memory window 3054 * @ibmw: memory window structure. 3055 */ 3056 static int irdma_dealloc_mw(struct ib_mw *ibmw) 3057 { 3058 struct ib_pd *ibpd = ibmw->pd; 3059 struct irdma_pd *iwpd = to_iwpd(ibpd); 3060 struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); 3061 struct irdma_device *iwdev = to_iwdev(ibmw->device); 3062 struct irdma_cqp_request *cqp_request; 3063 struct cqp_cmds_info *cqp_info; 3064 struct irdma_dealloc_stag_info *info; 3065 3066 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3067 if (!cqp_request) 3068 return -ENOMEM; 3069 3070 cqp_info = &cqp_request->info; 3071 info = &cqp_info->in.u.dealloc_stag.info; 3072 memset(info, 0, sizeof(*info)); 3073 info->pd_id = iwpd->sc_pd.pd_id; 3074 info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3075 info->mr = false; 3076 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3077 cqp_info->post_sq = 1; 3078 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3079 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3080 irdma_handle_cqp_op(iwdev->rf, cqp_request); 3081 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3082 irdma_free_stag(iwdev, iwmr->stag); 3083 3084 return 0; 3085 } 3086 3087 /** 3088 * irdma_hw_alloc_stag - cqp command to allocate stag 3089 * @iwdev: irdma device 3090 * @iwmr: irdma mr pointer 3091 */ 3092 static int irdma_hw_alloc_stag(struct irdma_device *iwdev, 3093 struct irdma_mr *iwmr) 3094 { 3095 struct irdma_allocate_stag_info *info; 3096 struct ib_pd *pd = iwmr->ibmr.pd; 3097 struct irdma_pd *iwpd = to_iwpd(pd); 3098 int status; 3099 struct irdma_cqp_request *cqp_request; 3100 struct cqp_cmds_info *cqp_info; 3101 3102 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3103 if (!cqp_request) 3104 return -ENOMEM; 3105 3106 cqp_info = &cqp_request->info; 3107 info = &cqp_info->in.u.alloc_stag.info; 3108 memset(info, 0, sizeof(*info)); 3109 info->page_size = PAGE_SIZE; 3110 info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3111 info->pd_id = iwpd->sc_pd.pd_id; 3112 info->total_len = iwmr->len; 3113 info->all_memory = pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY; 3114 info->remote_access = true; 3115 cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; 3116 cqp_info->post_sq = 1; 3117 cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; 3118 cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; 3119 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3120 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3121 if (status) 3122 return status; 3123 3124 iwmr->is_hwreg = 1; 3125 return 0; 3126 } 3127 3128 /** 3129 * irdma_alloc_mr - register stag for fast memory registration 3130 * @pd: ibpd pointer 3131 * @mr_type: memory for stag registrion 3132 * @max_num_sg: man number of pages 3133 */ 3134 static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 3135 u32 max_num_sg) 3136 { 3137 struct irdma_device *iwdev = to_iwdev(pd->device); 3138 struct irdma_pble_alloc *palloc; 3139 struct irdma_pbl *iwpbl; 3140 struct irdma_mr *iwmr; 3141 u32 stag; 3142 int err_code; 3143 3144 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 3145 if (!iwmr) 3146 return ERR_PTR(-ENOMEM); 3147 3148 stag = irdma_create_stag(iwdev); 3149 if (!stag) { 3150 err_code = -ENOMEM; 3151 goto err; 3152 } 3153 3154 iwmr->stag = stag; 3155 iwmr->ibmr.rkey = stag; 3156 iwmr->ibmr.lkey = stag; 3157 iwmr->ibmr.pd = pd; 3158 iwmr->ibmr.device = pd->device; 3159 iwpbl = &iwmr->iwpbl; 3160 iwpbl->iwmr = iwmr; 3161 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 3162 palloc = &iwpbl->pble_alloc; 3163 iwmr->page_cnt = max_num_sg; 3164 /* Use system PAGE_SIZE as the sg page sizes are unknown at this point */ 3165 iwmr->len = max_num_sg * PAGE_SIZE; 3166 err_code = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 3167 false); 3168 if (err_code) 3169 goto err_get_pble; 3170 3171 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 3172 if (err_code) 3173 goto err_alloc_stag; 3174 3175 iwpbl->pbl_allocated = true; 3176 3177 return &iwmr->ibmr; 3178 err_alloc_stag: 3179 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3180 err_get_pble: 3181 irdma_free_stag(iwdev, stag); 3182 err: 3183 kfree(iwmr); 3184 3185 return ERR_PTR(err_code); 3186 } 3187 3188 /** 3189 * irdma_set_page - populate pbl list for fmr 3190 * @ibmr: ib mem to access iwarp mr pointer 3191 * @addr: page dma address fro pbl list 3192 */ 3193 static int irdma_set_page(struct ib_mr *ibmr, u64 addr) 3194 { 3195 struct irdma_mr *iwmr = to_iwmr(ibmr); 3196 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3197 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3198 u64 *pbl; 3199 3200 if (unlikely(iwmr->npages == iwmr->page_cnt)) 3201 return -ENOMEM; 3202 3203 if (palloc->level == PBLE_LEVEL_2) { 3204 struct irdma_pble_info *palloc_info = 3205 palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT); 3206 3207 palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr; 3208 } else { 3209 pbl = palloc->level1.addr; 3210 pbl[iwmr->npages] = addr; 3211 } 3212 iwmr->npages++; 3213 3214 return 0; 3215 } 3216 3217 /** 3218 * irdma_map_mr_sg - map of sg list for fmr 3219 * @ibmr: ib mem to access iwarp mr pointer 3220 * @sg: scatter gather list 3221 * @sg_nents: number of sg pages 3222 * @sg_offset: scatter gather list for fmr 3223 */ 3224 static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3225 int sg_nents, unsigned int *sg_offset) 3226 { 3227 struct irdma_mr *iwmr = to_iwmr(ibmr); 3228 3229 iwmr->npages = 0; 3230 3231 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); 3232 } 3233 3234 /** 3235 * irdma_hwreg_mr - send cqp command for memory registration 3236 * @iwdev: irdma device 3237 * @iwmr: irdma mr pointer 3238 * @access: access for MR 3239 */ 3240 static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, 3241 u16 access) 3242 { 3243 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3244 struct irdma_reg_ns_stag_info *stag_info; 3245 struct ib_pd *pd = iwmr->ibmr.pd; 3246 struct irdma_pd *iwpd = to_iwpd(pd); 3247 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3248 struct irdma_cqp_request *cqp_request; 3249 struct cqp_cmds_info *cqp_info; 3250 int ret; 3251 3252 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3253 if (!cqp_request) 3254 return -ENOMEM; 3255 3256 cqp_info = &cqp_request->info; 3257 stag_info = &cqp_info->in.u.mr_reg_non_shared.info; 3258 memset(stag_info, 0, sizeof(*stag_info)); 3259 stag_info->va = iwpbl->user_base; 3260 stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 3261 stag_info->stag_key = (u8)iwmr->stag; 3262 stag_info->total_len = iwmr->len; 3263 stag_info->access_rights = irdma_get_mr_access(access, 3264 iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev); 3265 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_ATOMIC_OPS) 3266 stag_info->remote_atomics_en = (access & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3267 stag_info->pd_id = iwpd->sc_pd.pd_id; 3268 stag_info->all_memory = pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY; 3269 if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) 3270 stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; 3271 else 3272 stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; 3273 stag_info->page_size = iwmr->page_size; 3274 3275 if (iwpbl->pbl_allocated) { 3276 if (palloc->level == PBLE_LEVEL_1) { 3277 stag_info->first_pm_pbl_index = palloc->level1.idx; 3278 stag_info->chunk_size = 1; 3279 } else { 3280 stag_info->first_pm_pbl_index = palloc->level2.root.idx; 3281 stag_info->chunk_size = 3; 3282 } 3283 } else { 3284 stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; 3285 } 3286 3287 cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; 3288 cqp_info->post_sq = 1; 3289 cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; 3290 cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; 3291 ret = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3292 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3293 3294 if (!ret) 3295 iwmr->is_hwreg = 1; 3296 3297 return ret; 3298 } 3299 3300 static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, 3301 bool create_stag) 3302 { 3303 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3304 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3305 u32 stag = 0; 3306 u8 lvl; 3307 int err; 3308 3309 lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; 3310 3311 err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); 3312 if (err) 3313 return err; 3314 3315 if (lvl) { 3316 err = irdma_check_mr_contiguous(&iwpbl->pble_alloc, 3317 iwmr->page_size); 3318 if (err) { 3319 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 3320 iwpbl->pbl_allocated = false; 3321 } 3322 } 3323 3324 if (create_stag) { 3325 stag = irdma_create_stag(iwdev); 3326 if (!stag) { 3327 err = -ENOMEM; 3328 goto free_pble; 3329 } 3330 3331 iwmr->stag = stag; 3332 iwmr->ibmr.rkey = stag; 3333 iwmr->ibmr.lkey = stag; 3334 } 3335 3336 err = irdma_hwreg_mr(iwdev, iwmr, access); 3337 if (err) 3338 goto err_hwreg; 3339 3340 return 0; 3341 3342 err_hwreg: 3343 if (stag) 3344 irdma_free_stag(iwdev, stag); 3345 3346 free_pble: 3347 if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 3348 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 3349 3350 return err; 3351 } 3352 3353 static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, 3354 struct ib_pd *pd, u64 virt, 3355 enum irdma_memreg_type reg_type) 3356 { 3357 struct irdma_device *iwdev = to_iwdev(pd->device); 3358 struct irdma_pbl *iwpbl; 3359 struct irdma_mr *iwmr; 3360 unsigned long pgsz_bitmap; 3361 3362 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 3363 if (!iwmr) 3364 return ERR_PTR(-ENOMEM); 3365 3366 iwpbl = &iwmr->iwpbl; 3367 iwpbl->iwmr = iwmr; 3368 iwmr->region = region; 3369 iwmr->ibmr.pd = pd; 3370 iwmr->ibmr.device = pd->device; 3371 iwmr->ibmr.iova = virt; 3372 iwmr->type = reg_type; 3373 3374 pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ? 3375 iwdev->rf->sc_dev.hw_attrs.page_size_cap : SZ_4K; 3376 3377 iwmr->page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt); 3378 if (unlikely(!iwmr->page_size)) { 3379 kfree(iwmr); 3380 return ERR_PTR(-EOPNOTSUPP); 3381 } 3382 3383 iwmr->len = region->length; 3384 iwpbl->user_base = virt; 3385 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 3386 3387 return iwmr; 3388 } 3389 3390 static void irdma_free_iwmr(struct irdma_mr *iwmr) 3391 { 3392 kfree(iwmr); 3393 } 3394 3395 static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req, 3396 struct ib_udata *udata, 3397 struct irdma_mr *iwmr) 3398 { 3399 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3400 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3401 struct irdma_ucontext *ucontext = NULL; 3402 unsigned long flags; 3403 u32 total; 3404 int err; 3405 u8 lvl; 3406 3407 /* iWarp: Catch page not starting on OS page boundary */ 3408 if (!rdma_protocol_roce(&iwdev->ibdev, 1) && 3409 ib_umem_offset(iwmr->region)) 3410 return -EINVAL; 3411 3412 total = req.sq_pages + req.rq_pages + 1; 3413 if (total > iwmr->page_cnt) 3414 return -EINVAL; 3415 3416 total = req.sq_pages + req.rq_pages; 3417 lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3418 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3419 if (err) 3420 return err; 3421 3422 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3423 ibucontext); 3424 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 3425 list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 3426 iwpbl->on_list = true; 3427 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 3428 3429 return 0; 3430 } 3431 3432 static int irdma_reg_user_mr_type_srq(struct irdma_mem_reg_req req, 3433 struct ib_udata *udata, 3434 struct irdma_mr *iwmr) 3435 { 3436 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3437 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3438 struct irdma_ucontext *ucontext; 3439 unsigned long flags; 3440 u32 total; 3441 int err; 3442 u8 lvl; 3443 3444 total = req.rq_pages + IRDMA_SHADOW_PGCNT; 3445 if (total > iwmr->page_cnt) 3446 return -EINVAL; 3447 3448 lvl = req.rq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3449 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3450 if (err) 3451 return err; 3452 3453 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3454 ibucontext); 3455 spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); 3456 list_add_tail(&iwpbl->list, &ucontext->srq_reg_mem_list); 3457 iwpbl->on_list = true; 3458 spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); 3459 3460 return 0; 3461 } 3462 3463 static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req, 3464 struct ib_udata *udata, 3465 struct irdma_mr *iwmr) 3466 { 3467 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3468 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3469 struct irdma_ucontext *ucontext = NULL; 3470 u8 shadow_pgcnt = 1; 3471 unsigned long flags; 3472 u32 total; 3473 int err; 3474 u8 lvl; 3475 3476 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 3477 shadow_pgcnt = 0; 3478 total = req.cq_pages + shadow_pgcnt; 3479 if (total > iwmr->page_cnt) 3480 return -EINVAL; 3481 3482 lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; 3483 err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); 3484 if (err) 3485 return err; 3486 3487 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 3488 ibucontext); 3489 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 3490 list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 3491 iwpbl->on_list = true; 3492 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 3493 3494 return 0; 3495 } 3496 3497 /** 3498 * irdma_reg_user_mr - Register a user memory region 3499 * @pd: ptr of pd 3500 * @start: virtual start address 3501 * @len: length of mr 3502 * @virt: virtual address 3503 * @access: access of mr 3504 * @dmah: dma handle 3505 * @udata: user data 3506 */ 3507 static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, 3508 u64 virt, int access, 3509 struct ib_dmah *dmah, 3510 struct ib_udata *udata) 3511 { 3512 #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) 3513 struct irdma_device *iwdev = to_iwdev(pd->device); 3514 struct irdma_mem_reg_req req = {}; 3515 struct ib_umem *region = NULL; 3516 struct irdma_mr *iwmr = NULL; 3517 int err; 3518 3519 if (dmah) 3520 return ERR_PTR(-EOPNOTSUPP); 3521 3522 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3523 return ERR_PTR(-EINVAL); 3524 3525 if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) 3526 return ERR_PTR(-EINVAL); 3527 3528 region = ib_umem_get(pd->device, start, len, access); 3529 3530 if (IS_ERR(region)) { 3531 ibdev_dbg(&iwdev->ibdev, 3532 "VERBS: Failed to create ib_umem region\n"); 3533 return (struct ib_mr *)region; 3534 } 3535 3536 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { 3537 ib_umem_release(region); 3538 return ERR_PTR(-EFAULT); 3539 } 3540 3541 iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type); 3542 if (IS_ERR(iwmr)) { 3543 ib_umem_release(region); 3544 return (struct ib_mr *)iwmr; 3545 } 3546 3547 switch (req.reg_type) { 3548 case IRDMA_MEMREG_TYPE_QP: 3549 err = irdma_reg_user_mr_type_qp(req, udata, iwmr); 3550 if (err) 3551 goto error; 3552 3553 break; 3554 case IRDMA_MEMREG_TYPE_SRQ: 3555 err = irdma_reg_user_mr_type_srq(req, udata, iwmr); 3556 if (err) 3557 goto error; 3558 3559 break; 3560 case IRDMA_MEMREG_TYPE_CQ: 3561 err = irdma_reg_user_mr_type_cq(req, udata, iwmr); 3562 if (err) 3563 goto error; 3564 break; 3565 case IRDMA_MEMREG_TYPE_MEM: 3566 err = irdma_reg_user_mr_type_mem(iwmr, access, true); 3567 if (err) 3568 goto error; 3569 3570 break; 3571 default: 3572 err = -EINVAL; 3573 goto error; 3574 } 3575 3576 return &iwmr->ibmr; 3577 error: 3578 ib_umem_release(region); 3579 irdma_free_iwmr(iwmr); 3580 3581 return ERR_PTR(err); 3582 } 3583 3584 static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start, 3585 u64 len, u64 virt, 3586 int fd, int access, 3587 struct ib_dmah *dmah, 3588 struct uverbs_attr_bundle *attrs) 3589 { 3590 struct irdma_device *iwdev = to_iwdev(pd->device); 3591 struct ib_umem_dmabuf *umem_dmabuf; 3592 struct irdma_mr *iwmr; 3593 int err; 3594 3595 if (dmah) 3596 return ERR_PTR(-EOPNOTSUPP); 3597 3598 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3599 return ERR_PTR(-EINVAL); 3600 3601 umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd, access); 3602 if (IS_ERR(umem_dmabuf)) { 3603 err = PTR_ERR(umem_dmabuf); 3604 ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n", err); 3605 return ERR_PTR(err); 3606 } 3607 3608 iwmr = irdma_alloc_iwmr(&umem_dmabuf->umem, pd, virt, IRDMA_MEMREG_TYPE_MEM); 3609 if (IS_ERR(iwmr)) { 3610 err = PTR_ERR(iwmr); 3611 goto err_release; 3612 } 3613 3614 err = irdma_reg_user_mr_type_mem(iwmr, access, true); 3615 if (err) 3616 goto err_iwmr; 3617 3618 return &iwmr->ibmr; 3619 3620 err_iwmr: 3621 irdma_free_iwmr(iwmr); 3622 3623 err_release: 3624 ib_umem_release(&umem_dmabuf->umem); 3625 3626 return ERR_PTR(err); 3627 } 3628 3629 static int irdma_hwdereg_mr(struct ib_mr *ib_mr) 3630 { 3631 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3632 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3633 struct irdma_pd *iwpd = to_iwpd(ib_mr->pd); 3634 struct irdma_dealloc_stag_info *info; 3635 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3636 struct irdma_cqp_request *cqp_request; 3637 struct cqp_cmds_info *cqp_info; 3638 int status; 3639 3640 /* Skip HW MR de-register when it is already de-registered 3641 * during an MR re-reregister and the re-registration fails 3642 */ 3643 if (!iwmr->is_hwreg) 3644 return 0; 3645 3646 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3647 if (!cqp_request) 3648 return -ENOMEM; 3649 3650 cqp_info = &cqp_request->info; 3651 info = &cqp_info->in.u.dealloc_stag.info; 3652 memset(info, 0, sizeof(*info)); 3653 info->pd_id = iwpd->sc_pd.pd_id; 3654 info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3655 info->mr = true; 3656 if (iwpbl->pbl_allocated) 3657 info->dealloc_pbl = true; 3658 3659 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3660 cqp_info->post_sq = 1; 3661 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3662 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3663 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3664 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3665 if (status) 3666 return status; 3667 3668 iwmr->is_hwreg = 0; 3669 return 0; 3670 } 3671 3672 /* 3673 * irdma_rereg_mr_trans - Re-register a user MR for a change translation. 3674 * @iwmr: ptr of iwmr 3675 * @start: virtual start address 3676 * @len: length of mr 3677 * @virt: virtual address 3678 * 3679 * Re-register a user memory region when a change translation is requested. 3680 * Re-register a new region while reusing the stag from the original registration. 3681 */ 3682 static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, 3683 u64 virt) 3684 { 3685 struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); 3686 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3687 struct ib_pd *pd = iwmr->ibmr.pd; 3688 struct ib_umem *region; 3689 int err; 3690 3691 region = ib_umem_get(pd->device, start, len, iwmr->access); 3692 if (IS_ERR(region)) 3693 return PTR_ERR(region); 3694 3695 iwmr->region = region; 3696 iwmr->ibmr.iova = virt; 3697 iwmr->ibmr.pd = pd; 3698 iwmr->page_size = ib_umem_find_best_pgsz(region, 3699 iwdev->rf->sc_dev.hw_attrs.page_size_cap, 3700 virt); 3701 if (unlikely(!iwmr->page_size)) { 3702 err = -EOPNOTSUPP; 3703 goto err; 3704 } 3705 3706 iwmr->len = region->length; 3707 iwpbl->user_base = virt; 3708 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 3709 3710 err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false); 3711 if (err) 3712 goto err; 3713 3714 return 0; 3715 3716 err: 3717 ib_umem_release(region); 3718 return err; 3719 } 3720 3721 /* 3722 * irdma_rereg_user_mr - Re-Register a user memory region(MR) 3723 * @ibmr: ib mem to access iwarp mr pointer 3724 * @flags: bit mask to indicate which of the attr's of MR modified 3725 * @start: virtual start address 3726 * @len: length of mr 3727 * @virt: virtual address 3728 * @new_access: bit mask of access flags 3729 * @new_pd: ptr of pd 3730 * @udata: user data 3731 * 3732 * Return: 3733 * NULL - Success, existing MR updated 3734 * ERR_PTR - error occurred 3735 */ 3736 static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, 3737 u64 start, u64 len, u64 virt, 3738 int new_access, struct ib_pd *new_pd, 3739 struct ib_udata *udata) 3740 { 3741 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3742 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3743 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3744 int ret; 3745 3746 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 3747 return ERR_PTR(-EINVAL); 3748 3749 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) 3750 return ERR_PTR(-EOPNOTSUPP); 3751 3752 ret = irdma_hwdereg_mr(ib_mr); 3753 if (ret) 3754 return ERR_PTR(ret); 3755 3756 if (flags & IB_MR_REREG_ACCESS) 3757 iwmr->access = new_access; 3758 3759 if (flags & IB_MR_REREG_PD) { 3760 iwmr->ibmr.pd = new_pd; 3761 iwmr->ibmr.device = new_pd->device; 3762 } 3763 3764 if (flags & IB_MR_REREG_TRANS) { 3765 if (iwpbl->pbl_allocated) { 3766 irdma_free_pble(iwdev->rf->pble_rsrc, 3767 &iwpbl->pble_alloc); 3768 iwpbl->pbl_allocated = false; 3769 } 3770 if (iwmr->region) { 3771 ib_umem_release(iwmr->region); 3772 iwmr->region = NULL; 3773 } 3774 3775 ret = irdma_rereg_mr_trans(iwmr, start, len, virt); 3776 } else 3777 ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access); 3778 if (ret) 3779 return ERR_PTR(ret); 3780 3781 return NULL; 3782 } 3783 3784 /** 3785 * irdma_reg_phys_mr - register kernel physical memory 3786 * @pd: ibpd pointer 3787 * @addr: physical address of memory to register 3788 * @size: size of memory to register 3789 * @access: Access rights 3790 * @iova_start: start of virtual address for physical buffers 3791 */ 3792 struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, 3793 u64 *iova_start) 3794 { 3795 struct irdma_device *iwdev = to_iwdev(pd->device); 3796 struct irdma_pbl *iwpbl; 3797 struct irdma_mr *iwmr; 3798 u32 stag; 3799 int ret; 3800 3801 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 3802 if (!iwmr) 3803 return ERR_PTR(-ENOMEM); 3804 3805 iwmr->ibmr.pd = pd; 3806 iwmr->ibmr.device = pd->device; 3807 iwpbl = &iwmr->iwpbl; 3808 iwpbl->iwmr = iwmr; 3809 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 3810 iwpbl->user_base = *iova_start; 3811 stag = irdma_create_stag(iwdev); 3812 if (!stag) { 3813 ret = -ENOMEM; 3814 goto err; 3815 } 3816 3817 iwmr->stag = stag; 3818 iwmr->ibmr.iova = *iova_start; 3819 iwmr->ibmr.rkey = stag; 3820 iwmr->ibmr.lkey = stag; 3821 iwmr->page_cnt = 1; 3822 iwmr->pgaddrmem[0] = addr; 3823 iwmr->len = size; 3824 iwmr->page_size = SZ_4K; 3825 ret = irdma_hwreg_mr(iwdev, iwmr, access); 3826 if (ret) { 3827 irdma_free_stag(iwdev, stag); 3828 goto err; 3829 } 3830 3831 return &iwmr->ibmr; 3832 3833 err: 3834 kfree(iwmr); 3835 3836 return ERR_PTR(ret); 3837 } 3838 3839 /** 3840 * irdma_get_dma_mr - register physical mem 3841 * @pd: ptr of pd 3842 * @acc: access for memory 3843 */ 3844 static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) 3845 { 3846 u64 kva = 0; 3847 3848 return irdma_reg_phys_mr(pd, 0, 0, acc, &kva); 3849 } 3850 3851 /** 3852 * irdma_del_memlist - Deleting pbl list entries for CQ/QP 3853 * @iwmr: iwmr for IB's user page addresses 3854 * @ucontext: ptr to user context 3855 */ 3856 static void irdma_del_memlist(struct irdma_mr *iwmr, 3857 struct irdma_ucontext *ucontext) 3858 { 3859 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3860 unsigned long flags; 3861 3862 switch (iwmr->type) { 3863 case IRDMA_MEMREG_TYPE_CQ: 3864 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 3865 if (iwpbl->on_list) { 3866 iwpbl->on_list = false; 3867 list_del(&iwpbl->list); 3868 } 3869 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 3870 break; 3871 case IRDMA_MEMREG_TYPE_QP: 3872 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 3873 if (iwpbl->on_list) { 3874 iwpbl->on_list = false; 3875 list_del(&iwpbl->list); 3876 } 3877 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 3878 break; 3879 case IRDMA_MEMREG_TYPE_SRQ: 3880 spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); 3881 if (iwpbl->on_list) { 3882 iwpbl->on_list = false; 3883 list_del(&iwpbl->list); 3884 } 3885 spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); 3886 break; 3887 default: 3888 break; 3889 } 3890 } 3891 3892 /** 3893 * irdma_dereg_mr - deregister mr 3894 * @ib_mr: mr ptr for dereg 3895 * @udata: user data 3896 */ 3897 static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3898 { 3899 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3900 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3901 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3902 int ret; 3903 3904 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 3905 if (iwmr->region) { 3906 struct irdma_ucontext *ucontext; 3907 3908 ucontext = rdma_udata_to_drv_context(udata, 3909 struct irdma_ucontext, 3910 ibucontext); 3911 irdma_del_memlist(iwmr, ucontext); 3912 } 3913 goto done; 3914 } 3915 3916 ret = irdma_hwdereg_mr(ib_mr); 3917 if (ret) 3918 return ret; 3919 3920 irdma_free_stag(iwdev, iwmr->stag); 3921 done: 3922 if (iwpbl->pbl_allocated) 3923 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 3924 3925 if (iwmr->region) 3926 ib_umem_release(iwmr->region); 3927 3928 kfree(iwmr); 3929 3930 return 0; 3931 } 3932 3933 /** 3934 * irdma_post_send - kernel application wr 3935 * @ibqp: qp ptr for wr 3936 * @ib_wr: work request ptr 3937 * @bad_wr: return of bad wr if err 3938 */ 3939 static int irdma_post_send(struct ib_qp *ibqp, 3940 const struct ib_send_wr *ib_wr, 3941 const struct ib_send_wr **bad_wr) 3942 { 3943 struct irdma_qp *iwqp; 3944 struct irdma_qp_uk *ukqp; 3945 struct irdma_sc_dev *dev; 3946 struct irdma_post_sq_info info; 3947 int err = 0; 3948 unsigned long flags; 3949 bool inv_stag; 3950 struct irdma_ah *ah; 3951 3952 iwqp = to_iwqp(ibqp); 3953 ukqp = &iwqp->sc_qp.qp_uk; 3954 dev = &iwqp->iwdev->rf->sc_dev; 3955 3956 spin_lock_irqsave(&iwqp->lock, flags); 3957 while (ib_wr) { 3958 memset(&info, 0, sizeof(info)); 3959 inv_stag = false; 3960 info.wr_id = (ib_wr->wr_id); 3961 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) 3962 info.signaled = true; 3963 if (ib_wr->send_flags & IB_SEND_FENCE) 3964 info.read_fence = true; 3965 switch (ib_wr->opcode) { 3966 case IB_WR_ATOMIC_CMP_AND_SWP: 3967 if (unlikely(!(dev->hw_attrs.uk_attrs.feature_flags & 3968 IRDMA_FEATURE_ATOMIC_OPS))) { 3969 err = EINVAL; 3970 break; 3971 } 3972 info.op_type = IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP; 3973 info.op.atomic_compare_swap.tagged_offset = ib_wr->sg_list[0].addr; 3974 info.op.atomic_compare_swap.remote_tagged_offset = 3975 atomic_wr(ib_wr)->remote_addr; 3976 info.op.atomic_compare_swap.swap_data_bytes = atomic_wr(ib_wr)->swap; 3977 info.op.atomic_compare_swap.compare_data_bytes = 3978 atomic_wr(ib_wr)->compare_add; 3979 info.op.atomic_compare_swap.stag = ib_wr->sg_list[0].lkey; 3980 info.op.atomic_compare_swap.remote_stag = atomic_wr(ib_wr)->rkey; 3981 err = irdma_uk_atomic_compare_swap(ukqp, &info, false); 3982 break; 3983 case IB_WR_ATOMIC_FETCH_AND_ADD: 3984 if (unlikely(!(dev->hw_attrs.uk_attrs.feature_flags & 3985 IRDMA_FEATURE_ATOMIC_OPS))) { 3986 err = EINVAL; 3987 break; 3988 } 3989 info.op_type = IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD; 3990 info.op.atomic_fetch_add.tagged_offset = ib_wr->sg_list[0].addr; 3991 info.op.atomic_fetch_add.remote_tagged_offset = 3992 atomic_wr(ib_wr)->remote_addr; 3993 info.op.atomic_fetch_add.fetch_add_data_bytes = 3994 atomic_wr(ib_wr)->compare_add; 3995 info.op.atomic_fetch_add.stag = ib_wr->sg_list[0].lkey; 3996 info.op.atomic_fetch_add.remote_stag = 3997 atomic_wr(ib_wr)->rkey; 3998 err = irdma_uk_atomic_fetch_add(ukqp, &info, false); 3999 break; 4000 case IB_WR_SEND_WITH_IMM: 4001 if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { 4002 info.imm_data_valid = true; 4003 info.imm_data = ntohl(ib_wr->ex.imm_data); 4004 } else { 4005 err = -EINVAL; 4006 break; 4007 } 4008 fallthrough; 4009 case IB_WR_SEND: 4010 case IB_WR_SEND_WITH_INV: 4011 if (ib_wr->opcode == IB_WR_SEND || 4012 ib_wr->opcode == IB_WR_SEND_WITH_IMM) { 4013 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4014 info.op_type = IRDMA_OP_TYPE_SEND_SOL; 4015 else 4016 info.op_type = IRDMA_OP_TYPE_SEND; 4017 } else { 4018 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4019 info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; 4020 else 4021 info.op_type = IRDMA_OP_TYPE_SEND_INV; 4022 info.stag_to_inv = ib_wr->ex.invalidate_rkey; 4023 } 4024 4025 info.op.send.num_sges = ib_wr->num_sge; 4026 info.op.send.sg_list = ib_wr->sg_list; 4027 if (iwqp->ibqp.qp_type == IB_QPT_UD || 4028 iwqp->ibqp.qp_type == IB_QPT_GSI) { 4029 ah = to_iwah(ud_wr(ib_wr)->ah); 4030 info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; 4031 info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; 4032 info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; 4033 } 4034 4035 if (ib_wr->send_flags & IB_SEND_INLINE) 4036 err = irdma_uk_inline_send(ukqp, &info, false); 4037 else 4038 err = irdma_uk_send(ukqp, &info, false); 4039 break; 4040 case IB_WR_RDMA_WRITE_WITH_IMM: 4041 if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { 4042 info.imm_data_valid = true; 4043 info.imm_data = ntohl(ib_wr->ex.imm_data); 4044 } else { 4045 err = -EINVAL; 4046 break; 4047 } 4048 fallthrough; 4049 case IB_WR_RDMA_WRITE: 4050 if (ib_wr->send_flags & IB_SEND_SOLICITED) 4051 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; 4052 else 4053 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; 4054 4055 info.op.rdma_write.num_lo_sges = ib_wr->num_sge; 4056 info.op.rdma_write.lo_sg_list = ib_wr->sg_list; 4057 info.op.rdma_write.rem_addr.addr = 4058 rdma_wr(ib_wr)->remote_addr; 4059 info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 4060 if (ib_wr->send_flags & IB_SEND_INLINE) 4061 err = irdma_uk_inline_rdma_write(ukqp, &info, false); 4062 else 4063 err = irdma_uk_rdma_write(ukqp, &info, false); 4064 break; 4065 case IB_WR_RDMA_READ_WITH_INV: 4066 inv_stag = true; 4067 fallthrough; 4068 case IB_WR_RDMA_READ: 4069 if (ib_wr->num_sge > 4070 dev->hw_attrs.uk_attrs.max_hw_read_sges) { 4071 err = -EINVAL; 4072 break; 4073 } 4074 info.op_type = IRDMA_OP_TYPE_RDMA_READ; 4075 info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 4076 info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 4077 info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; 4078 info.op.rdma_read.num_lo_sges = ib_wr->num_sge; 4079 err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); 4080 break; 4081 case IB_WR_LOCAL_INV: 4082 info.op_type = IRDMA_OP_TYPE_INV_STAG; 4083 info.local_fence = info.read_fence; 4084 info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; 4085 err = irdma_uk_stag_local_invalidate(ukqp, &info, true); 4086 break; 4087 case IB_WR_REG_MR: { 4088 struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); 4089 struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; 4090 struct irdma_fast_reg_stag_info stag_info = {}; 4091 4092 stag_info.signaled = info.signaled; 4093 stag_info.read_fence = info.read_fence; 4094 stag_info.access_rights = 4095 irdma_get_mr_access(reg_wr(ib_wr)->access, 4096 dev->hw_attrs.uk_attrs.hw_rev); 4097 stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; 4098 stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; 4099 stag_info.page_size = reg_wr(ib_wr)->mr->page_size; 4100 stag_info.wr_id = ib_wr->wr_id; 4101 stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; 4102 stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; 4103 stag_info.total_len = iwmr->ibmr.length; 4104 stag_info.reg_addr_pa = *palloc->level1.addr; 4105 stag_info.first_pm_pbl_index = palloc->level1.idx; 4106 stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; 4107 if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) 4108 stag_info.chunk_size = 1; 4109 err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, 4110 true); 4111 break; 4112 } 4113 default: 4114 err = -EINVAL; 4115 ibdev_dbg(&iwqp->iwdev->ibdev, 4116 "VERBS: upost_send bad opcode = 0x%x\n", 4117 ib_wr->opcode); 4118 break; 4119 } 4120 4121 if (err) 4122 break; 4123 ib_wr = ib_wr->next; 4124 } 4125 4126 if (!iwqp->flush_issued) { 4127 if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) 4128 irdma_uk_qp_post_wr(ukqp); 4129 spin_unlock_irqrestore(&iwqp->lock, flags); 4130 } else { 4131 spin_unlock_irqrestore(&iwqp->lock, flags); 4132 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 4133 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 4134 } 4135 4136 if (err) 4137 *bad_wr = ib_wr; 4138 4139 return err; 4140 } 4141 4142 /** 4143 * irdma_post_srq_recv - post receive wr for kernel application 4144 * @ibsrq: ib srq pointer 4145 * @ib_wr: work request for receive 4146 * @bad_wr: bad wr caused an error 4147 */ 4148 static int irdma_post_srq_recv(struct ib_srq *ibsrq, 4149 const struct ib_recv_wr *ib_wr, 4150 const struct ib_recv_wr **bad_wr) 4151 { 4152 struct irdma_srq *iwsrq = to_iwsrq(ibsrq); 4153 struct irdma_srq_uk *uksrq = &iwsrq->sc_srq.srq_uk; 4154 struct irdma_post_rq_info post_recv = {}; 4155 unsigned long flags; 4156 int err = 0; 4157 4158 spin_lock_irqsave(&iwsrq->lock, flags); 4159 while (ib_wr) { 4160 if (ib_wr->num_sge > uksrq->max_srq_frag_cnt) { 4161 err = -EINVAL; 4162 goto out; 4163 } 4164 post_recv.num_sges = ib_wr->num_sge; 4165 post_recv.wr_id = ib_wr->wr_id; 4166 post_recv.sg_list = ib_wr->sg_list; 4167 err = irdma_uk_srq_post_receive(uksrq, &post_recv); 4168 if (err) 4169 goto out; 4170 4171 ib_wr = ib_wr->next; 4172 } 4173 4174 out: 4175 spin_unlock_irqrestore(&iwsrq->lock, flags); 4176 4177 if (err) 4178 *bad_wr = ib_wr; 4179 4180 return err; 4181 } 4182 4183 /** 4184 * irdma_post_recv - post receive wr for kernel application 4185 * @ibqp: ib qp pointer 4186 * @ib_wr: work request for receive 4187 * @bad_wr: bad wr caused an error 4188 */ 4189 static int irdma_post_recv(struct ib_qp *ibqp, 4190 const struct ib_recv_wr *ib_wr, 4191 const struct ib_recv_wr **bad_wr) 4192 { 4193 struct irdma_qp *iwqp; 4194 struct irdma_qp_uk *ukqp; 4195 struct irdma_post_rq_info post_recv = {}; 4196 unsigned long flags; 4197 int err = 0; 4198 4199 iwqp = to_iwqp(ibqp); 4200 ukqp = &iwqp->sc_qp.qp_uk; 4201 4202 if (ukqp->srq_uk) { 4203 *bad_wr = ib_wr; 4204 return -EINVAL; 4205 } 4206 4207 spin_lock_irqsave(&iwqp->lock, flags); 4208 while (ib_wr) { 4209 post_recv.num_sges = ib_wr->num_sge; 4210 post_recv.wr_id = ib_wr->wr_id; 4211 post_recv.sg_list = ib_wr->sg_list; 4212 err = irdma_uk_post_receive(ukqp, &post_recv); 4213 if (err) { 4214 ibdev_dbg(&iwqp->iwdev->ibdev, 4215 "VERBS: post_recv err %d\n", err); 4216 goto out; 4217 } 4218 4219 ib_wr = ib_wr->next; 4220 } 4221 4222 out: 4223 spin_unlock_irqrestore(&iwqp->lock, flags); 4224 if (iwqp->flush_issued) 4225 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 4226 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 4227 4228 if (err) 4229 *bad_wr = ib_wr; 4230 4231 return err; 4232 } 4233 4234 /** 4235 * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status 4236 * @opcode: iwarp flush code 4237 */ 4238 static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) 4239 { 4240 switch (opcode) { 4241 case FLUSH_PROT_ERR: 4242 return IB_WC_LOC_PROT_ERR; 4243 case FLUSH_REM_ACCESS_ERR: 4244 return IB_WC_REM_ACCESS_ERR; 4245 case FLUSH_LOC_QP_OP_ERR: 4246 return IB_WC_LOC_QP_OP_ERR; 4247 case FLUSH_REM_OP_ERR: 4248 return IB_WC_REM_OP_ERR; 4249 case FLUSH_LOC_LEN_ERR: 4250 return IB_WC_LOC_LEN_ERR; 4251 case FLUSH_GENERAL_ERR: 4252 return IB_WC_WR_FLUSH_ERR; 4253 case FLUSH_RETRY_EXC_ERR: 4254 return IB_WC_RETRY_EXC_ERR; 4255 case FLUSH_MW_BIND_ERR: 4256 return IB_WC_MW_BIND_ERR; 4257 case FLUSH_REM_INV_REQ_ERR: 4258 return IB_WC_REM_INV_REQ_ERR; 4259 case FLUSH_RNR_RETRY_EXC_ERR: 4260 return IB_WC_RNR_RETRY_EXC_ERR; 4261 case FLUSH_FATAL_ERR: 4262 default: 4263 return IB_WC_FATAL_ERR; 4264 } 4265 } 4266 4267 /** 4268 * irdma_process_cqe - process cqe info 4269 * @entry: processed cqe 4270 * @cq_poll_info: cqe info 4271 */ 4272 static void irdma_process_cqe(struct ib_wc *entry, 4273 struct irdma_cq_poll_info *cq_poll_info) 4274 { 4275 struct irdma_sc_qp *qp; 4276 4277 entry->wc_flags = 0; 4278 entry->pkey_index = 0; 4279 entry->wr_id = cq_poll_info->wr_id; 4280 4281 qp = cq_poll_info->qp_handle; 4282 entry->qp = qp->qp_uk.back_qp; 4283 4284 if (cq_poll_info->error) { 4285 entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? 4286 irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; 4287 4288 entry->vendor_err = cq_poll_info->major_err << 16 | 4289 cq_poll_info->minor_err; 4290 } else { 4291 entry->status = IB_WC_SUCCESS; 4292 if (cq_poll_info->imm_valid) { 4293 entry->ex.imm_data = htonl(cq_poll_info->imm_data); 4294 entry->wc_flags |= IB_WC_WITH_IMM; 4295 } 4296 if (cq_poll_info->ud_smac_valid) { 4297 ether_addr_copy(entry->smac, cq_poll_info->ud_smac); 4298 entry->wc_flags |= IB_WC_WITH_SMAC; 4299 } 4300 4301 if (cq_poll_info->ud_vlan_valid) { 4302 u16 vlan = cq_poll_info->ud_vlan & VLAN_VID_MASK; 4303 4304 entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; 4305 if (vlan) { 4306 entry->vlan_id = vlan; 4307 entry->wc_flags |= IB_WC_WITH_VLAN; 4308 } 4309 } else { 4310 entry->sl = 0; 4311 } 4312 } 4313 4314 if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) { 4315 set_ib_wc_op_sq(cq_poll_info, entry); 4316 } else { 4317 if (qp->dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) 4318 set_ib_wc_op_rq(cq_poll_info, entry, 4319 qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM ? 4320 true : false); 4321 else 4322 set_ib_wc_op_rq_gen_3(cq_poll_info, entry); 4323 if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && 4324 cq_poll_info->stag_invalid_set) { 4325 entry->ex.invalidate_rkey = cq_poll_info->inv_stag; 4326 entry->wc_flags |= IB_WC_WITH_INVALIDATE; 4327 } 4328 } 4329 4330 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { 4331 entry->src_qp = cq_poll_info->ud_src_qpn; 4332 entry->slid = 0; 4333 entry->wc_flags |= 4334 (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); 4335 entry->network_hdr_type = cq_poll_info->ipv4 ? 4336 RDMA_NETWORK_IPV4 : 4337 RDMA_NETWORK_IPV6; 4338 } else { 4339 entry->src_qp = cq_poll_info->qp_id; 4340 } 4341 4342 entry->byte_len = cq_poll_info->bytes_xfered; 4343 } 4344 4345 /** 4346 * irdma_poll_one - poll one entry of the CQ 4347 * @ukcq: ukcq to poll 4348 * @cur_cqe: current CQE info to be filled in 4349 * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ 4350 * 4351 * Returns the internal irdma device error code or 0 on success 4352 */ 4353 static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, 4354 struct irdma_cq_poll_info *cur_cqe, 4355 struct ib_wc *entry) 4356 { 4357 int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); 4358 4359 if (ret) 4360 return ret; 4361 4362 irdma_process_cqe(entry, cur_cqe); 4363 4364 return 0; 4365 } 4366 4367 /** 4368 * __irdma_poll_cq - poll cq for completion (kernel apps) 4369 * @iwcq: cq to poll 4370 * @num_entries: number of entries to poll 4371 * @entry: wr of a completed entry 4372 */ 4373 static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) 4374 { 4375 struct list_head *tmp_node, *list_node; 4376 struct irdma_cq_buf *last_buf = NULL; 4377 struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; 4378 struct irdma_cq_buf *cq_buf; 4379 int ret; 4380 struct irdma_device *iwdev; 4381 struct irdma_cq_uk *ukcq; 4382 bool cq_new_cqe = false; 4383 int resized_bufs = 0; 4384 int npolled = 0; 4385 4386 iwdev = to_iwdev(iwcq->ibcq.device); 4387 ukcq = &iwcq->sc_cq.cq_uk; 4388 4389 /* go through the list of previously resized CQ buffers */ 4390 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 4391 cq_buf = container_of(list_node, struct irdma_cq_buf, list); 4392 while (npolled < num_entries) { 4393 ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); 4394 if (!ret) { 4395 ++npolled; 4396 cq_new_cqe = true; 4397 continue; 4398 } 4399 if (ret == -ENOENT) 4400 break; 4401 /* QP using the CQ is destroyed. Skip reporting this CQE */ 4402 if (ret == -EFAULT) { 4403 cq_new_cqe = true; 4404 continue; 4405 } 4406 goto error; 4407 } 4408 4409 /* save the resized CQ buffer which received the last cqe */ 4410 if (cq_new_cqe) 4411 last_buf = cq_buf; 4412 cq_new_cqe = false; 4413 } 4414 4415 /* check the current CQ for new cqes */ 4416 while (npolled < num_entries) { 4417 ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); 4418 if (ret == -ENOENT) { 4419 ret = irdma_generated_cmpls(iwcq, cur_cqe); 4420 if (!ret) 4421 irdma_process_cqe(entry + npolled, cur_cqe); 4422 } 4423 if (!ret) { 4424 ++npolled; 4425 cq_new_cqe = true; 4426 continue; 4427 } 4428 4429 if (ret == -ENOENT) 4430 break; 4431 /* QP using the CQ is destroyed. Skip reporting this CQE */ 4432 if (ret == -EFAULT) { 4433 cq_new_cqe = true; 4434 continue; 4435 } 4436 goto error; 4437 } 4438 4439 if (cq_new_cqe) 4440 /* all previous CQ resizes are complete */ 4441 resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); 4442 else if (last_buf) 4443 /* only CQ resizes up to the last_buf are complete */ 4444 resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); 4445 if (resized_bufs) 4446 /* report to the HW the number of complete CQ resizes */ 4447 irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); 4448 4449 return npolled; 4450 error: 4451 ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n", 4452 __func__, ret); 4453 4454 return ret; 4455 } 4456 4457 /** 4458 * irdma_poll_cq - poll cq for completion (kernel apps) 4459 * @ibcq: cq to poll 4460 * @num_entries: number of entries to poll 4461 * @entry: wr of a completed entry 4462 */ 4463 static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, 4464 struct ib_wc *entry) 4465 { 4466 struct irdma_cq *iwcq; 4467 unsigned long flags; 4468 int ret; 4469 4470 iwcq = to_iwcq(ibcq); 4471 4472 spin_lock_irqsave(&iwcq->lock, flags); 4473 ret = __irdma_poll_cq(iwcq, num_entries, entry); 4474 spin_unlock_irqrestore(&iwcq->lock, flags); 4475 4476 return ret; 4477 } 4478 4479 /** 4480 * irdma_req_notify_cq - arm cq kernel application 4481 * @ibcq: cq to arm 4482 * @notify_flags: notofication flags 4483 */ 4484 static int irdma_req_notify_cq(struct ib_cq *ibcq, 4485 enum ib_cq_notify_flags notify_flags) 4486 { 4487 struct irdma_cq *iwcq; 4488 struct irdma_cq_uk *ukcq; 4489 unsigned long flags; 4490 enum irdma_cmpl_notify cq_notify; 4491 bool promo_event = false; 4492 int ret = 0; 4493 4494 cq_notify = notify_flags == IB_CQ_SOLICITED ? 4495 IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT; 4496 iwcq = to_iwcq(ibcq); 4497 ukcq = &iwcq->sc_cq.cq_uk; 4498 4499 spin_lock_irqsave(&iwcq->lock, flags); 4500 /* Only promote to arm the CQ for any event if the last arm event was solicited. */ 4501 if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED && notify_flags != IB_CQ_SOLICITED) 4502 promo_event = true; 4503 4504 if (!atomic_cmpxchg(&iwcq->armed, 0, 1) || promo_event) { 4505 iwcq->last_notify = cq_notify; 4506 irdma_uk_cq_request_notification(ukcq, cq_notify); 4507 } 4508 4509 if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && 4510 (!irdma_cq_empty(iwcq) || !list_empty(&iwcq->cmpl_generated))) 4511 ret = 1; 4512 spin_unlock_irqrestore(&iwcq->lock, flags); 4513 4514 return ret; 4515 } 4516 4517 static const struct rdma_stat_desc irdma_hw_stat_descs[] = { 4518 /* gen1 - 32-bit */ 4519 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards", 4520 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts", 4521 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes", 4522 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards", 4523 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts", 4524 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes", 4525 [IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors", 4526 /* gen1 - 64-bit */ 4527 [IRDMA_HW_STAT_INDEX_IP4RXOCTS].name = "ip4InOctets", 4528 [IRDMA_HW_STAT_INDEX_IP4RXPKTS].name = "ip4InPkts", 4529 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS].name = "ip4InReasmRqd", 4530 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS].name = "ip4InMcastPkts", 4531 [IRDMA_HW_STAT_INDEX_IP4TXOCTS].name = "ip4OutOctets", 4532 [IRDMA_HW_STAT_INDEX_IP4TXPKTS].name = "ip4OutPkts", 4533 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS].name = "ip4OutSegRqd", 4534 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS].name = "ip4OutMcastPkts", 4535 [IRDMA_HW_STAT_INDEX_IP6RXOCTS].name = "ip6InOctets", 4536 [IRDMA_HW_STAT_INDEX_IP6RXPKTS].name = "ip6InPkts", 4537 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS].name = "ip6InReasmRqd", 4538 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS].name = "ip6InMcastPkts", 4539 [IRDMA_HW_STAT_INDEX_IP6TXOCTS].name = "ip6OutOctets", 4540 [IRDMA_HW_STAT_INDEX_IP6TXPKTS].name = "ip6OutPkts", 4541 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS].name = "ip6OutSegRqd", 4542 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS].name = "ip6OutMcastPkts", 4543 [IRDMA_HW_STAT_INDEX_RDMARXRDS].name = "InRdmaReads", 4544 [IRDMA_HW_STAT_INDEX_RDMARXSNDS].name = "InRdmaSends", 4545 [IRDMA_HW_STAT_INDEX_RDMARXWRS].name = "InRdmaWrites", 4546 [IRDMA_HW_STAT_INDEX_RDMATXRDS].name = "OutRdmaReads", 4547 [IRDMA_HW_STAT_INDEX_RDMATXSNDS].name = "OutRdmaSends", 4548 [IRDMA_HW_STAT_INDEX_RDMATXWRS].name = "OutRdmaWrites", 4549 [IRDMA_HW_STAT_INDEX_RDMAVBND].name = "RdmaBnd", 4550 [IRDMA_HW_STAT_INDEX_RDMAVINV].name = "RdmaInv", 4551 4552 /* gen2 - 32-bit */ 4553 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled", 4554 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored", 4555 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent", 4556 /* gen2 - 64-bit */ 4557 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS].name = "ip4InMcastOctets", 4558 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS].name = "ip4OutMcastOctets", 4559 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS].name = "ip6InMcastOctets", 4560 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS].name = "ip6OutMcastOctets", 4561 [IRDMA_HW_STAT_INDEX_UDPRXPKTS].name = "RxUDP", 4562 [IRDMA_HW_STAT_INDEX_UDPTXPKTS].name = "TxUDP", 4563 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS].name = "RxECNMrkd", 4564 [IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "RetransSegs", 4565 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "InOptErrors", 4566 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "InProtoErrors", 4567 [IRDMA_HW_STAT_INDEX_TCPRXSEGS].name = "InSegs", 4568 [IRDMA_HW_STAT_INDEX_TCPTXSEG].name = "OutSegs", 4569 4570 /* gen3 */ 4571 [IRDMA_HW_STAT_INDEX_RNR_SENT].name = "RNR sent", 4572 [IRDMA_HW_STAT_INDEX_RNR_RCVD].name = "RNR received", 4573 [IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT].name = "ord limit count", 4574 [IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT].name = "ird limit count", 4575 [IRDMA_HW_STAT_INDEX_RDMARXATS].name = "Rx atomics", 4576 [IRDMA_HW_STAT_INDEX_RDMATXATS].name = "Tx atomics", 4577 [IRDMA_HW_STAT_INDEX_NAKSEQERR].name = "Nak Sequence Error", 4578 [IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED].name = "Nak Sequence Error Implied", 4579 [IRDMA_HW_STAT_INDEX_RTO].name = "RTO", 4580 [IRDMA_HW_STAT_INDEX_RXOOOPKTS].name = "Rcvd Out of order packets", 4581 [IRDMA_HW_STAT_INDEX_ICRCERR].name = "CRC errors", 4582 }; 4583 4584 static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, 4585 struct ib_port_immutable *immutable) 4586 { 4587 struct ib_port_attr attr; 4588 int err; 4589 4590 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 4591 err = ib_query_port(ibdev, port_num, &attr); 4592 if (err) 4593 return err; 4594 4595 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 4596 immutable->pkey_tbl_len = attr.pkey_tbl_len; 4597 immutable->gid_tbl_len = attr.gid_tbl_len; 4598 4599 return 0; 4600 } 4601 4602 static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num, 4603 struct ib_port_immutable *immutable) 4604 { 4605 struct ib_port_attr attr; 4606 int err; 4607 4608 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 4609 err = ib_query_port(ibdev, port_num, &attr); 4610 if (err) 4611 return err; 4612 immutable->gid_tbl_len = attr.gid_tbl_len; 4613 4614 return 0; 4615 } 4616 4617 static void irdma_get_dev_fw_str(struct ib_device *dev, char *str) 4618 { 4619 struct irdma_device *iwdev = to_iwdev(dev); 4620 4621 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", 4622 irdma_fw_major_ver(&iwdev->rf->sc_dev), 4623 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 4624 } 4625 4626 /** 4627 * irdma_alloc_hw_port_stats - Allocate a hw stats structure 4628 * @ibdev: device pointer from stack 4629 * @port_num: port number 4630 */ 4631 static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev, 4632 u32 port_num) 4633 { 4634 struct irdma_device *iwdev = to_iwdev(ibdev); 4635 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 4636 4637 int num_counters = dev->hw_attrs.max_stat_idx; 4638 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 4639 4640 return rdma_alloc_hw_stats_struct(irdma_hw_stat_descs, num_counters, 4641 lifespan); 4642 } 4643 4644 /** 4645 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 4646 * @ibdev: device pointer from stack 4647 * @stats: stats pointer from stack 4648 * @port_num: port number 4649 * @index: which hw counter the stack is requesting we update 4650 */ 4651 static int irdma_get_hw_stats(struct ib_device *ibdev, 4652 struct rdma_hw_stats *stats, u32 port_num, 4653 int index) 4654 { 4655 struct irdma_device *iwdev = to_iwdev(ibdev); 4656 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 4657 4658 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 4659 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 4660 else 4661 irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat); 4662 4663 memcpy(&stats->value[0], hw_stats, sizeof(u64) * stats->num_counters); 4664 4665 return stats->num_counters; 4666 } 4667 4668 /** 4669 * irdma_query_gid - Query port GID 4670 * @ibdev: device pointer from stack 4671 * @port: port number 4672 * @index: Entry index 4673 * @gid: Global ID 4674 */ 4675 static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index, 4676 union ib_gid *gid) 4677 { 4678 struct irdma_device *iwdev = to_iwdev(ibdev); 4679 4680 memset(gid->raw, 0, sizeof(gid->raw)); 4681 ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); 4682 4683 return 0; 4684 } 4685 4686 /** 4687 * mcast_list_add - Add a new mcast item to list 4688 * @rf: RDMA PCI function 4689 * @new_elem: pointer to element to add 4690 */ 4691 static void mcast_list_add(struct irdma_pci_f *rf, 4692 struct mc_table_list *new_elem) 4693 { 4694 list_add(&new_elem->list, &rf->mc_qht_list.list); 4695 } 4696 4697 /** 4698 * mcast_list_del - Remove an mcast item from list 4699 * @mc_qht_elem: pointer to mcast table list element 4700 */ 4701 static void mcast_list_del(struct mc_table_list *mc_qht_elem) 4702 { 4703 if (mc_qht_elem) 4704 list_del(&mc_qht_elem->list); 4705 } 4706 4707 /** 4708 * mcast_list_lookup_ip - Search mcast list for address 4709 * @rf: RDMA PCI function 4710 * @ip_mcast: pointer to mcast IP address 4711 */ 4712 static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf, 4713 u32 *ip_mcast) 4714 { 4715 struct mc_table_list *mc_qht_el; 4716 struct list_head *pos, *q; 4717 4718 list_for_each_safe (pos, q, &rf->mc_qht_list.list) { 4719 mc_qht_el = list_entry(pos, struct mc_table_list, list); 4720 if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, 4721 sizeof(mc_qht_el->mc_info.dest_ip))) 4722 return mc_qht_el; 4723 } 4724 4725 return NULL; 4726 } 4727 4728 /** 4729 * irdma_mcast_cqp_op - perform a mcast cqp operation 4730 * @iwdev: irdma device 4731 * @mc_grp_ctx: mcast group info 4732 * @op: operation 4733 * 4734 * returns error status 4735 */ 4736 static int irdma_mcast_cqp_op(struct irdma_device *iwdev, 4737 struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) 4738 { 4739 struct cqp_cmds_info *cqp_info; 4740 struct irdma_cqp_request *cqp_request; 4741 int status; 4742 4743 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 4744 if (!cqp_request) 4745 return -ENOMEM; 4746 4747 cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; 4748 cqp_info = &cqp_request->info; 4749 cqp_info->cqp_cmd = op; 4750 cqp_info->post_sq = 1; 4751 cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; 4752 cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; 4753 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 4754 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 4755 4756 return status; 4757 } 4758 4759 /** 4760 * irdma_mcast_mac - Get the multicast MAC for an IP address 4761 * @ip_addr: IPv4 or IPv6 address 4762 * @mac: pointer to result MAC address 4763 * @ipv4: flag indicating IPv4 or IPv6 4764 * 4765 */ 4766 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4) 4767 { 4768 u8 *ip = (u8 *)ip_addr; 4769 4770 if (ipv4) { 4771 unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00, 4772 0x00, 0x00}; 4773 4774 mac4[3] = ip[2] & 0x7F; 4775 mac4[4] = ip[1]; 4776 mac4[5] = ip[0]; 4777 ether_addr_copy(mac, mac4); 4778 } else { 4779 unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00, 4780 0x00, 0x00}; 4781 4782 mac6[2] = ip[3]; 4783 mac6[3] = ip[2]; 4784 mac6[4] = ip[1]; 4785 mac6[5] = ip[0]; 4786 ether_addr_copy(mac, mac6); 4787 } 4788 } 4789 4790 /** 4791 * irdma_attach_mcast - attach a qp to a multicast group 4792 * @ibqp: ptr to qp 4793 * @ibgid: pointer to global ID 4794 * @lid: local ID 4795 * 4796 * returns error status 4797 */ 4798 static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 4799 { 4800 struct irdma_qp *iwqp = to_iwqp(ibqp); 4801 struct irdma_device *iwdev = iwqp->iwdev; 4802 struct irdma_pci_f *rf = iwdev->rf; 4803 struct mc_table_list *mc_qht_elem; 4804 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 4805 unsigned long flags; 4806 u32 ip_addr[4] = {}; 4807 u32 mgn; 4808 u32 no_mgs; 4809 int ret = 0; 4810 bool ipv4; 4811 u16 vlan_id; 4812 union irdma_sockaddr sgid_addr; 4813 unsigned char dmac[ETH_ALEN]; 4814 4815 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 4816 4817 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { 4818 irdma_copy_ip_ntohl(ip_addr, 4819 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4820 irdma_get_vlan_mac_ipv6(ip_addr, &vlan_id, NULL); 4821 ipv4 = false; 4822 ibdev_dbg(&iwdev->ibdev, 4823 "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, 4824 ip_addr); 4825 irdma_mcast_mac(ip_addr, dmac, false); 4826 } else { 4827 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4828 ipv4 = true; 4829 vlan_id = irdma_get_vlan_ipv4(ip_addr); 4830 irdma_mcast_mac(ip_addr, dmac, true); 4831 ibdev_dbg(&iwdev->ibdev, 4832 "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n", 4833 ibqp->qp_num, ip_addr, dmac); 4834 } 4835 4836 spin_lock_irqsave(&rf->qh_list_lock, flags); 4837 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 4838 if (!mc_qht_elem) { 4839 struct irdma_dma_mem *dma_mem_mc; 4840 4841 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4842 mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL); 4843 if (!mc_qht_elem) 4844 return -ENOMEM; 4845 4846 mc_qht_elem->mc_info.ipv4_valid = ipv4; 4847 memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, 4848 sizeof(mc_qht_elem->mc_info.dest_ip)); 4849 ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, 4850 &mgn, &rf->next_mcg); 4851 if (ret) { 4852 kfree(mc_qht_elem); 4853 return -ENOMEM; 4854 } 4855 4856 mc_qht_elem->mc_info.mgn = mgn; 4857 dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; 4858 dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX, 4859 IRDMA_HW_PAGE_SIZE); 4860 dma_mem_mc->va = dma_alloc_coherent(rf->hw.device, 4861 dma_mem_mc->size, 4862 &dma_mem_mc->pa, 4863 GFP_KERNEL); 4864 if (!dma_mem_mc->va) { 4865 irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); 4866 kfree(mc_qht_elem); 4867 return -ENOMEM; 4868 } 4869 4870 mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; 4871 memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, 4872 sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); 4873 mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; 4874 mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; 4875 if (vlan_id < VLAN_N_VID) 4876 mc_qht_elem->mc_grp_ctx.vlan_valid = true; 4877 mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id; 4878 mc_qht_elem->mc_grp_ctx.qs_handle = 4879 iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; 4880 ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); 4881 4882 spin_lock_irqsave(&rf->qh_list_lock, flags); 4883 mcast_list_add(rf, mc_qht_elem); 4884 } else { 4885 if (mc_qht_elem->mc_grp_ctx.no_of_mgs == 4886 IRDMA_MAX_MGS_PER_CTX) { 4887 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4888 return -ENOMEM; 4889 } 4890 } 4891 4892 mcg_info.qp_id = iwqp->ibqp.qp_num; 4893 no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; 4894 irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4895 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4896 4897 /* Only if there is a change do we need to modify or create */ 4898 if (!no_mgs) { 4899 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4900 IRDMA_OP_MC_CREATE); 4901 } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4902 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4903 IRDMA_OP_MC_MODIFY); 4904 } else { 4905 return 0; 4906 } 4907 4908 if (ret) 4909 goto error; 4910 4911 return 0; 4912 4913 error: 4914 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4915 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4916 mcast_list_del(mc_qht_elem); 4917 dma_free_coherent(rf->hw.device, 4918 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4919 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4920 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4921 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4922 irdma_free_rsrc(rf, rf->allocated_mcgs, 4923 mc_qht_elem->mc_grp_ctx.mg_id); 4924 kfree(mc_qht_elem); 4925 } 4926 4927 return ret; 4928 } 4929 4930 /** 4931 * irdma_detach_mcast - detach a qp from a multicast group 4932 * @ibqp: ptr to qp 4933 * @ibgid: pointer to global ID 4934 * @lid: local ID 4935 * 4936 * returns error status 4937 */ 4938 static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 4939 { 4940 struct irdma_qp *iwqp = to_iwqp(ibqp); 4941 struct irdma_device *iwdev = iwqp->iwdev; 4942 struct irdma_pci_f *rf = iwdev->rf; 4943 u32 ip_addr[4] = {}; 4944 struct mc_table_list *mc_qht_elem; 4945 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 4946 int ret; 4947 unsigned long flags; 4948 union irdma_sockaddr sgid_addr; 4949 4950 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 4951 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) 4952 irdma_copy_ip_ntohl(ip_addr, 4953 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4954 else 4955 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4956 4957 spin_lock_irqsave(&rf->qh_list_lock, flags); 4958 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 4959 if (!mc_qht_elem) { 4960 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4961 ibdev_dbg(&iwdev->ibdev, 4962 "VERBS: address not found MCG\n"); 4963 return 0; 4964 } 4965 4966 mcg_info.qp_id = iwqp->ibqp.qp_num; 4967 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4968 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4969 mcast_list_del(mc_qht_elem); 4970 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4971 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4972 IRDMA_OP_MC_DESTROY); 4973 if (ret) { 4974 ibdev_dbg(&iwdev->ibdev, 4975 "VERBS: failed MC_DESTROY MCG\n"); 4976 spin_lock_irqsave(&rf->qh_list_lock, flags); 4977 mcast_list_add(rf, mc_qht_elem); 4978 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4979 return -EAGAIN; 4980 } 4981 4982 dma_free_coherent(rf->hw.device, 4983 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4984 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4985 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4986 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4987 irdma_free_rsrc(rf, rf->allocated_mcgs, 4988 mc_qht_elem->mc_grp_ctx.mg_id); 4989 kfree(mc_qht_elem); 4990 } else { 4991 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4992 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4993 IRDMA_OP_MC_MODIFY); 4994 if (ret) { 4995 ibdev_dbg(&iwdev->ibdev, 4996 "VERBS: failed Modify MCG\n"); 4997 return ret; 4998 } 4999 } 5000 5001 return 0; 5002 } 5003 5004 static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep) 5005 { 5006 struct irdma_pci_f *rf = iwdev->rf; 5007 int err; 5008 5009 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah->sc_ah.ah_info.ah_idx, 5010 &rf->next_ah); 5011 if (err) 5012 return err; 5013 5014 err = irdma_ah_cqp_op(rf, &ah->sc_ah, IRDMA_OP_AH_CREATE, sleep, 5015 irdma_gsi_ud_qp_ah_cb, &ah->sc_ah); 5016 5017 if (err) { 5018 ibdev_dbg(&iwdev->ibdev, "VERBS: CQP-OP Create AH fail"); 5019 goto err_ah_create; 5020 } 5021 5022 if (!sleep) { 5023 int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD; 5024 5025 do { 5026 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 5027 mdelay(1); 5028 } while (!ah->sc_ah.ah_info.ah_valid && --cnt); 5029 5030 if (!cnt) { 5031 ibdev_dbg(&iwdev->ibdev, "VERBS: CQP create AH timed out"); 5032 err = -ETIMEDOUT; 5033 goto err_ah_create; 5034 } 5035 } 5036 return 0; 5037 5038 err_ah_create: 5039 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx); 5040 5041 return err; 5042 } 5043 5044 static int irdma_setup_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr) 5045 { 5046 struct irdma_pd *pd = to_iwpd(ibah->pd); 5047 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5048 struct rdma_ah_attr *ah_attr = attr->ah_attr; 5049 const struct ib_gid_attr *sgid_attr; 5050 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5051 struct irdma_pci_f *rf = iwdev->rf; 5052 struct irdma_sc_ah *sc_ah; 5053 struct irdma_ah_info *ah_info; 5054 union irdma_sockaddr sgid_addr, dgid_addr; 5055 int err; 5056 u8 dmac[ETH_ALEN]; 5057 5058 ah->pd = pd; 5059 sc_ah = &ah->sc_ah; 5060 sc_ah->ah_info.vsi = &iwdev->vsi; 5061 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 5062 ah->sgid_index = ah_attr->grh.sgid_index; 5063 sgid_attr = ah_attr->grh.sgid_attr; 5064 memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid)); 5065 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid_attr->gid); 5066 rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid); 5067 ah->av.attrs = *ah_attr; 5068 ah->av.net_type = rdma_gid_attr_network_type(sgid_attr); 5069 ah_info = &sc_ah->ah_info; 5070 ah_info->pd_idx = pd->sc_pd.pd_id; 5071 if (ah_attr->ah_flags & IB_AH_GRH) { 5072 ah_info->flow_label = ah_attr->grh.flow_label; 5073 ah_info->hop_ttl = ah_attr->grh.hop_limit; 5074 ah_info->tc_tos = ah_attr->grh.traffic_class; 5075 } 5076 5077 ether_addr_copy(dmac, ah_attr->roce.dmac); 5078 if (ah->av.net_type == RDMA_NETWORK_IPV4) { 5079 ah_info->ipv4_valid = true; 5080 ah_info->dest_ip_addr[0] = 5081 ntohl(dgid_addr.saddr_in.sin_addr.s_addr); 5082 ah_info->src_ip_addr[0] = 5083 ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 5084 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 5085 ah_info->dest_ip_addr[0]); 5086 if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) { 5087 ah_info->do_lpbk = true; 5088 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true); 5089 } 5090 } else { 5091 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 5092 dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 5093 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 5094 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 5095 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 5096 ah_info->dest_ip_addr); 5097 if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr)) { 5098 ah_info->do_lpbk = true; 5099 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false); 5100 } 5101 } 5102 5103 err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag, 5104 ah_info->mac_addr); 5105 if (err) 5106 return err; 5107 5108 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, 5109 ah_info->ipv4_valid, dmac); 5110 5111 if (ah_info->dst_arpindex == -1) 5112 return -EINVAL; 5113 5114 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) 5115 ah_info->vlan_tag = 0; 5116 5117 if (ah_info->vlan_tag < VLAN_N_VID) { 5118 u8 prio = rt_tos2priority(ah_info->tc_tos); 5119 5120 prio = irdma_roce_get_vlan_prio(sgid_attr, prio); 5121 5122 ah_info->vlan_tag |= (u16)prio << VLAN_PRIO_SHIFT; 5123 ah_info->insert_vlan_tag = true; 5124 } 5125 5126 return 0; 5127 } 5128 5129 /** 5130 * irdma_ah_exists - Check for existing identical AH 5131 * @iwdev: irdma device 5132 * @new_ah: AH to check for 5133 * 5134 * returns true if AH is found, false if not found. 5135 */ 5136 static bool irdma_ah_exists(struct irdma_device *iwdev, 5137 struct irdma_ah *new_ah) 5138 { 5139 struct irdma_ah *ah; 5140 u32 key = new_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 5141 new_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 5142 new_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 5143 new_ah->sc_ah.ah_info.dest_ip_addr[3]; 5144 5145 hash_for_each_possible(iwdev->rf->ah_hash_tbl, ah, list, key) { 5146 /* Set ah_valid and ah_id the same so memcmp can work */ 5147 new_ah->sc_ah.ah_info.ah_idx = ah->sc_ah.ah_info.ah_idx; 5148 new_ah->sc_ah.ah_info.ah_valid = ah->sc_ah.ah_info.ah_valid; 5149 if (!memcmp(&ah->sc_ah.ah_info, &new_ah->sc_ah.ah_info, 5150 sizeof(ah->sc_ah.ah_info))) { 5151 refcount_inc(&ah->refcnt); 5152 new_ah->parent_ah = ah; 5153 return true; 5154 } 5155 } 5156 5157 return false; 5158 } 5159 5160 /** 5161 * irdma_destroy_ah - Destroy address handle 5162 * @ibah: pointer to address handle 5163 * @ah_flags: flags for sleepable 5164 */ 5165 static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 5166 { 5167 struct irdma_device *iwdev = to_iwdev(ibah->device); 5168 struct irdma_ah *ah = to_iwah(ibah); 5169 5170 if ((ah_flags & RDMA_DESTROY_AH_SLEEPABLE) && ah->parent_ah) { 5171 mutex_lock(&iwdev->rf->ah_tbl_lock); 5172 if (!refcount_dec_and_test(&ah->parent_ah->refcnt)) { 5173 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5174 return 0; 5175 } 5176 hash_del(&ah->parent_ah->list); 5177 kfree(ah->parent_ah); 5178 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5179 } 5180 5181 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 5182 false, NULL, ah); 5183 5184 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 5185 ah->sc_ah.ah_info.ah_idx); 5186 5187 return 0; 5188 } 5189 5190 /** 5191 * irdma_create_user_ah - create user address handle 5192 * @ibah: address handle 5193 * @attr: address handle attributes 5194 * @udata: User data 5195 * 5196 * returns 0 on success, error otherwise 5197 */ 5198 static int irdma_create_user_ah(struct ib_ah *ibah, 5199 struct rdma_ah_init_attr *attr, 5200 struct ib_udata *udata) 5201 { 5202 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) 5203 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5204 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5205 struct irdma_create_ah_resp uresp; 5206 struct irdma_ah *parent_ah; 5207 int err; 5208 5209 if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) 5210 return -EINVAL; 5211 5212 err = irdma_setup_ah(ibah, attr); 5213 if (err) 5214 return err; 5215 mutex_lock(&iwdev->rf->ah_tbl_lock); 5216 if (!irdma_ah_exists(iwdev, ah)) { 5217 err = irdma_create_hw_ah(iwdev, ah, true); 5218 if (err) { 5219 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5220 return err; 5221 } 5222 /* Add new AH to list */ 5223 parent_ah = kmemdup(ah, sizeof(*ah), GFP_KERNEL); 5224 if (parent_ah) { 5225 u32 key = parent_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 5226 parent_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 5227 parent_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 5228 parent_ah->sc_ah.ah_info.dest_ip_addr[3]; 5229 5230 ah->parent_ah = parent_ah; 5231 hash_add(iwdev->rf->ah_hash_tbl, &parent_ah->list, key); 5232 refcount_set(&parent_ah->refcnt, 1); 5233 } 5234 } 5235 mutex_unlock(&iwdev->rf->ah_tbl_lock); 5236 5237 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 5238 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); 5239 if (err) 5240 irdma_destroy_ah(ibah, attr->flags); 5241 5242 return err; 5243 } 5244 5245 /** 5246 * irdma_create_ah - create address handle 5247 * @ibah: address handle 5248 * @attr: address handle attributes 5249 * @udata: NULL 5250 * 5251 * returns 0 on success, error otherwise 5252 */ 5253 static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr, 5254 struct ib_udata *udata) 5255 { 5256 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 5257 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 5258 int err; 5259 5260 err = irdma_setup_ah(ibah, attr); 5261 if (err) 5262 return err; 5263 err = irdma_create_hw_ah(iwdev, ah, attr->flags & RDMA_CREATE_AH_SLEEPABLE); 5264 5265 return err; 5266 } 5267 5268 /** 5269 * irdma_query_ah - Query address handle 5270 * @ibah: pointer to address handle 5271 * @ah_attr: address handle attributes 5272 */ 5273 static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 5274 { 5275 struct irdma_ah *ah = to_iwah(ibah); 5276 5277 memset(ah_attr, 0, sizeof(*ah_attr)); 5278 if (ah->av.attrs.ah_flags & IB_AH_GRH) { 5279 ah_attr->ah_flags = IB_AH_GRH; 5280 ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; 5281 ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; 5282 ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; 5283 ah_attr->grh.sgid_index = ah->sgid_index; 5284 memcpy(&ah_attr->grh.dgid, &ah->dgid, 5285 sizeof(ah_attr->grh.dgid)); 5286 } 5287 5288 return 0; 5289 } 5290 5291 static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, 5292 u32 port_num) 5293 { 5294 return IB_LINK_LAYER_ETHERNET; 5295 } 5296 5297 static const struct ib_device_ops irdma_gen1_dev_ops = { 5298 .dealloc_driver = irdma_ib_dealloc_device, 5299 }; 5300 5301 static const struct ib_device_ops irdma_gen3_dev_ops = { 5302 .alloc_mw = irdma_alloc_mw, 5303 .create_srq = irdma_create_srq, 5304 .dealloc_mw = irdma_dealloc_mw, 5305 .destroy_srq = irdma_destroy_srq, 5306 .modify_srq = irdma_modify_srq, 5307 .post_srq_recv = irdma_post_srq_recv, 5308 .query_srq = irdma_query_srq, 5309 }; 5310 5311 static const struct ib_device_ops irdma_roce_dev_ops = { 5312 .attach_mcast = irdma_attach_mcast, 5313 .create_ah = irdma_create_ah, 5314 .create_user_ah = irdma_create_user_ah, 5315 .destroy_ah = irdma_destroy_ah, 5316 .detach_mcast = irdma_detach_mcast, 5317 .get_link_layer = irdma_get_link_layer, 5318 .get_port_immutable = irdma_roce_port_immutable, 5319 .modify_qp = irdma_modify_qp_roce, 5320 .query_ah = irdma_query_ah, 5321 .query_pkey = irdma_query_pkey, 5322 }; 5323 5324 static const struct ib_device_ops irdma_iw_dev_ops = { 5325 .get_port_immutable = irdma_iw_port_immutable, 5326 .iw_accept = irdma_accept, 5327 .iw_add_ref = irdma_qp_add_ref, 5328 .iw_connect = irdma_connect, 5329 .iw_create_listen = irdma_create_listen, 5330 .iw_destroy_listen = irdma_destroy_listen, 5331 .iw_get_qp = irdma_get_qp, 5332 .iw_reject = irdma_reject, 5333 .iw_rem_ref = irdma_qp_rem_ref, 5334 .modify_qp = irdma_modify_qp, 5335 .query_gid = irdma_query_gid, 5336 }; 5337 5338 static const struct ib_device_ops irdma_dev_ops = { 5339 .owner = THIS_MODULE, 5340 .driver_id = RDMA_DRIVER_IRDMA, 5341 .uverbs_abi_ver = IRDMA_ABI_VER, 5342 5343 .alloc_hw_port_stats = irdma_alloc_hw_port_stats, 5344 .alloc_mr = irdma_alloc_mr, 5345 .alloc_pd = irdma_alloc_pd, 5346 .alloc_ucontext = irdma_alloc_ucontext, 5347 .create_cq = irdma_create_cq, 5348 .create_qp = irdma_create_qp, 5349 .dealloc_driver = irdma_ib_dealloc_device, 5350 .dealloc_mw = irdma_dealloc_mw, 5351 .dealloc_pd = irdma_dealloc_pd, 5352 .dealloc_ucontext = irdma_dealloc_ucontext, 5353 .dereg_mr = irdma_dereg_mr, 5354 .destroy_cq = irdma_destroy_cq, 5355 .destroy_qp = irdma_destroy_qp, 5356 .disassociate_ucontext = irdma_disassociate_ucontext, 5357 .get_dev_fw_str = irdma_get_dev_fw_str, 5358 .get_dma_mr = irdma_get_dma_mr, 5359 .get_hw_stats = irdma_get_hw_stats, 5360 .map_mr_sg = irdma_map_mr_sg, 5361 .mmap = irdma_mmap, 5362 .mmap_free = irdma_mmap_free, 5363 .poll_cq = irdma_poll_cq, 5364 .post_recv = irdma_post_recv, 5365 .post_send = irdma_post_send, 5366 .query_device = irdma_query_device, 5367 .query_port = irdma_query_port, 5368 .query_qp = irdma_query_qp, 5369 .reg_user_mr = irdma_reg_user_mr, 5370 .reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf, 5371 .rereg_user_mr = irdma_rereg_user_mr, 5372 .req_notify_cq = irdma_req_notify_cq, 5373 .resize_cq = irdma_resize_cq, 5374 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd), 5375 INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext), 5376 INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), 5377 INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), 5378 INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), 5379 INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp), 5380 INIT_RDMA_OBJ_SIZE(ib_srq, irdma_srq, ibsrq), 5381 }; 5382 5383 /** 5384 * irdma_init_roce_device - initialization of roce rdma device 5385 * @iwdev: irdma device 5386 */ 5387 static void irdma_init_roce_device(struct irdma_device *iwdev) 5388 { 5389 iwdev->ibdev.node_type = RDMA_NODE_IB_CA; 5390 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 5391 iwdev->netdev->dev_addr); 5392 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); 5393 } 5394 5395 /** 5396 * irdma_init_iw_device - initialization of iwarp rdma device 5397 * @iwdev: irdma device 5398 */ 5399 static void irdma_init_iw_device(struct irdma_device *iwdev) 5400 { 5401 struct net_device *netdev = iwdev->netdev; 5402 5403 iwdev->ibdev.node_type = RDMA_NODE_RNIC; 5404 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 5405 netdev->dev_addr); 5406 memcpy(iwdev->ibdev.iw_ifname, netdev->name, 5407 sizeof(iwdev->ibdev.iw_ifname)); 5408 ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops); 5409 } 5410 5411 /** 5412 * irdma_init_rdma_device - initialization of rdma device 5413 * @iwdev: irdma device 5414 */ 5415 static void irdma_init_rdma_device(struct irdma_device *iwdev) 5416 { 5417 struct pci_dev *pcidev = iwdev->rf->pcidev; 5418 5419 if (iwdev->roce_mode) 5420 irdma_init_roce_device(iwdev); 5421 else 5422 irdma_init_iw_device(iwdev); 5423 5424 iwdev->ibdev.phys_port_cnt = 1; 5425 iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; 5426 iwdev->ibdev.dev.parent = &pcidev->dev; 5427 ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops); 5428 if (iwdev->rf->rdma_ver == IRDMA_GEN_1) 5429 ib_set_device_ops(&iwdev->ibdev, &irdma_gen1_dev_ops); 5430 if (iwdev->rf->rdma_ver >= IRDMA_GEN_3) 5431 ib_set_device_ops(&iwdev->ibdev, &irdma_gen3_dev_ops); 5432 } 5433 5434 /** 5435 * irdma_port_ibevent - indicate port event 5436 * @iwdev: irdma device 5437 */ 5438 void irdma_port_ibevent(struct irdma_device *iwdev) 5439 { 5440 struct ib_event event; 5441 5442 event.device = &iwdev->ibdev; 5443 event.element.port_num = 1; 5444 event.event = 5445 iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; 5446 ib_dispatch_event(&event); 5447 } 5448 5449 /** 5450 * irdma_ib_unregister_device - unregister rdma device from IB 5451 * core 5452 * @iwdev: irdma device 5453 */ 5454 void irdma_ib_unregister_device(struct irdma_device *iwdev) 5455 { 5456 iwdev->iw_status = 0; 5457 irdma_port_ibevent(iwdev); 5458 ib_unregister_device(&iwdev->ibdev); 5459 } 5460 5461 /** 5462 * irdma_ib_register_device - register irdma device to IB core 5463 * @iwdev: irdma device 5464 */ 5465 int irdma_ib_register_device(struct irdma_device *iwdev) 5466 { 5467 int ret; 5468 5469 irdma_init_rdma_device(iwdev); 5470 5471 ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1); 5472 if (ret) 5473 goto error; 5474 dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX); 5475 ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device); 5476 if (ret) 5477 goto error; 5478 5479 iwdev->iw_status = 1; 5480 irdma_port_ibevent(iwdev); 5481 5482 return 0; 5483 5484 error: 5485 if (ret) 5486 ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n"); 5487 5488 return ret; 5489 } 5490 5491 /** 5492 * irdma_ib_dealloc_device 5493 * @ibdev: ib device 5494 * 5495 * callback from ibdev dealloc_driver to deallocate resources 5496 * unber irdma device 5497 */ 5498 void irdma_ib_dealloc_device(struct ib_device *ibdev) 5499 { 5500 struct irdma_device *iwdev = to_iwdev(ibdev); 5501 5502 irdma_rt_deinit_hw(iwdev); 5503 if (!iwdev->is_vport) { 5504 irdma_ctrl_deinit_hw(iwdev->rf); 5505 if (iwdev->rf->vchnl_wq) 5506 destroy_workqueue(iwdev->rf->vchnl_wq); 5507 } 5508 } 5509