1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_query_device - get device attributes 7 * @ibdev: device pointer from stack 8 * @props: returning device attributes 9 * @udata: user data 10 */ 11 static int irdma_query_device(struct ib_device *ibdev, 12 struct ib_device_attr *props, 13 struct ib_udata *udata) 14 { 15 struct irdma_device *iwdev = to_iwdev(ibdev); 16 struct irdma_pci_f *rf = iwdev->rf; 17 struct pci_dev *pcidev = iwdev->rf->pcidev; 18 struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; 19 20 if (udata->inlen || udata->outlen) 21 return -EINVAL; 22 23 memset(props, 0, sizeof(*props)); 24 ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr); 25 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | 26 irdma_fw_minor_ver(&rf->sc_dev); 27 props->device_cap_flags = iwdev->device_cap_flags; 28 props->vendor_id = pcidev->vendor; 29 props->vendor_part_id = pcidev->device; 30 31 props->hw_ver = rf->pcidev->revision; 32 props->page_size_cap = SZ_4K | SZ_2M | SZ_1G; 33 props->max_mr_size = hw_attrs->max_mr_size; 34 props->max_qp = rf->max_qp - rf->used_qps; 35 props->max_qp_wr = hw_attrs->max_qp_wr; 36 props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 37 props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 38 props->max_cq = rf->max_cq - rf->used_cqs; 39 props->max_cqe = rf->max_cqe; 40 props->max_mr = rf->max_mr - rf->used_mrs; 41 props->max_mw = props->max_mr; 42 props->max_pd = rf->max_pd - rf->used_pds; 43 props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; 44 props->max_qp_rd_atom = hw_attrs->max_hw_ird; 45 props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; 46 if (rdma_protocol_roce(ibdev, 1)) 47 props->max_pkeys = IRDMA_PKEY_TBL_SZ; 48 props->max_ah = rf->max_ah; 49 props->max_mcast_grp = rf->max_mcg; 50 props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; 51 props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; 52 props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; 53 #define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff 54 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2) 55 props->timestamp_mask = HCA_CLOCK_TIMESTAMP_MASK; 56 57 return 0; 58 } 59 60 /** 61 * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed 62 * @link_speed: netdev phy link speed 63 * @active_speed: IB port speed 64 * @active_width: IB port width 65 */ 66 static void irdma_get_eth_speed_and_width(u32 link_speed, u16 *active_speed, 67 u8 *active_width) 68 { 69 if (link_speed <= SPEED_1000) { 70 *active_width = IB_WIDTH_1X; 71 *active_speed = IB_SPEED_SDR; 72 } else if (link_speed <= SPEED_10000) { 73 *active_width = IB_WIDTH_1X; 74 *active_speed = IB_SPEED_FDR10; 75 } else if (link_speed <= SPEED_20000) { 76 *active_width = IB_WIDTH_4X; 77 *active_speed = IB_SPEED_DDR; 78 } else if (link_speed <= SPEED_25000) { 79 *active_width = IB_WIDTH_1X; 80 *active_speed = IB_SPEED_EDR; 81 } else if (link_speed <= SPEED_40000) { 82 *active_width = IB_WIDTH_4X; 83 *active_speed = IB_SPEED_FDR10; 84 } else { 85 *active_width = IB_WIDTH_4X; 86 *active_speed = IB_SPEED_EDR; 87 } 88 } 89 90 /** 91 * irdma_query_port - get port attributes 92 * @ibdev: device pointer from stack 93 * @port: port number for query 94 * @props: returning device attributes 95 */ 96 static int irdma_query_port(struct ib_device *ibdev, u32 port, 97 struct ib_port_attr *props) 98 { 99 struct irdma_device *iwdev = to_iwdev(ibdev); 100 struct net_device *netdev = iwdev->netdev; 101 102 /* no need to zero out pros here. done by caller */ 103 104 props->max_mtu = IB_MTU_4096; 105 props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); 106 props->lid = 1; 107 props->lmc = 0; 108 props->sm_lid = 0; 109 props->sm_sl = 0; 110 if (netif_carrier_ok(netdev) && netif_running(netdev)) { 111 props->state = IB_PORT_ACTIVE; 112 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 113 } else { 114 props->state = IB_PORT_DOWN; 115 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 116 } 117 irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, 118 &props->active_width); 119 120 if (rdma_protocol_roce(ibdev, 1)) { 121 props->gid_tbl_len = 32; 122 props->ip_gids = true; 123 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 124 } else { 125 props->gid_tbl_len = 1; 126 } 127 props->qkey_viol_cntr = 0; 128 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 129 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 130 131 return 0; 132 } 133 134 /** 135 * irdma_disassociate_ucontext - Disassociate user context 136 * @context: ib user context 137 */ 138 static void irdma_disassociate_ucontext(struct ib_ucontext *context) 139 { 140 } 141 142 static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, 143 struct vm_area_struct *vma) 144 { 145 u64 pfn; 146 147 if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) 148 return -EINVAL; 149 150 vma->vm_private_data = ucontext; 151 pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + 152 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 153 154 return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, 155 pgprot_noncached(vma->vm_page_prot), NULL); 156 } 157 158 static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 159 { 160 struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); 161 162 kfree(entry); 163 } 164 165 static struct rdma_user_mmap_entry* 166 irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, 167 enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) 168 { 169 struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); 170 int ret; 171 172 if (!entry) 173 return NULL; 174 175 entry->bar_offset = bar_offset; 176 entry->mmap_flag = mmap_flag; 177 178 ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, 179 &entry->rdma_entry, PAGE_SIZE); 180 if (ret) { 181 kfree(entry); 182 return NULL; 183 } 184 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 185 186 return &entry->rdma_entry; 187 } 188 189 /** 190 * irdma_mmap - user memory map 191 * @context: context created during alloc 192 * @vma: kernel info for user memory map 193 */ 194 static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 195 { 196 struct rdma_user_mmap_entry *rdma_entry; 197 struct irdma_user_mmap_entry *entry; 198 struct irdma_ucontext *ucontext; 199 u64 pfn; 200 int ret; 201 202 ucontext = to_ucontext(context); 203 204 /* Legacy support for libi40iw with hard-coded mmap key */ 205 if (ucontext->legacy_mode) 206 return irdma_mmap_legacy(ucontext, vma); 207 208 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 209 if (!rdma_entry) { 210 ibdev_dbg(&ucontext->iwdev->ibdev, 211 "VERBS: pgoff[0x%lx] does not have valid entry\n", 212 vma->vm_pgoff); 213 return -EINVAL; 214 } 215 216 entry = to_irdma_mmap_entry(rdma_entry); 217 ibdev_dbg(&ucontext->iwdev->ibdev, 218 "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n", 219 entry->bar_offset, entry->mmap_flag); 220 221 pfn = (entry->bar_offset + 222 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 223 224 switch (entry->mmap_flag) { 225 case IRDMA_MMAP_IO_NC: 226 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 227 pgprot_noncached(vma->vm_page_prot), 228 rdma_entry); 229 break; 230 case IRDMA_MMAP_IO_WC: 231 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 232 pgprot_writecombine(vma->vm_page_prot), 233 rdma_entry); 234 break; 235 default: 236 ret = -EINVAL; 237 } 238 239 if (ret) 240 ibdev_dbg(&ucontext->iwdev->ibdev, 241 "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n", 242 entry->bar_offset, entry->mmap_flag, ret); 243 rdma_user_mmap_entry_put(rdma_entry); 244 245 return ret; 246 } 247 248 /** 249 * irdma_alloc_push_page - allocate a push page for qp 250 * @iwqp: qp pointer 251 */ 252 static void irdma_alloc_push_page(struct irdma_qp *iwqp) 253 { 254 struct irdma_cqp_request *cqp_request; 255 struct cqp_cmds_info *cqp_info; 256 struct irdma_device *iwdev = iwqp->iwdev; 257 struct irdma_sc_qp *qp = &iwqp->sc_qp; 258 enum irdma_status_code status; 259 260 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 261 if (!cqp_request) 262 return; 263 264 cqp_info = &cqp_request->info; 265 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 266 cqp_info->post_sq = 1; 267 cqp_info->in.u.manage_push_page.info.push_idx = 0; 268 cqp_info->in.u.manage_push_page.info.qs_handle = 269 qp->vsi->qos[qp->user_pri].qs_handle; 270 cqp_info->in.u.manage_push_page.info.free_page = 0; 271 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 272 cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; 273 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 274 275 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 276 if (!status && cqp_request->compl_info.op_ret_val < 277 iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { 278 qp->push_idx = cqp_request->compl_info.op_ret_val; 279 qp->push_offset = 0; 280 } 281 282 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 283 } 284 285 /** 286 * irdma_alloc_ucontext - Allocate the user context data structure 287 * @uctx: uverbs context pointer 288 * @udata: user data 289 * 290 * This keeps track of all objects associated with a particular 291 * user-mode client. 292 */ 293 static int irdma_alloc_ucontext(struct ib_ucontext *uctx, 294 struct ib_udata *udata) 295 { 296 struct ib_device *ibdev = uctx->device; 297 struct irdma_device *iwdev = to_iwdev(ibdev); 298 struct irdma_alloc_ucontext_req req; 299 struct irdma_alloc_ucontext_resp uresp = {}; 300 struct irdma_ucontext *ucontext = to_ucontext(uctx); 301 struct irdma_uk_attrs *uk_attrs; 302 303 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 304 return -EINVAL; 305 306 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 307 goto ver_error; 308 309 ucontext->iwdev = iwdev; 310 ucontext->abi_ver = req.userspace_ver; 311 312 uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 313 /* GEN_1 legacy support with libi40iw */ 314 if (udata->outlen < sizeof(uresp)) { 315 if (uk_attrs->hw_rev != IRDMA_GEN_1) 316 return -EOPNOTSUPP; 317 318 ucontext->legacy_mode = true; 319 uresp.max_qps = iwdev->rf->max_qp; 320 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 321 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 322 uresp.kernel_ver = req.userspace_ver; 323 if (ib_copy_to_udata(udata, &uresp, 324 min(sizeof(uresp), udata->outlen))) 325 return -EFAULT; 326 } else { 327 u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 328 329 ucontext->db_mmap_entry = 330 irdma_user_mmap_entry_insert(ucontext, bar_off, 331 IRDMA_MMAP_IO_NC, 332 &uresp.db_mmap_key); 333 if (!ucontext->db_mmap_entry) 334 return -ENOMEM; 335 336 uresp.kernel_ver = IRDMA_ABI_VER; 337 uresp.feature_flags = uk_attrs->feature_flags; 338 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 339 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 340 uresp.max_hw_inline = uk_attrs->max_hw_inline; 341 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 342 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 343 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 344 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 345 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 346 uresp.hw_rev = uk_attrs->hw_rev; 347 if (ib_copy_to_udata(udata, &uresp, 348 min(sizeof(uresp), udata->outlen))) { 349 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 350 return -EFAULT; 351 } 352 } 353 354 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 355 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 356 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 357 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 358 359 return 0; 360 361 ver_error: 362 ibdev_err(&iwdev->ibdev, 363 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 364 req.userspace_ver, IRDMA_ABI_VER); 365 return -EINVAL; 366 } 367 368 /** 369 * irdma_dealloc_ucontext - deallocate the user context data structure 370 * @context: user context created during alloc 371 */ 372 static void irdma_dealloc_ucontext(struct ib_ucontext *context) 373 { 374 struct irdma_ucontext *ucontext = to_ucontext(context); 375 376 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 377 } 378 379 /** 380 * irdma_alloc_pd - allocate protection domain 381 * @pd: PD pointer 382 * @udata: user data 383 */ 384 static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 385 { 386 struct irdma_pd *iwpd = to_iwpd(pd); 387 struct irdma_device *iwdev = to_iwdev(pd->device); 388 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 389 struct irdma_pci_f *rf = iwdev->rf; 390 struct irdma_alloc_pd_resp uresp = {}; 391 struct irdma_sc_pd *sc_pd; 392 u32 pd_id = 0; 393 int err; 394 395 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 396 &rf->next_pd); 397 if (err) 398 return err; 399 400 sc_pd = &iwpd->sc_pd; 401 if (udata) { 402 struct irdma_ucontext *ucontext = 403 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 404 ibucontext); 405 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 406 uresp.pd_id = pd_id; 407 if (ib_copy_to_udata(udata, &uresp, 408 min(sizeof(uresp), udata->outlen))) { 409 err = -EFAULT; 410 goto error; 411 } 412 } else { 413 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 414 } 415 416 return 0; 417 error: 418 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 419 420 return err; 421 } 422 423 /** 424 * irdma_dealloc_pd - deallocate pd 425 * @ibpd: ptr of pd to be deallocated 426 * @udata: user data 427 */ 428 static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 429 { 430 struct irdma_pd *iwpd = to_iwpd(ibpd); 431 struct irdma_device *iwdev = to_iwdev(ibpd->device); 432 433 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 434 435 return 0; 436 } 437 438 /** 439 * irdma_get_pbl - Retrieve pbl from a list given a virtual 440 * address 441 * @va: user virtual address 442 * @pbl_list: pbl list to search in (QP's or CQ's) 443 */ 444 static struct irdma_pbl *irdma_get_pbl(unsigned long va, 445 struct list_head *pbl_list) 446 { 447 struct irdma_pbl *iwpbl; 448 449 list_for_each_entry (iwpbl, pbl_list, list) { 450 if (iwpbl->user_base == va) { 451 list_del(&iwpbl->list); 452 iwpbl->on_list = false; 453 return iwpbl; 454 } 455 } 456 457 return NULL; 458 } 459 460 /** 461 * irdma_clean_cqes - clean cq entries for qp 462 * @iwqp: qp ptr (user or kernel) 463 * @iwcq: cq ptr 464 */ 465 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) 466 { 467 struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; 468 unsigned long flags; 469 470 spin_lock_irqsave(&iwcq->lock, flags); 471 irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); 472 spin_unlock_irqrestore(&iwcq->lock, flags); 473 } 474 475 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) 476 { 477 if (iwqp->push_db_mmap_entry) { 478 rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); 479 iwqp->push_db_mmap_entry = NULL; 480 } 481 if (iwqp->push_wqe_mmap_entry) { 482 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 483 iwqp->push_wqe_mmap_entry = NULL; 484 } 485 } 486 487 static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, 488 struct irdma_qp *iwqp, 489 u64 *push_wqe_mmap_key, 490 u64 *push_db_mmap_key) 491 { 492 struct irdma_device *iwdev = ucontext->iwdev; 493 u64 rsvd, bar_off; 494 495 rsvd = IRDMA_PF_BAR_RSVD; 496 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 497 /* skip over db page */ 498 bar_off += IRDMA_HW_PAGE_SIZE; 499 /* push wqe page */ 500 bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; 501 iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 502 bar_off, IRDMA_MMAP_IO_WC, 503 push_wqe_mmap_key); 504 if (!iwqp->push_wqe_mmap_entry) 505 return -ENOMEM; 506 507 /* push doorbell page */ 508 bar_off += IRDMA_HW_PAGE_SIZE; 509 iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 510 bar_off, IRDMA_MMAP_IO_NC, 511 push_db_mmap_key); 512 if (!iwqp->push_db_mmap_entry) { 513 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 514 return -ENOMEM; 515 } 516 517 return 0; 518 } 519 520 /** 521 * irdma_destroy_qp - destroy qp 522 * @ibqp: qp's ib pointer also to get to device's qp address 523 * @udata: user data 524 */ 525 static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 526 { 527 struct irdma_qp *iwqp = to_iwqp(ibqp); 528 struct irdma_device *iwdev = iwqp->iwdev; 529 530 iwqp->sc_qp.qp_uk.destroy_pending = true; 531 532 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) 533 irdma_modify_qp_to_err(&iwqp->sc_qp); 534 535 irdma_qp_rem_ref(&iwqp->ibqp); 536 wait_for_completion(&iwqp->free_qp); 537 irdma_free_lsmm_rsrc(iwqp); 538 if (!iwdev->reset) 539 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); 540 541 if (!iwqp->user_mode) { 542 if (iwqp->iwscq) { 543 irdma_clean_cqes(iwqp, iwqp->iwscq); 544 if (iwqp->iwrcq != iwqp->iwscq) 545 irdma_clean_cqes(iwqp, iwqp->iwrcq); 546 } 547 } 548 irdma_remove_push_mmap_entries(iwqp); 549 irdma_free_qp_rsrc(iwqp); 550 551 return 0; 552 } 553 554 /** 555 * irdma_setup_virt_qp - setup for allocation of virtual qp 556 * @iwdev: irdma device 557 * @iwqp: qp ptr 558 * @init_info: initialize info to return 559 */ 560 static void irdma_setup_virt_qp(struct irdma_device *iwdev, 561 struct irdma_qp *iwqp, 562 struct irdma_qp_init_info *init_info) 563 { 564 struct irdma_pbl *iwpbl = iwqp->iwpbl; 565 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 566 567 iwqp->page = qpmr->sq_page; 568 init_info->shadow_area_pa = qpmr->shadow; 569 if (iwpbl->pbl_allocated) { 570 init_info->virtual_map = true; 571 init_info->sq_pa = qpmr->sq_pbl.idx; 572 init_info->rq_pa = qpmr->rq_pbl.idx; 573 } else { 574 init_info->sq_pa = qpmr->sq_pbl.addr; 575 init_info->rq_pa = qpmr->rq_pbl.addr; 576 } 577 } 578 579 /** 580 * irdma_setup_kmode_qp - setup initialization for kernel mode qp 581 * @iwdev: iwarp device 582 * @iwqp: qp ptr (user or kernel) 583 * @info: initialize info to return 584 * @init_attr: Initial QP create attributes 585 */ 586 static int irdma_setup_kmode_qp(struct irdma_device *iwdev, 587 struct irdma_qp *iwqp, 588 struct irdma_qp_init_info *info, 589 struct ib_qp_init_attr *init_attr) 590 { 591 struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; 592 u32 sqdepth, rqdepth; 593 u8 sqshift, rqshift; 594 u32 size; 595 enum irdma_status_code status; 596 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 597 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 598 599 irdma_get_wqe_shift(uk_attrs, 600 uk_attrs->hw_rev >= IRDMA_GEN_2 ? ukinfo->max_sq_frag_cnt + 1 : 601 ukinfo->max_sq_frag_cnt, 602 ukinfo->max_inline_data, &sqshift); 603 status = irdma_get_sqdepth(uk_attrs, ukinfo->sq_size, sqshift, 604 &sqdepth); 605 if (status) 606 return -ENOMEM; 607 608 if (uk_attrs->hw_rev == IRDMA_GEN_1) 609 rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; 610 else 611 irdma_get_wqe_shift(uk_attrs, ukinfo->max_rq_frag_cnt, 0, 612 &rqshift); 613 614 status = irdma_get_rqdepth(uk_attrs, ukinfo->rq_size, rqshift, 615 &rqdepth); 616 if (status) 617 return -ENOMEM; 618 619 iwqp->kqp.sq_wrid_mem = 620 kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL); 621 if (!iwqp->kqp.sq_wrid_mem) 622 return -ENOMEM; 623 624 iwqp->kqp.rq_wrid_mem = 625 kcalloc(rqdepth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL); 626 if (!iwqp->kqp.rq_wrid_mem) { 627 kfree(iwqp->kqp.sq_wrid_mem); 628 iwqp->kqp.sq_wrid_mem = NULL; 629 return -ENOMEM; 630 } 631 632 ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; 633 ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; 634 635 size = (sqdepth + rqdepth) * IRDMA_QP_WQE_MIN_SIZE; 636 size += (IRDMA_SHADOW_AREA_SIZE << 3); 637 638 mem->size = ALIGN(size, 256); 639 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 640 &mem->pa, GFP_KERNEL); 641 if (!mem->va) { 642 kfree(iwqp->kqp.sq_wrid_mem); 643 iwqp->kqp.sq_wrid_mem = NULL; 644 kfree(iwqp->kqp.rq_wrid_mem); 645 iwqp->kqp.rq_wrid_mem = NULL; 646 return -ENOMEM; 647 } 648 649 ukinfo->sq = mem->va; 650 info->sq_pa = mem->pa; 651 ukinfo->rq = &ukinfo->sq[sqdepth]; 652 info->rq_pa = info->sq_pa + (sqdepth * IRDMA_QP_WQE_MIN_SIZE); 653 ukinfo->shadow_area = ukinfo->rq[rqdepth].elem; 654 info->shadow_area_pa = info->rq_pa + (rqdepth * IRDMA_QP_WQE_MIN_SIZE); 655 ukinfo->sq_size = sqdepth >> sqshift; 656 ukinfo->rq_size = rqdepth >> rqshift; 657 ukinfo->qp_id = iwqp->ibqp.qp_num; 658 659 init_attr->cap.max_send_wr = (sqdepth - IRDMA_SQ_RSVD) >> sqshift; 660 init_attr->cap.max_recv_wr = (rqdepth - IRDMA_RQ_RSVD) >> rqshift; 661 662 return 0; 663 } 664 665 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) 666 { 667 struct irdma_pci_f *rf = iwqp->iwdev->rf; 668 struct irdma_cqp_request *cqp_request; 669 struct cqp_cmds_info *cqp_info; 670 struct irdma_create_qp_info *qp_info; 671 enum irdma_status_code status; 672 673 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 674 if (!cqp_request) 675 return -ENOMEM; 676 677 cqp_info = &cqp_request->info; 678 qp_info = &cqp_request->info.in.u.qp_create.info; 679 memset(qp_info, 0, sizeof(*qp_info)); 680 qp_info->mac_valid = true; 681 qp_info->cq_num_valid = true; 682 qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; 683 684 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 685 cqp_info->post_sq = 1; 686 cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; 687 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 688 status = irdma_handle_cqp_op(rf, cqp_request); 689 irdma_put_cqp_request(&rf->cqp, cqp_request); 690 691 return status ? -ENOMEM : 0; 692 } 693 694 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 695 struct irdma_qp_host_ctx_info *ctx_info) 696 { 697 struct irdma_device *iwdev = iwqp->iwdev; 698 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 699 struct irdma_roce_offload_info *roce_info; 700 struct irdma_udp_offload_info *udp_info; 701 702 udp_info = &iwqp->udp_info; 703 udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); 704 udp_info->cwnd = iwdev->roce_cwnd; 705 udp_info->rexmit_thresh = 2; 706 udp_info->rnr_nak_thresh = 2; 707 udp_info->src_port = 0xc000; 708 udp_info->dst_port = ROCE_V2_UDP_DPORT; 709 roce_info = &iwqp->roce_info; 710 ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr); 711 712 roce_info->rd_en = true; 713 roce_info->wr_rdresp_en = true; 714 roce_info->bind_en = true; 715 roce_info->dcqcn_en = false; 716 roce_info->rtomin = 5; 717 718 roce_info->ack_credits = iwdev->roce_ackcreds; 719 roce_info->ird_size = dev->hw_attrs.max_hw_ird; 720 roce_info->ord_size = dev->hw_attrs.max_hw_ord; 721 722 if (!iwqp->user_mode) { 723 roce_info->priv_mode_en = true; 724 roce_info->fast_reg_en = true; 725 roce_info->udprivcq_en = true; 726 } 727 roce_info->roce_tver = 0; 728 729 ctx_info->roce_info = &iwqp->roce_info; 730 ctx_info->udp_info = &iwqp->udp_info; 731 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 732 } 733 734 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 735 struct irdma_qp_host_ctx_info *ctx_info) 736 { 737 struct irdma_device *iwdev = iwqp->iwdev; 738 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 739 struct irdma_iwarp_offload_info *iwarp_info; 740 741 iwarp_info = &iwqp->iwarp_info; 742 ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr); 743 iwarp_info->rd_en = true; 744 iwarp_info->wr_rdresp_en = true; 745 iwarp_info->bind_en = true; 746 iwarp_info->ecn_en = true; 747 iwarp_info->rtomin = 5; 748 749 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 750 iwarp_info->ib_rd_en = true; 751 if (!iwqp->user_mode) { 752 iwarp_info->priv_mode_en = true; 753 iwarp_info->fast_reg_en = true; 754 } 755 iwarp_info->ddp_ver = 1; 756 iwarp_info->rdmap_ver = 1; 757 758 ctx_info->iwarp_info = &iwqp->iwarp_info; 759 ctx_info->iwarp_info_valid = true; 760 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 761 ctx_info->iwarp_info_valid = false; 762 } 763 764 static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, 765 struct irdma_device *iwdev) 766 { 767 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 768 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 769 770 if (init_attr->create_flags) 771 return -EOPNOTSUPP; 772 773 if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || 774 init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || 775 init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags) 776 return -EINVAL; 777 778 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 779 if (init_attr->qp_type != IB_QPT_RC && 780 init_attr->qp_type != IB_QPT_UD && 781 init_attr->qp_type != IB_QPT_GSI) 782 return -EOPNOTSUPP; 783 } else { 784 if (init_attr->qp_type != IB_QPT_RC) 785 return -EOPNOTSUPP; 786 } 787 788 return 0; 789 } 790 791 /** 792 * irdma_create_qp - create qp 793 * @ibpd: ptr of pd 794 * @init_attr: attributes for qp 795 * @udata: user data for create qp 796 */ 797 static struct ib_qp *irdma_create_qp(struct ib_pd *ibpd, 798 struct ib_qp_init_attr *init_attr, 799 struct ib_udata *udata) 800 { 801 struct irdma_pd *iwpd = to_iwpd(ibpd); 802 struct irdma_device *iwdev = to_iwdev(ibpd->device); 803 struct irdma_pci_f *rf = iwdev->rf; 804 struct irdma_qp *iwqp; 805 struct irdma_create_qp_req req; 806 struct irdma_create_qp_resp uresp = {}; 807 u32 qp_num = 0; 808 enum irdma_status_code ret; 809 int err_code; 810 int sq_size; 811 int rq_size; 812 struct irdma_sc_qp *qp; 813 struct irdma_sc_dev *dev = &rf->sc_dev; 814 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 815 struct irdma_qp_init_info init_info = {}; 816 struct irdma_qp_host_ctx_info *ctx_info; 817 unsigned long flags; 818 819 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 820 if (err_code) 821 return ERR_PTR(err_code); 822 823 sq_size = init_attr->cap.max_send_wr; 824 rq_size = init_attr->cap.max_recv_wr; 825 826 init_info.vsi = &iwdev->vsi; 827 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 828 init_info.qp_uk_init_info.sq_size = sq_size; 829 init_info.qp_uk_init_info.rq_size = rq_size; 830 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 831 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 832 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 833 834 iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); 835 if (!iwqp) 836 return ERR_PTR(-ENOMEM); 837 838 qp = &iwqp->sc_qp; 839 qp->qp_uk.back_qp = iwqp; 840 qp->qp_uk.lock = &iwqp->lock; 841 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 842 843 iwqp->iwdev = iwdev; 844 iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, 845 256); 846 iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, 847 iwqp->q2_ctx_mem.size, 848 &iwqp->q2_ctx_mem.pa, 849 GFP_KERNEL); 850 if (!iwqp->q2_ctx_mem.va) { 851 err_code = -ENOMEM; 852 goto error; 853 } 854 855 init_info.q2 = iwqp->q2_ctx_mem.va; 856 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 857 init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE); 858 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 859 860 if (init_attr->qp_type == IB_QPT_GSI) 861 qp_num = 1; 862 else 863 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 864 &qp_num, &rf->next_qp); 865 if (err_code) 866 goto error; 867 868 iwqp->iwpd = iwpd; 869 iwqp->ibqp.qp_num = qp_num; 870 qp = &iwqp->sc_qp; 871 iwqp->iwscq = to_iwcq(init_attr->send_cq); 872 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 873 iwqp->host_ctx.va = init_info.host_ctx; 874 iwqp->host_ctx.pa = init_info.host_ctx_pa; 875 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 876 877 init_info.pd = &iwpd->sc_pd; 878 init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; 879 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 880 init_info.qp_uk_init_info.first_sq_wq = 1; 881 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 882 init_waitqueue_head(&iwqp->waitq); 883 init_waitqueue_head(&iwqp->mod_qp_waitq); 884 885 if (udata) { 886 err_code = ib_copy_from_udata(&req, udata, 887 min(sizeof(req), udata->inlen)); 888 if (err_code) { 889 ibdev_dbg(&iwdev->ibdev, 890 "VERBS: ib_copy_from_data fail\n"); 891 goto error; 892 } 893 894 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; 895 iwqp->user_mode = 1; 896 if (req.user_wqe_bufs) { 897 struct irdma_ucontext *ucontext = 898 rdma_udata_to_drv_context(udata, 899 struct irdma_ucontext, 900 ibucontext); 901 902 init_info.qp_uk_init_info.legacy_mode = ucontext->legacy_mode; 903 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 904 iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, 905 &ucontext->qp_reg_mem_list); 906 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 907 908 if (!iwqp->iwpbl) { 909 err_code = -ENODATA; 910 ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); 911 goto error; 912 } 913 } 914 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 915 irdma_setup_virt_qp(iwdev, iwqp, &init_info); 916 } else { 917 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 918 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 919 } 920 921 if (err_code) { 922 ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n"); 923 goto error; 924 } 925 926 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 927 if (init_attr->qp_type == IB_QPT_RC) { 928 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 929 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 930 IRDMA_WRITE_WITH_IMM | 931 IRDMA_ROCE; 932 } else { 933 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 934 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 935 IRDMA_ROCE; 936 } 937 } else { 938 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 939 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 940 } 941 942 if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) 943 init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE; 944 945 ret = irdma_sc_qp_init(qp, &init_info); 946 if (ret) { 947 err_code = -EPROTO; 948 ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n"); 949 goto error; 950 } 951 952 ctx_info = &iwqp->ctx_info; 953 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 954 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 955 956 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 957 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 958 else 959 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 960 961 err_code = irdma_cqp_create_qp_cmd(iwqp); 962 if (err_code) 963 goto error; 964 965 refcount_set(&iwqp->refcnt, 1); 966 spin_lock_init(&iwqp->lock); 967 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 968 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; 969 rf->qp_table[qp_num] = iwqp; 970 iwqp->max_send_wr = sq_size; 971 iwqp->max_recv_wr = rq_size; 972 973 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 974 if (dev->ws_add(&iwdev->vsi, 0)) { 975 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 976 err_code = -EINVAL; 977 goto error; 978 } 979 980 irdma_qp_add_qos(&iwqp->sc_qp); 981 } 982 983 if (udata) { 984 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 985 if (udata->outlen < sizeof(uresp)) { 986 uresp.lsmm = 1; 987 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 988 } else { 989 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 990 uresp.lsmm = 1; 991 } 992 uresp.actual_sq_size = sq_size; 993 uresp.actual_rq_size = rq_size; 994 uresp.qp_id = qp_num; 995 uresp.qp_caps = qp->qp_uk.qp_caps; 996 997 err_code = ib_copy_to_udata(udata, &uresp, 998 min(sizeof(uresp), udata->outlen)); 999 if (err_code) { 1000 ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); 1001 irdma_destroy_qp(&iwqp->ibqp, udata); 1002 return ERR_PTR(err_code); 1003 } 1004 } 1005 1006 init_completion(&iwqp->free_qp); 1007 return &iwqp->ibqp; 1008 1009 error: 1010 irdma_free_qp_rsrc(iwqp); 1011 1012 return ERR_PTR(err_code); 1013 } 1014 1015 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) 1016 { 1017 int acc_flags = 0; 1018 1019 if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { 1020 if (iwqp->roce_info.wr_rdresp_en) { 1021 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1022 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1023 } 1024 if (iwqp->roce_info.rd_en) 1025 acc_flags |= IB_ACCESS_REMOTE_READ; 1026 if (iwqp->roce_info.bind_en) 1027 acc_flags |= IB_ACCESS_MW_BIND; 1028 } else { 1029 if (iwqp->iwarp_info.wr_rdresp_en) { 1030 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1031 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1032 } 1033 if (iwqp->iwarp_info.rd_en) 1034 acc_flags |= IB_ACCESS_REMOTE_READ; 1035 if (iwqp->iwarp_info.bind_en) 1036 acc_flags |= IB_ACCESS_MW_BIND; 1037 } 1038 return acc_flags; 1039 } 1040 1041 /** 1042 * irdma_query_qp - query qp attributes 1043 * @ibqp: qp pointer 1044 * @attr: attributes pointer 1045 * @attr_mask: Not used 1046 * @init_attr: qp attributes to return 1047 */ 1048 static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1049 int attr_mask, struct ib_qp_init_attr *init_attr) 1050 { 1051 struct irdma_qp *iwqp = to_iwqp(ibqp); 1052 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1053 1054 memset(attr, 0, sizeof(*attr)); 1055 memset(init_attr, 0, sizeof(*init_attr)); 1056 1057 attr->qp_state = iwqp->ibqp_state; 1058 attr->cur_qp_state = iwqp->ibqp_state; 1059 attr->cap.max_send_wr = iwqp->max_send_wr; 1060 attr->cap.max_recv_wr = iwqp->max_recv_wr; 1061 attr->cap.max_inline_data = qp->qp_uk.max_inline_data; 1062 attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; 1063 attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; 1064 attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); 1065 attr->port_num = 1; 1066 if (rdma_protocol_roce(ibqp->device, 1)) { 1067 attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); 1068 attr->qkey = iwqp->roce_info.qkey; 1069 attr->rq_psn = iwqp->udp_info.epsn; 1070 attr->sq_psn = iwqp->udp_info.psn_nxt; 1071 attr->dest_qp_num = iwqp->roce_info.dest_qp; 1072 attr->pkey_index = iwqp->roce_info.p_key; 1073 attr->retry_cnt = iwqp->udp_info.rexmit_thresh; 1074 attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; 1075 attr->max_rd_atomic = iwqp->roce_info.ord_size; 1076 attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; 1077 } 1078 1079 init_attr->event_handler = iwqp->ibqp.event_handler; 1080 init_attr->qp_context = iwqp->ibqp.qp_context; 1081 init_attr->send_cq = iwqp->ibqp.send_cq; 1082 init_attr->recv_cq = iwqp->ibqp.recv_cq; 1083 init_attr->cap = attr->cap; 1084 1085 return 0; 1086 } 1087 1088 /** 1089 * irdma_query_pkey - Query partition key 1090 * @ibdev: device pointer from stack 1091 * @port: port number 1092 * @index: index of pkey 1093 * @pkey: pointer to store the pkey 1094 */ 1095 static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 1096 u16 *pkey) 1097 { 1098 if (index >= IRDMA_PKEY_TBL_SZ) 1099 return -EINVAL; 1100 1101 *pkey = IRDMA_DEFAULT_PKEY; 1102 return 0; 1103 } 1104 1105 /** 1106 * irdma_modify_qp_roce - modify qp request 1107 * @ibqp: qp's pointer for modify 1108 * @attr: access attributes 1109 * @attr_mask: state mask 1110 * @udata: user data 1111 */ 1112 int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1113 int attr_mask, struct ib_udata *udata) 1114 { 1115 struct irdma_pd *iwpd = to_iwpd(ibqp->pd); 1116 struct irdma_qp *iwqp = to_iwqp(ibqp); 1117 struct irdma_device *iwdev = iwqp->iwdev; 1118 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1119 struct irdma_qp_host_ctx_info *ctx_info; 1120 struct irdma_roce_offload_info *roce_info; 1121 struct irdma_udp_offload_info *udp_info; 1122 struct irdma_modify_qp_info info = {}; 1123 struct irdma_modify_qp_resp uresp = {}; 1124 struct irdma_modify_qp_req ureq = {}; 1125 unsigned long flags; 1126 u8 issue_modify_qp = 0; 1127 int ret = 0; 1128 1129 ctx_info = &iwqp->ctx_info; 1130 roce_info = &iwqp->roce_info; 1131 udp_info = &iwqp->udp_info; 1132 1133 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1134 return -EOPNOTSUPP; 1135 1136 if (attr_mask & IB_QP_DEST_QPN) 1137 roce_info->dest_qp = attr->dest_qp_num; 1138 1139 if (attr_mask & IB_QP_PKEY_INDEX) { 1140 ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, 1141 &roce_info->p_key); 1142 if (ret) 1143 return ret; 1144 } 1145 1146 if (attr_mask & IB_QP_QKEY) 1147 roce_info->qkey = attr->qkey; 1148 1149 if (attr_mask & IB_QP_PATH_MTU) 1150 udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); 1151 1152 if (attr_mask & IB_QP_SQ_PSN) { 1153 udp_info->psn_nxt = attr->sq_psn; 1154 udp_info->lsn = 0xffff; 1155 udp_info->psn_una = attr->sq_psn; 1156 udp_info->psn_max = attr->sq_psn; 1157 } 1158 1159 if (attr_mask & IB_QP_RQ_PSN) 1160 udp_info->epsn = attr->rq_psn; 1161 1162 if (attr_mask & IB_QP_RNR_RETRY) 1163 udp_info->rnr_nak_thresh = attr->rnr_retry; 1164 1165 if (attr_mask & IB_QP_RETRY_CNT) 1166 udp_info->rexmit_thresh = attr->retry_cnt; 1167 1168 ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; 1169 1170 if (attr_mask & IB_QP_AV) { 1171 struct irdma_av *av = &iwqp->roce_ah.av; 1172 const struct ib_gid_attr *sgid_attr; 1173 u16 vlan_id = VLAN_N_VID; 1174 u32 local_ip[4]; 1175 1176 memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); 1177 if (attr->ah_attr.ah_flags & IB_AH_GRH) { 1178 udp_info->ttl = attr->ah_attr.grh.hop_limit; 1179 udp_info->flow_label = attr->ah_attr.grh.flow_label; 1180 udp_info->tos = attr->ah_attr.grh.traffic_class; 1181 irdma_qp_rem_qos(&iwqp->sc_qp); 1182 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); 1183 ctx_info->user_pri = rt_tos2priority(udp_info->tos); 1184 iwqp->sc_qp.user_pri = ctx_info->user_pri; 1185 if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) 1186 return -ENOMEM; 1187 irdma_qp_add_qos(&iwqp->sc_qp); 1188 } 1189 sgid_attr = attr->ah_attr.grh.sgid_attr; 1190 ret = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, 1191 ctx_info->roce_info->mac_addr); 1192 if (ret) 1193 return ret; 1194 1195 if (vlan_id >= VLAN_N_VID && iwdev->dcb) 1196 vlan_id = 0; 1197 if (vlan_id < VLAN_N_VID) { 1198 udp_info->insert_vlan_tag = true; 1199 udp_info->vlan_tag = vlan_id | 1200 ctx_info->user_pri << VLAN_PRIO_SHIFT; 1201 } else { 1202 udp_info->insert_vlan_tag = false; 1203 } 1204 1205 av->attrs = attr->ah_attr; 1206 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid); 1207 rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); 1208 roce_info->local_qp = ibqp->qp_num; 1209 if (av->sgid_addr.saddr.sa_family == AF_INET6) { 1210 __be32 *daddr = 1211 av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1212 __be32 *saddr = 1213 av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1214 1215 irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); 1216 irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); 1217 1218 udp_info->ipv4 = false; 1219 irdma_copy_ip_ntohl(local_ip, daddr); 1220 1221 udp_info->arp_idx = irdma_arp_table(iwdev->rf, 1222 &local_ip[0], 1223 false, NULL, 1224 IRDMA_ARP_RESOLVE); 1225 } else { 1226 __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; 1227 __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; 1228 1229 local_ip[0] = ntohl(daddr); 1230 1231 udp_info->ipv4 = true; 1232 udp_info->dest_ip_addr[0] = 0; 1233 udp_info->dest_ip_addr[1] = 0; 1234 udp_info->dest_ip_addr[2] = 0; 1235 udp_info->dest_ip_addr[3] = local_ip[0]; 1236 1237 udp_info->local_ipaddr[0] = 0; 1238 udp_info->local_ipaddr[1] = 0; 1239 udp_info->local_ipaddr[2] = 0; 1240 udp_info->local_ipaddr[3] = ntohl(saddr); 1241 } 1242 udp_info->arp_idx = 1243 irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4, 1244 attr->ah_attr.roce.dmac); 1245 } 1246 1247 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1248 if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { 1249 ibdev_err(&iwdev->ibdev, 1250 "rd_atomic = %d, above max_hw_ord=%d\n", 1251 attr->max_rd_atomic, 1252 dev->hw_attrs.max_hw_ord); 1253 return -EINVAL; 1254 } 1255 if (attr->max_rd_atomic) 1256 roce_info->ord_size = attr->max_rd_atomic; 1257 info.ord_valid = true; 1258 } 1259 1260 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1261 if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { 1262 ibdev_err(&iwdev->ibdev, 1263 "rd_atomic = %d, above max_hw_ird=%d\n", 1264 attr->max_rd_atomic, 1265 dev->hw_attrs.max_hw_ird); 1266 return -EINVAL; 1267 } 1268 if (attr->max_dest_rd_atomic) 1269 roce_info->ird_size = attr->max_dest_rd_atomic; 1270 } 1271 1272 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1273 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1274 roce_info->wr_rdresp_en = true; 1275 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1276 roce_info->wr_rdresp_en = true; 1277 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1278 roce_info->rd_en = true; 1279 } 1280 1281 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1282 1283 ibdev_dbg(&iwdev->ibdev, 1284 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", 1285 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1286 iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); 1287 1288 spin_lock_irqsave(&iwqp->lock, flags); 1289 if (attr_mask & IB_QP_STATE) { 1290 if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, 1291 iwqp->ibqp.qp_type, attr_mask)) { 1292 ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", 1293 iwqp->ibqp.qp_num, iwqp->ibqp_state, 1294 attr->qp_state); 1295 ret = -EINVAL; 1296 goto exit; 1297 } 1298 info.curr_iwarp_state = iwqp->iwarp_state; 1299 1300 switch (attr->qp_state) { 1301 case IB_QPS_INIT: 1302 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1303 ret = -EINVAL; 1304 goto exit; 1305 } 1306 1307 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1308 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1309 issue_modify_qp = 1; 1310 } 1311 break; 1312 case IB_QPS_RTR: 1313 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1314 ret = -EINVAL; 1315 goto exit; 1316 } 1317 info.arp_cache_idx_valid = true; 1318 info.cq_num_valid = true; 1319 info.next_iwarp_state = IRDMA_QP_STATE_RTR; 1320 issue_modify_qp = 1; 1321 break; 1322 case IB_QPS_RTS: 1323 if (iwqp->ibqp_state < IB_QPS_RTR || 1324 iwqp->ibqp_state == IB_QPS_ERR) { 1325 ret = -EINVAL; 1326 goto exit; 1327 } 1328 1329 info.arp_cache_idx_valid = true; 1330 info.cq_num_valid = true; 1331 info.ord_valid = true; 1332 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1333 issue_modify_qp = 1; 1334 if (iwdev->push_mode && udata && 1335 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1336 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1337 spin_unlock_irqrestore(&iwqp->lock, flags); 1338 irdma_alloc_push_page(iwqp); 1339 spin_lock_irqsave(&iwqp->lock, flags); 1340 } 1341 break; 1342 case IB_QPS_SQD: 1343 if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) 1344 goto exit; 1345 1346 if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { 1347 ret = -EINVAL; 1348 goto exit; 1349 } 1350 1351 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1352 issue_modify_qp = 1; 1353 break; 1354 case IB_QPS_SQE: 1355 case IB_QPS_ERR: 1356 case IB_QPS_RESET: 1357 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { 1358 spin_unlock_irqrestore(&iwqp->lock, flags); 1359 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1360 irdma_hw_modify_qp(iwdev, iwqp, &info, true); 1361 spin_lock_irqsave(&iwqp->lock, flags); 1362 } 1363 1364 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1365 spin_unlock_irqrestore(&iwqp->lock, flags); 1366 if (udata) { 1367 if (ib_copy_from_udata(&ureq, udata, 1368 min(sizeof(ureq), udata->inlen))) 1369 return -EINVAL; 1370 1371 irdma_flush_wqes(iwqp, 1372 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1373 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1374 IRDMA_REFLUSH); 1375 } 1376 return 0; 1377 } 1378 1379 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1380 issue_modify_qp = 1; 1381 break; 1382 default: 1383 ret = -EINVAL; 1384 goto exit; 1385 } 1386 1387 iwqp->ibqp_state = attr->qp_state; 1388 } 1389 1390 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1391 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1392 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1393 spin_unlock_irqrestore(&iwqp->lock, flags); 1394 1395 if (attr_mask & IB_QP_STATE) { 1396 if (issue_modify_qp) { 1397 ctx_info->rem_endpoint_idx = udp_info->arp_idx; 1398 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1399 return -EINVAL; 1400 spin_lock_irqsave(&iwqp->lock, flags); 1401 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1402 iwqp->iwarp_state = info.next_iwarp_state; 1403 iwqp->ibqp_state = attr->qp_state; 1404 } 1405 if (iwqp->ibqp_state > IB_QPS_RTS && 1406 !iwqp->flush_issued) { 1407 iwqp->flush_issued = 1; 1408 spin_unlock_irqrestore(&iwqp->lock, flags); 1409 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | 1410 IRDMA_FLUSH_RQ | 1411 IRDMA_FLUSH_WAIT); 1412 } else { 1413 spin_unlock_irqrestore(&iwqp->lock, flags); 1414 } 1415 } else { 1416 iwqp->ibqp_state = attr->qp_state; 1417 } 1418 if (udata && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1419 struct irdma_ucontext *ucontext; 1420 1421 ucontext = rdma_udata_to_drv_context(udata, 1422 struct irdma_ucontext, ibucontext); 1423 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1424 !iwqp->push_wqe_mmap_entry && 1425 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1426 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1427 uresp.push_valid = 1; 1428 uresp.push_offset = iwqp->sc_qp.push_offset; 1429 } 1430 ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1431 udata->outlen)); 1432 if (ret) { 1433 irdma_remove_push_mmap_entries(iwqp); 1434 ibdev_dbg(&iwdev->ibdev, 1435 "VERBS: copy_to_udata failed\n"); 1436 return ret; 1437 } 1438 } 1439 } 1440 1441 return 0; 1442 exit: 1443 spin_unlock_irqrestore(&iwqp->lock, flags); 1444 1445 return ret; 1446 } 1447 1448 /** 1449 * irdma_modify_qp - modify qp request 1450 * @ibqp: qp's pointer for modify 1451 * @attr: access attributes 1452 * @attr_mask: state mask 1453 * @udata: user data 1454 */ 1455 int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1456 struct ib_udata *udata) 1457 { 1458 struct irdma_qp *iwqp = to_iwqp(ibqp); 1459 struct irdma_device *iwdev = iwqp->iwdev; 1460 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1461 struct irdma_qp_host_ctx_info *ctx_info; 1462 struct irdma_tcp_offload_info *tcp_info; 1463 struct irdma_iwarp_offload_info *offload_info; 1464 struct irdma_modify_qp_info info = {}; 1465 struct irdma_modify_qp_resp uresp = {}; 1466 struct irdma_modify_qp_req ureq = {}; 1467 u8 issue_modify_qp = 0; 1468 u8 dont_wait = 0; 1469 int err; 1470 unsigned long flags; 1471 1472 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1473 return -EOPNOTSUPP; 1474 1475 ctx_info = &iwqp->ctx_info; 1476 offload_info = &iwqp->iwarp_info; 1477 tcp_info = &iwqp->tcp_info; 1478 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1479 ibdev_dbg(&iwdev->ibdev, 1480 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", 1481 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1482 iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, 1483 iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); 1484 1485 spin_lock_irqsave(&iwqp->lock, flags); 1486 if (attr_mask & IB_QP_STATE) { 1487 info.curr_iwarp_state = iwqp->iwarp_state; 1488 switch (attr->qp_state) { 1489 case IB_QPS_INIT: 1490 case IB_QPS_RTR: 1491 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1492 err = -EINVAL; 1493 goto exit; 1494 } 1495 1496 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1497 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1498 issue_modify_qp = 1; 1499 } 1500 if (iwdev->push_mode && udata && 1501 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1502 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1503 spin_unlock_irqrestore(&iwqp->lock, flags); 1504 irdma_alloc_push_page(iwqp); 1505 spin_lock_irqsave(&iwqp->lock, flags); 1506 } 1507 break; 1508 case IB_QPS_RTS: 1509 if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || 1510 !iwqp->cm_id) { 1511 err = -EINVAL; 1512 goto exit; 1513 } 1514 1515 issue_modify_qp = 1; 1516 iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 1517 iwqp->hte_added = 1; 1518 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1519 info.tcp_ctx_valid = true; 1520 info.ord_valid = true; 1521 info.arp_cache_idx_valid = true; 1522 info.cq_num_valid = true; 1523 break; 1524 case IB_QPS_SQD: 1525 if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { 1526 err = 0; 1527 goto exit; 1528 } 1529 1530 if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || 1531 iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { 1532 err = 0; 1533 goto exit; 1534 } 1535 1536 if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { 1537 err = -EINVAL; 1538 goto exit; 1539 } 1540 1541 info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; 1542 issue_modify_qp = 1; 1543 break; 1544 case IB_QPS_SQE: 1545 if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { 1546 err = -EINVAL; 1547 goto exit; 1548 } 1549 1550 info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; 1551 issue_modify_qp = 1; 1552 break; 1553 case IB_QPS_ERR: 1554 case IB_QPS_RESET: 1555 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1556 spin_unlock_irqrestore(&iwqp->lock, flags); 1557 if (udata) { 1558 if (ib_copy_from_udata(&ureq, udata, 1559 min(sizeof(ureq), udata->inlen))) 1560 return -EINVAL; 1561 1562 irdma_flush_wqes(iwqp, 1563 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1564 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1565 IRDMA_REFLUSH); 1566 } 1567 return 0; 1568 } 1569 1570 if (iwqp->sc_qp.term_flags) { 1571 spin_unlock_irqrestore(&iwqp->lock, flags); 1572 irdma_terminate_del_timer(&iwqp->sc_qp); 1573 spin_lock_irqsave(&iwqp->lock, flags); 1574 } 1575 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1576 if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && 1577 iwdev->iw_status && 1578 iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) 1579 info.reset_tcp_conn = true; 1580 else 1581 dont_wait = 1; 1582 1583 issue_modify_qp = 1; 1584 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1585 break; 1586 default: 1587 err = -EINVAL; 1588 goto exit; 1589 } 1590 1591 iwqp->ibqp_state = attr->qp_state; 1592 } 1593 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1594 ctx_info->iwarp_info_valid = true; 1595 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1596 offload_info->wr_rdresp_en = true; 1597 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1598 offload_info->wr_rdresp_en = true; 1599 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1600 offload_info->rd_en = true; 1601 } 1602 1603 if (ctx_info->iwarp_info_valid) { 1604 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1605 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1606 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1607 } 1608 spin_unlock_irqrestore(&iwqp->lock, flags); 1609 1610 if (attr_mask & IB_QP_STATE) { 1611 if (issue_modify_qp) { 1612 ctx_info->rem_endpoint_idx = tcp_info->arp_idx; 1613 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1614 return -EINVAL; 1615 } 1616 1617 spin_lock_irqsave(&iwqp->lock, flags); 1618 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1619 iwqp->iwarp_state = info.next_iwarp_state; 1620 iwqp->ibqp_state = attr->qp_state; 1621 } 1622 spin_unlock_irqrestore(&iwqp->lock, flags); 1623 } 1624 1625 if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { 1626 if (dont_wait) { 1627 if (iwqp->cm_id && iwqp->hw_tcp_state) { 1628 spin_lock_irqsave(&iwqp->lock, flags); 1629 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1630 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1631 spin_unlock_irqrestore(&iwqp->lock, flags); 1632 irdma_cm_disconn(iwqp); 1633 } 1634 } else { 1635 int close_timer_started; 1636 1637 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 1638 1639 if (iwqp->cm_node) { 1640 refcount_inc(&iwqp->cm_node->refcnt); 1641 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1642 close_timer_started = atomic_inc_return(&iwqp->close_timer_started); 1643 if (iwqp->cm_id && close_timer_started == 1) 1644 irdma_schedule_cm_timer(iwqp->cm_node, 1645 (struct irdma_puda_buf *)iwqp, 1646 IRDMA_TIMER_TYPE_CLOSE, 1, 0); 1647 1648 irdma_rem_ref_cm_node(iwqp->cm_node); 1649 } else { 1650 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1651 } 1652 } 1653 } 1654 if (attr_mask & IB_QP_STATE && udata && 1655 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1656 struct irdma_ucontext *ucontext; 1657 1658 ucontext = rdma_udata_to_drv_context(udata, 1659 struct irdma_ucontext, ibucontext); 1660 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1661 !iwqp->push_wqe_mmap_entry && 1662 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1663 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1664 uresp.push_valid = 1; 1665 uresp.push_offset = iwqp->sc_qp.push_offset; 1666 } 1667 1668 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1669 udata->outlen)); 1670 if (err) { 1671 irdma_remove_push_mmap_entries(iwqp); 1672 ibdev_dbg(&iwdev->ibdev, 1673 "VERBS: copy_to_udata failed\n"); 1674 return err; 1675 } 1676 } 1677 1678 return 0; 1679 exit: 1680 spin_unlock_irqrestore(&iwqp->lock, flags); 1681 1682 return err; 1683 } 1684 1685 /** 1686 * irdma_cq_free_rsrc - free up resources for cq 1687 * @rf: RDMA PCI function 1688 * @iwcq: cq ptr 1689 */ 1690 static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) 1691 { 1692 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1693 1694 if (!iwcq->user_mode) { 1695 dma_free_coherent(rf->sc_dev.hw->device, iwcq->kmem.size, 1696 iwcq->kmem.va, iwcq->kmem.pa); 1697 iwcq->kmem.va = NULL; 1698 dma_free_coherent(rf->sc_dev.hw->device, 1699 iwcq->kmem_shadow.size, 1700 iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); 1701 iwcq->kmem_shadow.va = NULL; 1702 } 1703 1704 irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); 1705 } 1706 1707 /** 1708 * irdma_free_cqbuf - worker to free a cq buffer 1709 * @work: provides access to the cq buffer to free 1710 */ 1711 static void irdma_free_cqbuf(struct work_struct *work) 1712 { 1713 struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); 1714 1715 dma_free_coherent(cq_buf->hw->device, cq_buf->kmem_buf.size, 1716 cq_buf->kmem_buf.va, cq_buf->kmem_buf.pa); 1717 cq_buf->kmem_buf.va = NULL; 1718 kfree(cq_buf); 1719 } 1720 1721 /** 1722 * irdma_process_resize_list - remove resized cq buffers from the resize_list 1723 * @iwcq: cq which owns the resize_list 1724 * @iwdev: irdma device 1725 * @lcqe_buf: the buffer where the last cqe is received 1726 */ 1727 static int irdma_process_resize_list(struct irdma_cq *iwcq, 1728 struct irdma_device *iwdev, 1729 struct irdma_cq_buf *lcqe_buf) 1730 { 1731 struct list_head *tmp_node, *list_node; 1732 struct irdma_cq_buf *cq_buf; 1733 int cnt = 0; 1734 1735 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 1736 cq_buf = list_entry(list_node, struct irdma_cq_buf, list); 1737 if (cq_buf == lcqe_buf) 1738 return cnt; 1739 1740 list_del(&cq_buf->list); 1741 queue_work(iwdev->cleanup_wq, &cq_buf->work); 1742 cnt++; 1743 } 1744 1745 return cnt; 1746 } 1747 1748 /** 1749 * irdma_destroy_cq - destroy cq 1750 * @ib_cq: cq pointer 1751 * @udata: user data 1752 */ 1753 static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1754 { 1755 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1756 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1757 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1758 struct irdma_sc_dev *dev = cq->dev; 1759 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1760 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1761 unsigned long flags; 1762 1763 spin_lock_irqsave(&iwcq->lock, flags); 1764 if (!list_empty(&iwcq->resize_list)) 1765 irdma_process_resize_list(iwcq, iwdev, NULL); 1766 spin_unlock_irqrestore(&iwcq->lock, flags); 1767 1768 irdma_cq_wq_destroy(iwdev->rf, cq); 1769 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1770 1771 spin_lock_irqsave(&iwceq->ce_lock, flags); 1772 irdma_sc_cleanup_ceqes(cq, ceq); 1773 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1774 1775 return 0; 1776 } 1777 1778 /** 1779 * irdma_resize_cq - resize cq 1780 * @ibcq: cq to be resized 1781 * @entries: desired cq size 1782 * @udata: user data 1783 */ 1784 static int irdma_resize_cq(struct ib_cq *ibcq, int entries, 1785 struct ib_udata *udata) 1786 { 1787 struct irdma_cq *iwcq = to_iwcq(ibcq); 1788 struct irdma_sc_dev *dev = iwcq->sc_cq.dev; 1789 struct irdma_cqp_request *cqp_request; 1790 struct cqp_cmds_info *cqp_info; 1791 struct irdma_modify_cq_info *m_info; 1792 struct irdma_modify_cq_info info = {}; 1793 struct irdma_dma_mem kmem_buf; 1794 struct irdma_cq_mr *cqmr_buf; 1795 struct irdma_pbl *iwpbl_buf; 1796 struct irdma_device *iwdev; 1797 struct irdma_pci_f *rf; 1798 struct irdma_cq_buf *cq_buf = NULL; 1799 enum irdma_status_code status = 0; 1800 unsigned long flags; 1801 int ret; 1802 1803 iwdev = to_iwdev(ibcq->device); 1804 rf = iwdev->rf; 1805 1806 if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 1807 IRDMA_FEATURE_CQ_RESIZE)) 1808 return -EOPNOTSUPP; 1809 1810 if (entries > rf->max_cqe) 1811 return -EINVAL; 1812 1813 if (!iwcq->user_mode) { 1814 entries++; 1815 if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1816 entries *= 2; 1817 } 1818 1819 info.cq_size = max(entries, 4); 1820 1821 if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) 1822 return 0; 1823 1824 if (udata) { 1825 struct irdma_resize_cq_req req = {}; 1826 struct irdma_ucontext *ucontext = 1827 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 1828 ibucontext); 1829 1830 /* CQ resize not supported with legacy GEN_1 libi40iw */ 1831 if (ucontext->legacy_mode) 1832 return -EOPNOTSUPP; 1833 1834 if (ib_copy_from_udata(&req, udata, 1835 min(sizeof(req), udata->inlen))) 1836 return -EINVAL; 1837 1838 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1839 iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, 1840 &ucontext->cq_reg_mem_list); 1841 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1842 1843 if (!iwpbl_buf) 1844 return -ENOMEM; 1845 1846 cqmr_buf = &iwpbl_buf->cq_mr; 1847 if (iwpbl_buf->pbl_allocated) { 1848 info.virtual_map = true; 1849 info.pbl_chunk_size = 1; 1850 info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; 1851 } else { 1852 info.cq_pa = cqmr_buf->cq_pbl.addr; 1853 } 1854 } else { 1855 /* Kmode CQ resize */ 1856 int rsize; 1857 1858 rsize = info.cq_size * sizeof(struct irdma_cqe); 1859 kmem_buf.size = ALIGN(round_up(rsize, 256), 256); 1860 kmem_buf.va = dma_alloc_coherent(dev->hw->device, 1861 kmem_buf.size, &kmem_buf.pa, 1862 GFP_KERNEL); 1863 if (!kmem_buf.va) 1864 return -ENOMEM; 1865 1866 info.cq_base = kmem_buf.va; 1867 info.cq_pa = kmem_buf.pa; 1868 cq_buf = kzalloc(sizeof(*cq_buf), GFP_KERNEL); 1869 if (!cq_buf) { 1870 ret = -ENOMEM; 1871 goto error; 1872 } 1873 } 1874 1875 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1876 if (!cqp_request) { 1877 ret = -ENOMEM; 1878 goto error; 1879 } 1880 1881 info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; 1882 info.cq_resize = true; 1883 1884 cqp_info = &cqp_request->info; 1885 m_info = &cqp_info->in.u.cq_modify.info; 1886 memcpy(m_info, &info, sizeof(*m_info)); 1887 1888 cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; 1889 cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; 1890 cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; 1891 cqp_info->post_sq = 1; 1892 status = irdma_handle_cqp_op(rf, cqp_request); 1893 irdma_put_cqp_request(&rf->cqp, cqp_request); 1894 if (status) { 1895 ret = -EPROTO; 1896 goto error; 1897 } 1898 1899 spin_lock_irqsave(&iwcq->lock, flags); 1900 if (cq_buf) { 1901 cq_buf->kmem_buf = iwcq->kmem; 1902 cq_buf->hw = dev->hw; 1903 memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); 1904 INIT_WORK(&cq_buf->work, irdma_free_cqbuf); 1905 list_add_tail(&cq_buf->list, &iwcq->resize_list); 1906 iwcq->kmem = kmem_buf; 1907 } 1908 1909 irdma_sc_cq_resize(&iwcq->sc_cq, &info); 1910 ibcq->cqe = info.cq_size - 1; 1911 spin_unlock_irqrestore(&iwcq->lock, flags); 1912 1913 return 0; 1914 error: 1915 if (!udata) { 1916 dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, 1917 kmem_buf.pa); 1918 kmem_buf.va = NULL; 1919 } 1920 kfree(cq_buf); 1921 1922 return ret; 1923 } 1924 1925 static inline int cq_validate_flags(u32 flags, u8 hw_rev) 1926 { 1927 /* GEN1 does not support CQ create flags */ 1928 if (hw_rev == IRDMA_GEN_1) 1929 return flags ? -EOPNOTSUPP : 0; 1930 1931 return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; 1932 } 1933 1934 /** 1935 * irdma_create_cq - create cq 1936 * @ibcq: CQ allocated 1937 * @attr: attributes for cq 1938 * @udata: user data 1939 */ 1940 static int irdma_create_cq(struct ib_cq *ibcq, 1941 const struct ib_cq_init_attr *attr, 1942 struct ib_udata *udata) 1943 { 1944 struct ib_device *ibdev = ibcq->device; 1945 struct irdma_device *iwdev = to_iwdev(ibdev); 1946 struct irdma_pci_f *rf = iwdev->rf; 1947 struct irdma_cq *iwcq = to_iwcq(ibcq); 1948 u32 cq_num = 0; 1949 struct irdma_sc_cq *cq; 1950 struct irdma_sc_dev *dev = &rf->sc_dev; 1951 struct irdma_cq_init_info info = {}; 1952 enum irdma_status_code status; 1953 struct irdma_cqp_request *cqp_request; 1954 struct cqp_cmds_info *cqp_info; 1955 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 1956 unsigned long flags; 1957 int err_code; 1958 int entries = attr->cqe; 1959 1960 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 1961 if (err_code) 1962 return err_code; 1963 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 1964 &rf->next_cq); 1965 if (err_code) 1966 return err_code; 1967 1968 cq = &iwcq->sc_cq; 1969 cq->back_cq = iwcq; 1970 spin_lock_init(&iwcq->lock); 1971 INIT_LIST_HEAD(&iwcq->resize_list); 1972 info.dev = dev; 1973 ukinfo->cq_size = max(entries, 4); 1974 ukinfo->cq_id = cq_num; 1975 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 1976 if (attr->comp_vector < rf->ceqs_count) 1977 info.ceq_id = attr->comp_vector; 1978 info.ceq_id_valid = true; 1979 info.ceqe_mask = 1; 1980 info.type = IRDMA_CQ_TYPE_IWARP; 1981 info.vsi = &iwdev->vsi; 1982 1983 if (udata) { 1984 struct irdma_ucontext *ucontext; 1985 struct irdma_create_cq_req req = {}; 1986 struct irdma_cq_mr *cqmr; 1987 struct irdma_pbl *iwpbl; 1988 struct irdma_pbl *iwpbl_shadow; 1989 struct irdma_cq_mr *cqmr_shadow; 1990 1991 iwcq->user_mode = true; 1992 ucontext = 1993 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 1994 ibucontext); 1995 if (ib_copy_from_udata(&req, udata, 1996 min(sizeof(req), udata->inlen))) { 1997 err_code = -EFAULT; 1998 goto cq_free_rsrc; 1999 } 2000 2001 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2002 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 2003 &ucontext->cq_reg_mem_list); 2004 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2005 if (!iwpbl) { 2006 err_code = -EPROTO; 2007 goto cq_free_rsrc; 2008 } 2009 2010 iwcq->iwpbl = iwpbl; 2011 iwcq->cq_mem_size = 0; 2012 cqmr = &iwpbl->cq_mr; 2013 2014 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2015 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { 2016 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2017 iwpbl_shadow = irdma_get_pbl( 2018 (unsigned long)req.user_shadow_area, 2019 &ucontext->cq_reg_mem_list); 2020 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2021 2022 if (!iwpbl_shadow) { 2023 err_code = -EPROTO; 2024 goto cq_free_rsrc; 2025 } 2026 iwcq->iwpbl_shadow = iwpbl_shadow; 2027 cqmr_shadow = &iwpbl_shadow->cq_mr; 2028 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 2029 cqmr->split = true; 2030 } else { 2031 info.shadow_area_pa = cqmr->shadow; 2032 } 2033 if (iwpbl->pbl_allocated) { 2034 info.virtual_map = true; 2035 info.pbl_chunk_size = 1; 2036 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 2037 } else { 2038 info.cq_base_pa = cqmr->cq_pbl.addr; 2039 } 2040 } else { 2041 /* Kmode allocations */ 2042 int rsize; 2043 2044 if (entries > rf->max_cqe) { 2045 err_code = -EINVAL; 2046 goto cq_free_rsrc; 2047 } 2048 2049 entries++; 2050 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2051 entries *= 2; 2052 ukinfo->cq_size = entries; 2053 2054 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 2055 iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256); 2056 iwcq->kmem.va = dma_alloc_coherent(dev->hw->device, 2057 iwcq->kmem.size, 2058 &iwcq->kmem.pa, GFP_KERNEL); 2059 if (!iwcq->kmem.va) { 2060 err_code = -ENOMEM; 2061 goto cq_free_rsrc; 2062 } 2063 2064 iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, 2065 64); 2066 iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device, 2067 iwcq->kmem_shadow.size, 2068 &iwcq->kmem_shadow.pa, 2069 GFP_KERNEL); 2070 if (!iwcq->kmem_shadow.va) { 2071 err_code = -ENOMEM; 2072 goto cq_free_rsrc; 2073 } 2074 info.shadow_area_pa = iwcq->kmem_shadow.pa; 2075 ukinfo->shadow_area = iwcq->kmem_shadow.va; 2076 ukinfo->cq_base = iwcq->kmem.va; 2077 info.cq_base_pa = iwcq->kmem.pa; 2078 } 2079 2080 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2081 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 2082 (u32)IRDMA_MAX_CQ_READ_THRESH); 2083 2084 if (irdma_sc_cq_init(cq, &info)) { 2085 ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n"); 2086 err_code = -EPROTO; 2087 goto cq_free_rsrc; 2088 } 2089 2090 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2091 if (!cqp_request) { 2092 err_code = -ENOMEM; 2093 goto cq_free_rsrc; 2094 } 2095 2096 cqp_info = &cqp_request->info; 2097 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 2098 cqp_info->post_sq = 1; 2099 cqp_info->in.u.cq_create.cq = cq; 2100 cqp_info->in.u.cq_create.check_overflow = true; 2101 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 2102 status = irdma_handle_cqp_op(rf, cqp_request); 2103 irdma_put_cqp_request(&rf->cqp, cqp_request); 2104 if (status) { 2105 err_code = -ENOMEM; 2106 goto cq_free_rsrc; 2107 } 2108 2109 if (udata) { 2110 struct irdma_create_cq_resp resp = {}; 2111 2112 resp.cq_id = info.cq_uk_init_info.cq_id; 2113 resp.cq_size = info.cq_uk_init_info.cq_size; 2114 if (ib_copy_to_udata(udata, &resp, 2115 min(sizeof(resp), udata->outlen))) { 2116 ibdev_dbg(&iwdev->ibdev, 2117 "VERBS: copy to user data\n"); 2118 err_code = -EPROTO; 2119 goto cq_destroy; 2120 } 2121 } 2122 return 0; 2123 cq_destroy: 2124 irdma_cq_wq_destroy(rf, cq); 2125 cq_free_rsrc: 2126 irdma_cq_free_rsrc(rf, iwcq); 2127 2128 return err_code; 2129 } 2130 2131 /** 2132 * irdma_get_mr_access - get hw MR access permissions from IB access flags 2133 * @access: IB access flags 2134 */ 2135 static inline u16 irdma_get_mr_access(int access) 2136 { 2137 u16 hw_access = 0; 2138 2139 hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? 2140 IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; 2141 hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? 2142 IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; 2143 hw_access |= (access & IB_ACCESS_REMOTE_READ) ? 2144 IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; 2145 hw_access |= (access & IB_ACCESS_MW_BIND) ? 2146 IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; 2147 hw_access |= (access & IB_ZERO_BASED) ? 2148 IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; 2149 hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; 2150 2151 return hw_access; 2152 } 2153 2154 /** 2155 * irdma_free_stag - free stag resource 2156 * @iwdev: irdma device 2157 * @stag: stag to free 2158 */ 2159 static void irdma_free_stag(struct irdma_device *iwdev, u32 stag) 2160 { 2161 u32 stag_idx; 2162 2163 stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; 2164 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); 2165 } 2166 2167 /** 2168 * irdma_create_stag - create random stag 2169 * @iwdev: irdma device 2170 */ 2171 static u32 irdma_create_stag(struct irdma_device *iwdev) 2172 { 2173 u32 stag = 0; 2174 u32 stag_index = 0; 2175 u32 next_stag_index; 2176 u32 driver_key; 2177 u32 random; 2178 u8 consumer_key; 2179 int ret; 2180 2181 get_random_bytes(&random, sizeof(random)); 2182 consumer_key = (u8)random; 2183 2184 driver_key = random & ~iwdev->rf->mr_stagmask; 2185 next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; 2186 next_stag_index %= iwdev->rf->max_mr; 2187 2188 ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, 2189 iwdev->rf->max_mr, &stag_index, 2190 &next_stag_index); 2191 if (ret) 2192 return stag; 2193 stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; 2194 stag |= driver_key; 2195 stag += (u32)consumer_key; 2196 2197 return stag; 2198 } 2199 2200 /** 2201 * irdma_next_pbl_addr - Get next pbl address 2202 * @pbl: pointer to a pble 2203 * @pinfo: info pointer 2204 * @idx: index 2205 */ 2206 static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, 2207 u32 *idx) 2208 { 2209 *idx += 1; 2210 if (!(*pinfo) || *idx != (*pinfo)->cnt) 2211 return ++pbl; 2212 *idx = 0; 2213 (*pinfo)++; 2214 2215 return (*pinfo)->addr; 2216 } 2217 2218 /** 2219 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 2220 * @iwmr: iwmr for IB's user page addresses 2221 * @pbl: ple pointer to save 1 level or 0 level pble 2222 * @level: indicated level 0, 1 or 2 2223 */ 2224 static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 2225 enum irdma_pble_level level) 2226 { 2227 struct ib_umem *region = iwmr->region; 2228 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2229 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2230 struct irdma_pble_info *pinfo; 2231 struct ib_block_iter biter; 2232 u32 idx = 0; 2233 u32 pbl_cnt = 0; 2234 2235 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 2236 2237 if (iwmr->type == IRDMA_MEMREG_TYPE_QP) 2238 iwpbl->qp_mr.sq_page = sg_page(region->sg_head.sgl); 2239 2240 rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { 2241 *pbl = rdma_block_iter_dma_address(&biter); 2242 if (++pbl_cnt == palloc->total_cnt) 2243 break; 2244 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 2245 } 2246 } 2247 2248 /** 2249 * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous 2250 * @arr: lvl1 pbl array 2251 * @npages: page count 2252 * @pg_size: page size 2253 * 2254 */ 2255 static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) 2256 { 2257 u32 pg_idx; 2258 2259 for (pg_idx = 0; pg_idx < npages; pg_idx++) { 2260 if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) 2261 return false; 2262 } 2263 2264 return true; 2265 } 2266 2267 /** 2268 * irdma_check_mr_contiguous - check if MR is physically contiguous 2269 * @palloc: pbl allocation struct 2270 * @pg_size: page size 2271 */ 2272 static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, 2273 u32 pg_size) 2274 { 2275 struct irdma_pble_level2 *lvl2 = &palloc->level2; 2276 struct irdma_pble_info *leaf = lvl2->leaf; 2277 u64 *arr = NULL; 2278 u64 *start_addr = NULL; 2279 int i; 2280 bool ret; 2281 2282 if (palloc->level == PBLE_LEVEL_1) { 2283 arr = palloc->level1.addr; 2284 ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, 2285 pg_size); 2286 return ret; 2287 } 2288 2289 start_addr = leaf->addr; 2290 2291 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { 2292 arr = leaf->addr; 2293 if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) 2294 return false; 2295 ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); 2296 if (!ret) 2297 return false; 2298 } 2299 2300 return true; 2301 } 2302 2303 /** 2304 * irdma_setup_pbles - copy user pg address to pble's 2305 * @rf: RDMA PCI function 2306 * @iwmr: mr pointer for this memory registration 2307 * @use_pbles: flag if to use pble's 2308 */ 2309 static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, 2310 bool use_pbles) 2311 { 2312 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2313 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2314 struct irdma_pble_info *pinfo; 2315 u64 *pbl; 2316 enum irdma_status_code status; 2317 enum irdma_pble_level level = PBLE_LEVEL_1; 2318 2319 if (use_pbles) { 2320 status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, 2321 false); 2322 if (status) 2323 return -ENOMEM; 2324 2325 iwpbl->pbl_allocated = true; 2326 level = palloc->level; 2327 pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : 2328 palloc->level2.leaf; 2329 pbl = pinfo->addr; 2330 } else { 2331 pbl = iwmr->pgaddrmem; 2332 } 2333 2334 irdma_copy_user_pgaddrs(iwmr, pbl, level); 2335 2336 if (use_pbles) 2337 iwmr->pgaddrmem[0] = *pbl; 2338 2339 return 0; 2340 } 2341 2342 /** 2343 * irdma_handle_q_mem - handle memory for qp and cq 2344 * @iwdev: irdma device 2345 * @req: information for q memory management 2346 * @iwpbl: pble struct 2347 * @use_pbles: flag to use pble 2348 */ 2349 static int irdma_handle_q_mem(struct irdma_device *iwdev, 2350 struct irdma_mem_reg_req *req, 2351 struct irdma_pbl *iwpbl, bool use_pbles) 2352 { 2353 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2354 struct irdma_mr *iwmr = iwpbl->iwmr; 2355 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 2356 struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; 2357 struct irdma_hmc_pble *hmc_p; 2358 u64 *arr = iwmr->pgaddrmem; 2359 u32 pg_size, total; 2360 int err = 0; 2361 bool ret = true; 2362 2363 pg_size = iwmr->page_size; 2364 err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); 2365 if (err) 2366 return err; 2367 2368 if (use_pbles && palloc->level != PBLE_LEVEL_1) { 2369 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2370 iwpbl->pbl_allocated = false; 2371 return -ENOMEM; 2372 } 2373 2374 if (use_pbles) 2375 arr = palloc->level1.addr; 2376 2377 switch (iwmr->type) { 2378 case IRDMA_MEMREG_TYPE_QP: 2379 total = req->sq_pages + req->rq_pages; 2380 hmc_p = &qpmr->sq_pbl; 2381 qpmr->shadow = (dma_addr_t)arr[total]; 2382 2383 if (use_pbles) { 2384 ret = irdma_check_mem_contiguous(arr, req->sq_pages, 2385 pg_size); 2386 if (ret) 2387 ret = irdma_check_mem_contiguous(&arr[req->sq_pages], 2388 req->rq_pages, 2389 pg_size); 2390 } 2391 2392 if (!ret) { 2393 hmc_p->idx = palloc->level1.idx; 2394 hmc_p = &qpmr->rq_pbl; 2395 hmc_p->idx = palloc->level1.idx + req->sq_pages; 2396 } else { 2397 hmc_p->addr = arr[0]; 2398 hmc_p = &qpmr->rq_pbl; 2399 hmc_p->addr = arr[req->sq_pages]; 2400 } 2401 break; 2402 case IRDMA_MEMREG_TYPE_CQ: 2403 hmc_p = &cqmr->cq_pbl; 2404 2405 if (!cqmr->split) 2406 cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; 2407 2408 if (use_pbles) 2409 ret = irdma_check_mem_contiguous(arr, req->cq_pages, 2410 pg_size); 2411 2412 if (!ret) 2413 hmc_p->idx = palloc->level1.idx; 2414 else 2415 hmc_p->addr = arr[0]; 2416 break; 2417 default: 2418 ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n"); 2419 err = -EINVAL; 2420 } 2421 2422 if (use_pbles && ret) { 2423 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2424 iwpbl->pbl_allocated = false; 2425 } 2426 2427 return err; 2428 } 2429 2430 /** 2431 * irdma_hw_alloc_mw - create the hw memory window 2432 * @iwdev: irdma device 2433 * @iwmr: pointer to memory window info 2434 */ 2435 static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) 2436 { 2437 struct irdma_mw_alloc_info *info; 2438 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2439 struct irdma_cqp_request *cqp_request; 2440 struct cqp_cmds_info *cqp_info; 2441 enum irdma_status_code status; 2442 2443 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2444 if (!cqp_request) 2445 return -ENOMEM; 2446 2447 cqp_info = &cqp_request->info; 2448 info = &cqp_info->in.u.mw_alloc.info; 2449 memset(info, 0, sizeof(*info)); 2450 if (iwmr->ibmw.type == IB_MW_TYPE_1) 2451 info->mw_wide = true; 2452 2453 info->page_size = PAGE_SIZE; 2454 info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2455 info->pd_id = iwpd->sc_pd.pd_id; 2456 info->remote_access = true; 2457 cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; 2458 cqp_info->post_sq = 1; 2459 cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; 2460 cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; 2461 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2462 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2463 2464 return status ? -ENOMEM : 0; 2465 } 2466 2467 /** 2468 * irdma_alloc_mw - Allocate memory window 2469 * @ibmw: Memory Window 2470 * @udata: user data pointer 2471 */ 2472 static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 2473 { 2474 struct irdma_device *iwdev = to_iwdev(ibmw->device); 2475 struct irdma_mr *iwmr = to_iwmw(ibmw); 2476 int err_code; 2477 u32 stag; 2478 2479 stag = irdma_create_stag(iwdev); 2480 if (!stag) 2481 return -ENOMEM; 2482 2483 iwmr->stag = stag; 2484 ibmw->rkey = stag; 2485 2486 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 2487 if (err_code) { 2488 irdma_free_stag(iwdev, stag); 2489 return err_code; 2490 } 2491 2492 return 0; 2493 } 2494 2495 /** 2496 * irdma_dealloc_mw - Dealloc memory window 2497 * @ibmw: memory window structure. 2498 */ 2499 static int irdma_dealloc_mw(struct ib_mw *ibmw) 2500 { 2501 struct ib_pd *ibpd = ibmw->pd; 2502 struct irdma_pd *iwpd = to_iwpd(ibpd); 2503 struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); 2504 struct irdma_device *iwdev = to_iwdev(ibmw->device); 2505 struct irdma_cqp_request *cqp_request; 2506 struct cqp_cmds_info *cqp_info; 2507 struct irdma_dealloc_stag_info *info; 2508 2509 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2510 if (!cqp_request) 2511 return -ENOMEM; 2512 2513 cqp_info = &cqp_request->info; 2514 info = &cqp_info->in.u.dealloc_stag.info; 2515 memset(info, 0, sizeof(*info)); 2516 info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff; 2517 info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 2518 info->mr = false; 2519 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 2520 cqp_info->post_sq = 1; 2521 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 2522 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 2523 irdma_handle_cqp_op(iwdev->rf, cqp_request); 2524 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2525 irdma_free_stag(iwdev, iwmr->stag); 2526 2527 return 0; 2528 } 2529 2530 /** 2531 * irdma_hw_alloc_stag - cqp command to allocate stag 2532 * @iwdev: irdma device 2533 * @iwmr: irdma mr pointer 2534 */ 2535 static int irdma_hw_alloc_stag(struct irdma_device *iwdev, 2536 struct irdma_mr *iwmr) 2537 { 2538 struct irdma_allocate_stag_info *info; 2539 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2540 enum irdma_status_code status; 2541 int err = 0; 2542 struct irdma_cqp_request *cqp_request; 2543 struct cqp_cmds_info *cqp_info; 2544 2545 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2546 if (!cqp_request) 2547 return -ENOMEM; 2548 2549 cqp_info = &cqp_request->info; 2550 info = &cqp_info->in.u.alloc_stag.info; 2551 memset(info, 0, sizeof(*info)); 2552 info->page_size = PAGE_SIZE; 2553 info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2554 info->pd_id = iwpd->sc_pd.pd_id; 2555 info->total_len = iwmr->len; 2556 info->remote_access = true; 2557 cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; 2558 cqp_info->post_sq = 1; 2559 cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; 2560 cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; 2561 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2562 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2563 if (status) 2564 err = -ENOMEM; 2565 2566 return err; 2567 } 2568 2569 /** 2570 * irdma_alloc_mr - register stag for fast memory registration 2571 * @pd: ibpd pointer 2572 * @mr_type: memory for stag registrion 2573 * @max_num_sg: man number of pages 2574 */ 2575 static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 2576 u32 max_num_sg) 2577 { 2578 struct irdma_device *iwdev = to_iwdev(pd->device); 2579 struct irdma_pble_alloc *palloc; 2580 struct irdma_pbl *iwpbl; 2581 struct irdma_mr *iwmr; 2582 enum irdma_status_code status; 2583 u32 stag; 2584 int err_code = -ENOMEM; 2585 2586 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2587 if (!iwmr) 2588 return ERR_PTR(-ENOMEM); 2589 2590 stag = irdma_create_stag(iwdev); 2591 if (!stag) { 2592 err_code = -ENOMEM; 2593 goto err; 2594 } 2595 2596 iwmr->stag = stag; 2597 iwmr->ibmr.rkey = stag; 2598 iwmr->ibmr.lkey = stag; 2599 iwmr->ibmr.pd = pd; 2600 iwmr->ibmr.device = pd->device; 2601 iwpbl = &iwmr->iwpbl; 2602 iwpbl->iwmr = iwmr; 2603 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 2604 palloc = &iwpbl->pble_alloc; 2605 iwmr->page_cnt = max_num_sg; 2606 status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 2607 true); 2608 if (status) 2609 goto err_get_pble; 2610 2611 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 2612 if (err_code) 2613 goto err_alloc_stag; 2614 2615 iwpbl->pbl_allocated = true; 2616 2617 return &iwmr->ibmr; 2618 err_alloc_stag: 2619 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2620 err_get_pble: 2621 irdma_free_stag(iwdev, stag); 2622 err: 2623 kfree(iwmr); 2624 2625 return ERR_PTR(err_code); 2626 } 2627 2628 /** 2629 * irdma_set_page - populate pbl list for fmr 2630 * @ibmr: ib mem to access iwarp mr pointer 2631 * @addr: page dma address fro pbl list 2632 */ 2633 static int irdma_set_page(struct ib_mr *ibmr, u64 addr) 2634 { 2635 struct irdma_mr *iwmr = to_iwmr(ibmr); 2636 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2637 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2638 u64 *pbl; 2639 2640 if (unlikely(iwmr->npages == iwmr->page_cnt)) 2641 return -ENOMEM; 2642 2643 pbl = palloc->level1.addr; 2644 pbl[iwmr->npages++] = addr; 2645 2646 return 0; 2647 } 2648 2649 /** 2650 * irdma_map_mr_sg - map of sg list for fmr 2651 * @ibmr: ib mem to access iwarp mr pointer 2652 * @sg: scatter gather list 2653 * @sg_nents: number of sg pages 2654 * @sg_offset: scatter gather list for fmr 2655 */ 2656 static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2657 int sg_nents, unsigned int *sg_offset) 2658 { 2659 struct irdma_mr *iwmr = to_iwmr(ibmr); 2660 2661 iwmr->npages = 0; 2662 2663 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); 2664 } 2665 2666 /** 2667 * irdma_hwreg_mr - send cqp command for memory registration 2668 * @iwdev: irdma device 2669 * @iwmr: irdma mr pointer 2670 * @access: access for MR 2671 */ 2672 static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, 2673 u16 access) 2674 { 2675 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2676 struct irdma_reg_ns_stag_info *stag_info; 2677 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2678 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2679 enum irdma_status_code status; 2680 int err = 0; 2681 struct irdma_cqp_request *cqp_request; 2682 struct cqp_cmds_info *cqp_info; 2683 2684 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2685 if (!cqp_request) 2686 return -ENOMEM; 2687 2688 cqp_info = &cqp_request->info; 2689 stag_info = &cqp_info->in.u.mr_reg_non_shared.info; 2690 memset(stag_info, 0, sizeof(*stag_info)); 2691 stag_info->va = iwpbl->user_base; 2692 stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2693 stag_info->stag_key = (u8)iwmr->stag; 2694 stag_info->total_len = iwmr->len; 2695 stag_info->access_rights = irdma_get_mr_access(access); 2696 stag_info->pd_id = iwpd->sc_pd.pd_id; 2697 if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) 2698 stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; 2699 else 2700 stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; 2701 stag_info->page_size = iwmr->page_size; 2702 2703 if (iwpbl->pbl_allocated) { 2704 if (palloc->level == PBLE_LEVEL_1) { 2705 stag_info->first_pm_pbl_index = palloc->level1.idx; 2706 stag_info->chunk_size = 1; 2707 } else { 2708 stag_info->first_pm_pbl_index = palloc->level2.root.idx; 2709 stag_info->chunk_size = 3; 2710 } 2711 } else { 2712 stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; 2713 } 2714 2715 cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; 2716 cqp_info->post_sq = 1; 2717 cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; 2718 cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; 2719 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2720 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2721 if (status) 2722 err = -ENOMEM; 2723 2724 return err; 2725 } 2726 2727 /** 2728 * irdma_reg_user_mr - Register a user memory region 2729 * @pd: ptr of pd 2730 * @start: virtual start address 2731 * @len: length of mr 2732 * @virt: virtual address 2733 * @access: access of mr 2734 * @udata: user data 2735 */ 2736 static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, 2737 u64 virt, int access, 2738 struct ib_udata *udata) 2739 { 2740 struct irdma_device *iwdev = to_iwdev(pd->device); 2741 struct irdma_ucontext *ucontext; 2742 struct irdma_pble_alloc *palloc; 2743 struct irdma_pbl *iwpbl; 2744 struct irdma_mr *iwmr; 2745 struct ib_umem *region; 2746 struct irdma_mem_reg_req req; 2747 u32 total, stag = 0; 2748 u8 shadow_pgcnt = 1; 2749 bool use_pbles = false; 2750 unsigned long flags; 2751 int err = -EINVAL; 2752 int ret; 2753 2754 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 2755 return ERR_PTR(-EINVAL); 2756 2757 region = ib_umem_get(pd->device, start, len, access); 2758 2759 if (IS_ERR(region)) { 2760 ibdev_dbg(&iwdev->ibdev, 2761 "VERBS: Failed to create ib_umem region\n"); 2762 return (struct ib_mr *)region; 2763 } 2764 2765 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { 2766 ib_umem_release(region); 2767 return ERR_PTR(-EFAULT); 2768 } 2769 2770 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2771 if (!iwmr) { 2772 ib_umem_release(region); 2773 return ERR_PTR(-ENOMEM); 2774 } 2775 2776 iwpbl = &iwmr->iwpbl; 2777 iwpbl->iwmr = iwmr; 2778 iwmr->region = region; 2779 iwmr->ibmr.pd = pd; 2780 iwmr->ibmr.device = pd->device; 2781 iwmr->ibmr.iova = virt; 2782 iwmr->page_size = PAGE_SIZE; 2783 2784 if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) { 2785 iwmr->page_size = ib_umem_find_best_pgsz(region, 2786 SZ_4K | SZ_2M | SZ_1G, 2787 virt); 2788 if (unlikely(!iwmr->page_size)) { 2789 kfree(iwmr); 2790 ib_umem_release(region); 2791 return ERR_PTR(-EOPNOTSUPP); 2792 } 2793 } 2794 iwmr->len = region->length; 2795 iwpbl->user_base = virt; 2796 palloc = &iwpbl->pble_alloc; 2797 iwmr->type = req.reg_type; 2798 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 2799 2800 switch (req.reg_type) { 2801 case IRDMA_MEMREG_TYPE_QP: 2802 total = req.sq_pages + req.rq_pages + shadow_pgcnt; 2803 if (total > iwmr->page_cnt) { 2804 err = -EINVAL; 2805 goto error; 2806 } 2807 total = req.sq_pages + req.rq_pages; 2808 use_pbles = (total > 2); 2809 err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2810 if (err) 2811 goto error; 2812 2813 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2814 ibucontext); 2815 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2816 list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 2817 iwpbl->on_list = true; 2818 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2819 break; 2820 case IRDMA_MEMREG_TYPE_CQ: 2821 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 2822 shadow_pgcnt = 0; 2823 total = req.cq_pages + shadow_pgcnt; 2824 if (total > iwmr->page_cnt) { 2825 err = -EINVAL; 2826 goto error; 2827 } 2828 2829 use_pbles = (req.cq_pages > 1); 2830 err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2831 if (err) 2832 goto error; 2833 2834 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2835 ibucontext); 2836 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2837 list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 2838 iwpbl->on_list = true; 2839 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2840 break; 2841 case IRDMA_MEMREG_TYPE_MEM: 2842 use_pbles = (iwmr->page_cnt != 1); 2843 2844 err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); 2845 if (err) 2846 goto error; 2847 2848 if (use_pbles) { 2849 ret = irdma_check_mr_contiguous(palloc, 2850 iwmr->page_size); 2851 if (ret) { 2852 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2853 iwpbl->pbl_allocated = false; 2854 } 2855 } 2856 2857 stag = irdma_create_stag(iwdev); 2858 if (!stag) { 2859 err = -ENOMEM; 2860 goto error; 2861 } 2862 2863 iwmr->stag = stag; 2864 iwmr->ibmr.rkey = stag; 2865 iwmr->ibmr.lkey = stag; 2866 err = irdma_hwreg_mr(iwdev, iwmr, access); 2867 if (err) { 2868 irdma_free_stag(iwdev, stag); 2869 goto error; 2870 } 2871 2872 break; 2873 default: 2874 goto error; 2875 } 2876 2877 iwmr->type = req.reg_type; 2878 2879 return &iwmr->ibmr; 2880 2881 error: 2882 if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 2883 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2884 ib_umem_release(region); 2885 kfree(iwmr); 2886 2887 return ERR_PTR(err); 2888 } 2889 2890 /** 2891 * irdma_reg_phys_mr - register kernel physical memory 2892 * @pd: ibpd pointer 2893 * @addr: physical address of memory to register 2894 * @size: size of memory to register 2895 * @access: Access rights 2896 * @iova_start: start of virtual address for physical buffers 2897 */ 2898 struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, 2899 u64 *iova_start) 2900 { 2901 struct irdma_device *iwdev = to_iwdev(pd->device); 2902 struct irdma_pbl *iwpbl; 2903 struct irdma_mr *iwmr; 2904 enum irdma_status_code status; 2905 u32 stag; 2906 int ret; 2907 2908 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2909 if (!iwmr) 2910 return ERR_PTR(-ENOMEM); 2911 2912 iwmr->ibmr.pd = pd; 2913 iwmr->ibmr.device = pd->device; 2914 iwpbl = &iwmr->iwpbl; 2915 iwpbl->iwmr = iwmr; 2916 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 2917 iwpbl->user_base = *iova_start; 2918 stag = irdma_create_stag(iwdev); 2919 if (!stag) { 2920 ret = -ENOMEM; 2921 goto err; 2922 } 2923 2924 iwmr->stag = stag; 2925 iwmr->ibmr.iova = *iova_start; 2926 iwmr->ibmr.rkey = stag; 2927 iwmr->ibmr.lkey = stag; 2928 iwmr->page_cnt = 1; 2929 iwmr->pgaddrmem[0] = addr; 2930 iwmr->len = size; 2931 iwmr->page_size = SZ_4K; 2932 status = irdma_hwreg_mr(iwdev, iwmr, access); 2933 if (status) { 2934 irdma_free_stag(iwdev, stag); 2935 ret = -ENOMEM; 2936 goto err; 2937 } 2938 2939 return &iwmr->ibmr; 2940 2941 err: 2942 kfree(iwmr); 2943 2944 return ERR_PTR(ret); 2945 } 2946 2947 /** 2948 * irdma_get_dma_mr - register physical mem 2949 * @pd: ptr of pd 2950 * @acc: access for memory 2951 */ 2952 static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) 2953 { 2954 u64 kva = 0; 2955 2956 return irdma_reg_phys_mr(pd, 0, 0, acc, &kva); 2957 } 2958 2959 /** 2960 * irdma_del_memlist - Deleting pbl list entries for CQ/QP 2961 * @iwmr: iwmr for IB's user page addresses 2962 * @ucontext: ptr to user context 2963 */ 2964 static void irdma_del_memlist(struct irdma_mr *iwmr, 2965 struct irdma_ucontext *ucontext) 2966 { 2967 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2968 unsigned long flags; 2969 2970 switch (iwmr->type) { 2971 case IRDMA_MEMREG_TYPE_CQ: 2972 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2973 if (iwpbl->on_list) { 2974 iwpbl->on_list = false; 2975 list_del(&iwpbl->list); 2976 } 2977 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2978 break; 2979 case IRDMA_MEMREG_TYPE_QP: 2980 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2981 if (iwpbl->on_list) { 2982 iwpbl->on_list = false; 2983 list_del(&iwpbl->list); 2984 } 2985 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2986 break; 2987 default: 2988 break; 2989 } 2990 } 2991 2992 /** 2993 * irdma_dereg_mr - deregister mr 2994 * @ib_mr: mr ptr for dereg 2995 * @udata: user data 2996 */ 2997 static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 2998 { 2999 struct ib_pd *ibpd = ib_mr->pd; 3000 struct irdma_pd *iwpd = to_iwpd(ibpd); 3001 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3002 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3003 struct irdma_dealloc_stag_info *info; 3004 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3005 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3006 struct irdma_cqp_request *cqp_request; 3007 struct cqp_cmds_info *cqp_info; 3008 3009 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 3010 if (iwmr->region) { 3011 struct irdma_ucontext *ucontext; 3012 3013 ucontext = rdma_udata_to_drv_context(udata, 3014 struct irdma_ucontext, 3015 ibucontext); 3016 irdma_del_memlist(iwmr, ucontext); 3017 } 3018 goto done; 3019 } 3020 3021 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3022 if (!cqp_request) 3023 return -ENOMEM; 3024 3025 cqp_info = &cqp_request->info; 3026 info = &cqp_info->in.u.dealloc_stag.info; 3027 memset(info, 0, sizeof(*info)); 3028 info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff; 3029 info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3030 info->mr = true; 3031 if (iwpbl->pbl_allocated) 3032 info->dealloc_pbl = true; 3033 3034 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3035 cqp_info->post_sq = 1; 3036 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3037 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3038 irdma_handle_cqp_op(iwdev->rf, cqp_request); 3039 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3040 irdma_free_stag(iwdev, iwmr->stag); 3041 done: 3042 if (iwpbl->pbl_allocated) 3043 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3044 ib_umem_release(iwmr->region); 3045 kfree(iwmr); 3046 3047 return 0; 3048 } 3049 3050 /** 3051 * irdma_copy_sg_list - copy sg list for qp 3052 * @sg_list: copied into sg_list 3053 * @sgl: copy from sgl 3054 * @num_sges: count of sg entries 3055 */ 3056 static void irdma_copy_sg_list(struct irdma_sge *sg_list, struct ib_sge *sgl, 3057 int num_sges) 3058 { 3059 unsigned int i; 3060 3061 for (i = 0; (i < num_sges) && (i < IRDMA_MAX_WQ_FRAGMENT_COUNT); i++) { 3062 sg_list[i].tag_off = sgl[i].addr; 3063 sg_list[i].len = sgl[i].length; 3064 sg_list[i].stag = sgl[i].lkey; 3065 } 3066 } 3067 3068 /** 3069 * irdma_post_send - kernel application wr 3070 * @ibqp: qp ptr for wr 3071 * @ib_wr: work request ptr 3072 * @bad_wr: return of bad wr if err 3073 */ 3074 static int irdma_post_send(struct ib_qp *ibqp, 3075 const struct ib_send_wr *ib_wr, 3076 const struct ib_send_wr **bad_wr) 3077 { 3078 struct irdma_qp *iwqp; 3079 struct irdma_qp_uk *ukqp; 3080 struct irdma_sc_dev *dev; 3081 struct irdma_post_sq_info info; 3082 enum irdma_status_code ret; 3083 int err = 0; 3084 unsigned long flags; 3085 bool inv_stag; 3086 struct irdma_ah *ah; 3087 bool reflush = false; 3088 3089 iwqp = to_iwqp(ibqp); 3090 ukqp = &iwqp->sc_qp.qp_uk; 3091 dev = &iwqp->iwdev->rf->sc_dev; 3092 3093 spin_lock_irqsave(&iwqp->lock, flags); 3094 if (iwqp->flush_issued && ukqp->sq_flush_complete) 3095 reflush = true; 3096 while (ib_wr) { 3097 memset(&info, 0, sizeof(info)); 3098 inv_stag = false; 3099 info.wr_id = (ib_wr->wr_id); 3100 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) 3101 info.signaled = true; 3102 if (ib_wr->send_flags & IB_SEND_FENCE) 3103 info.read_fence = true; 3104 switch (ib_wr->opcode) { 3105 case IB_WR_SEND_WITH_IMM: 3106 if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { 3107 info.imm_data_valid = true; 3108 info.imm_data = ntohl(ib_wr->ex.imm_data); 3109 } else { 3110 err = -EINVAL; 3111 break; 3112 } 3113 fallthrough; 3114 case IB_WR_SEND: 3115 case IB_WR_SEND_WITH_INV: 3116 if (ib_wr->opcode == IB_WR_SEND || 3117 ib_wr->opcode == IB_WR_SEND_WITH_IMM) { 3118 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3119 info.op_type = IRDMA_OP_TYPE_SEND_SOL; 3120 else 3121 info.op_type = IRDMA_OP_TYPE_SEND; 3122 } else { 3123 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3124 info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; 3125 else 3126 info.op_type = IRDMA_OP_TYPE_SEND_INV; 3127 info.stag_to_inv = ib_wr->ex.invalidate_rkey; 3128 } 3129 3130 if (ib_wr->send_flags & IB_SEND_INLINE) { 3131 info.op.inline_send.data = (void *)(unsigned long) 3132 ib_wr->sg_list[0].addr; 3133 info.op.inline_send.len = ib_wr->sg_list[0].length; 3134 if (iwqp->ibqp.qp_type == IB_QPT_UD || 3135 iwqp->ibqp.qp_type == IB_QPT_GSI) { 3136 ah = to_iwah(ud_wr(ib_wr)->ah); 3137 info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx; 3138 info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey; 3139 info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn; 3140 } 3141 ret = irdma_uk_inline_send(ukqp, &info, false); 3142 } else { 3143 info.op.send.num_sges = ib_wr->num_sge; 3144 info.op.send.sg_list = (struct irdma_sge *) 3145 ib_wr->sg_list; 3146 if (iwqp->ibqp.qp_type == IB_QPT_UD || 3147 iwqp->ibqp.qp_type == IB_QPT_GSI) { 3148 ah = to_iwah(ud_wr(ib_wr)->ah); 3149 info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; 3150 info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; 3151 info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; 3152 } 3153 ret = irdma_uk_send(ukqp, &info, false); 3154 } 3155 3156 if (ret) { 3157 if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) 3158 err = -ENOMEM; 3159 else 3160 err = -EINVAL; 3161 } 3162 break; 3163 case IB_WR_RDMA_WRITE_WITH_IMM: 3164 if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { 3165 info.imm_data_valid = true; 3166 info.imm_data = ntohl(ib_wr->ex.imm_data); 3167 } else { 3168 err = -EINVAL; 3169 break; 3170 } 3171 fallthrough; 3172 case IB_WR_RDMA_WRITE: 3173 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3174 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; 3175 else 3176 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; 3177 3178 if (ib_wr->send_flags & IB_SEND_INLINE) { 3179 info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr; 3180 info.op.inline_rdma_write.len = ib_wr->sg_list[0].length; 3181 info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; 3182 info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey; 3183 ret = irdma_uk_inline_rdma_write(ukqp, &info, false); 3184 } else { 3185 info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; 3186 info.op.rdma_write.num_lo_sges = ib_wr->num_sge; 3187 info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; 3188 info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey; 3189 ret = irdma_uk_rdma_write(ukqp, &info, false); 3190 } 3191 3192 if (ret) { 3193 if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) 3194 err = -ENOMEM; 3195 else 3196 err = -EINVAL; 3197 } 3198 break; 3199 case IB_WR_RDMA_READ_WITH_INV: 3200 inv_stag = true; 3201 fallthrough; 3202 case IB_WR_RDMA_READ: 3203 if (ib_wr->num_sge > 3204 dev->hw_attrs.uk_attrs.max_hw_read_sges) { 3205 err = -EINVAL; 3206 break; 3207 } 3208 info.op_type = IRDMA_OP_TYPE_RDMA_READ; 3209 info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; 3210 info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey; 3211 info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; 3212 info.op.rdma_read.num_lo_sges = ib_wr->num_sge; 3213 3214 ret = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); 3215 if (ret) { 3216 if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) 3217 err = -ENOMEM; 3218 else 3219 err = -EINVAL; 3220 } 3221 break; 3222 case IB_WR_LOCAL_INV: 3223 info.op_type = IRDMA_OP_TYPE_INV_STAG; 3224 info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; 3225 ret = irdma_uk_stag_local_invalidate(ukqp, &info, true); 3226 if (ret) 3227 err = -ENOMEM; 3228 break; 3229 case IB_WR_REG_MR: { 3230 struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); 3231 struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; 3232 struct irdma_fast_reg_stag_info stag_info = {}; 3233 3234 stag_info.signaled = info.signaled; 3235 stag_info.read_fence = info.read_fence; 3236 stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access); 3237 stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; 3238 stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; 3239 stag_info.page_size = reg_wr(ib_wr)->mr->page_size; 3240 stag_info.wr_id = ib_wr->wr_id; 3241 stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; 3242 stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; 3243 stag_info.total_len = iwmr->ibmr.length; 3244 stag_info.reg_addr_pa = *palloc->level1.addr; 3245 stag_info.first_pm_pbl_index = palloc->level1.idx; 3246 stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; 3247 if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) 3248 stag_info.chunk_size = 1; 3249 ret = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, 3250 true); 3251 if (ret) 3252 err = -ENOMEM; 3253 break; 3254 } 3255 default: 3256 err = -EINVAL; 3257 ibdev_dbg(&iwqp->iwdev->ibdev, 3258 "VERBS: upost_send bad opcode = 0x%x\n", 3259 ib_wr->opcode); 3260 break; 3261 } 3262 3263 if (err) 3264 break; 3265 ib_wr = ib_wr->next; 3266 } 3267 3268 if (!iwqp->flush_issued && iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) { 3269 irdma_uk_qp_post_wr(ukqp); 3270 spin_unlock_irqrestore(&iwqp->lock, flags); 3271 } else if (reflush) { 3272 ukqp->sq_flush_complete = false; 3273 spin_unlock_irqrestore(&iwqp->lock, flags); 3274 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_REFLUSH); 3275 } else { 3276 spin_unlock_irqrestore(&iwqp->lock, flags); 3277 } 3278 if (err) 3279 *bad_wr = ib_wr; 3280 3281 return err; 3282 } 3283 3284 /** 3285 * irdma_post_recv - post receive wr for kernel application 3286 * @ibqp: ib qp pointer 3287 * @ib_wr: work request for receive 3288 * @bad_wr: bad wr caused an error 3289 */ 3290 static int irdma_post_recv(struct ib_qp *ibqp, 3291 const struct ib_recv_wr *ib_wr, 3292 const struct ib_recv_wr **bad_wr) 3293 { 3294 struct irdma_qp *iwqp; 3295 struct irdma_qp_uk *ukqp; 3296 struct irdma_post_rq_info post_recv = {}; 3297 struct irdma_sge sg_list[IRDMA_MAX_WQ_FRAGMENT_COUNT]; 3298 enum irdma_status_code ret = 0; 3299 unsigned long flags; 3300 int err = 0; 3301 bool reflush = false; 3302 3303 iwqp = to_iwqp(ibqp); 3304 ukqp = &iwqp->sc_qp.qp_uk; 3305 3306 spin_lock_irqsave(&iwqp->lock, flags); 3307 if (iwqp->flush_issued && ukqp->rq_flush_complete) 3308 reflush = true; 3309 while (ib_wr) { 3310 post_recv.num_sges = ib_wr->num_sge; 3311 post_recv.wr_id = ib_wr->wr_id; 3312 irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge); 3313 post_recv.sg_list = sg_list; 3314 ret = irdma_uk_post_receive(ukqp, &post_recv); 3315 if (ret) { 3316 ibdev_dbg(&iwqp->iwdev->ibdev, 3317 "VERBS: post_recv err %d\n", ret); 3318 if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) 3319 err = -ENOMEM; 3320 else 3321 err = -EINVAL; 3322 goto out; 3323 } 3324 3325 ib_wr = ib_wr->next; 3326 } 3327 3328 out: 3329 if (reflush) { 3330 ukqp->rq_flush_complete = false; 3331 spin_unlock_irqrestore(&iwqp->lock, flags); 3332 irdma_flush_wqes(iwqp, IRDMA_FLUSH_RQ | IRDMA_REFLUSH); 3333 } else { 3334 spin_unlock_irqrestore(&iwqp->lock, flags); 3335 } 3336 3337 if (err) 3338 *bad_wr = ib_wr; 3339 3340 return err; 3341 } 3342 3343 /** 3344 * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status 3345 * @opcode: iwarp flush code 3346 */ 3347 static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) 3348 { 3349 switch (opcode) { 3350 case FLUSH_PROT_ERR: 3351 return IB_WC_LOC_PROT_ERR; 3352 case FLUSH_REM_ACCESS_ERR: 3353 return IB_WC_REM_ACCESS_ERR; 3354 case FLUSH_LOC_QP_OP_ERR: 3355 return IB_WC_LOC_QP_OP_ERR; 3356 case FLUSH_REM_OP_ERR: 3357 return IB_WC_REM_OP_ERR; 3358 case FLUSH_LOC_LEN_ERR: 3359 return IB_WC_LOC_LEN_ERR; 3360 case FLUSH_GENERAL_ERR: 3361 return IB_WC_WR_FLUSH_ERR; 3362 case FLUSH_FATAL_ERR: 3363 default: 3364 return IB_WC_FATAL_ERR; 3365 } 3366 } 3367 3368 /** 3369 * irdma_process_cqe - process cqe info 3370 * @entry: processed cqe 3371 * @cq_poll_info: cqe info 3372 */ 3373 static void irdma_process_cqe(struct ib_wc *entry, 3374 struct irdma_cq_poll_info *cq_poll_info) 3375 { 3376 struct irdma_qp *iwqp; 3377 struct irdma_sc_qp *qp; 3378 3379 entry->wc_flags = 0; 3380 entry->pkey_index = 0; 3381 entry->wr_id = cq_poll_info->wr_id; 3382 3383 qp = cq_poll_info->qp_handle; 3384 iwqp = qp->qp_uk.back_qp; 3385 entry->qp = qp->qp_uk.back_qp; 3386 3387 if (cq_poll_info->error) { 3388 entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? 3389 irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; 3390 3391 entry->vendor_err = cq_poll_info->major_err << 16 | 3392 cq_poll_info->minor_err; 3393 } else { 3394 entry->status = IB_WC_SUCCESS; 3395 if (cq_poll_info->imm_valid) { 3396 entry->ex.imm_data = htonl(cq_poll_info->imm_data); 3397 entry->wc_flags |= IB_WC_WITH_IMM; 3398 } 3399 if (cq_poll_info->ud_smac_valid) { 3400 ether_addr_copy(entry->smac, cq_poll_info->ud_smac); 3401 entry->wc_flags |= IB_WC_WITH_SMAC; 3402 } 3403 3404 if (cq_poll_info->ud_vlan_valid) { 3405 entry->vlan_id = cq_poll_info->ud_vlan & VLAN_VID_MASK; 3406 entry->wc_flags |= IB_WC_WITH_VLAN; 3407 entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; 3408 } else { 3409 entry->sl = 0; 3410 } 3411 } 3412 3413 switch (cq_poll_info->op_type) { 3414 case IRDMA_OP_TYPE_RDMA_WRITE: 3415 case IRDMA_OP_TYPE_RDMA_WRITE_SOL: 3416 entry->opcode = IB_WC_RDMA_WRITE; 3417 break; 3418 case IRDMA_OP_TYPE_RDMA_READ_INV_STAG: 3419 case IRDMA_OP_TYPE_RDMA_READ: 3420 entry->opcode = IB_WC_RDMA_READ; 3421 break; 3422 case IRDMA_OP_TYPE_SEND_INV: 3423 case IRDMA_OP_TYPE_SEND_SOL: 3424 case IRDMA_OP_TYPE_SEND_SOL_INV: 3425 case IRDMA_OP_TYPE_SEND: 3426 entry->opcode = IB_WC_SEND; 3427 break; 3428 case IRDMA_OP_TYPE_FAST_REG_NSMR: 3429 entry->opcode = IB_WC_REG_MR; 3430 break; 3431 case IRDMA_OP_TYPE_INV_STAG: 3432 entry->opcode = IB_WC_LOCAL_INV; 3433 break; 3434 case IRDMA_OP_TYPE_REC_IMM: 3435 case IRDMA_OP_TYPE_REC: 3436 entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ? 3437 IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV; 3438 if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && 3439 cq_poll_info->stag_invalid_set) { 3440 entry->ex.invalidate_rkey = cq_poll_info->inv_stag; 3441 entry->wc_flags |= IB_WC_WITH_INVALIDATE; 3442 } 3443 break; 3444 default: 3445 ibdev_err(&iwqp->iwdev->ibdev, 3446 "Invalid opcode = %d in CQE\n", cq_poll_info->op_type); 3447 entry->status = IB_WC_GENERAL_ERR; 3448 return; 3449 } 3450 3451 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { 3452 entry->src_qp = cq_poll_info->ud_src_qpn; 3453 entry->slid = 0; 3454 entry->wc_flags |= 3455 (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); 3456 entry->network_hdr_type = cq_poll_info->ipv4 ? 3457 RDMA_NETWORK_IPV4 : 3458 RDMA_NETWORK_IPV6; 3459 } else { 3460 entry->src_qp = cq_poll_info->qp_id; 3461 } 3462 3463 entry->byte_len = cq_poll_info->bytes_xfered; 3464 } 3465 3466 /** 3467 * irdma_poll_one - poll one entry of the CQ 3468 * @ukcq: ukcq to poll 3469 * @cur_cqe: current CQE info to be filled in 3470 * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ 3471 * 3472 * Returns the internal irdma device error code or 0 on success 3473 */ 3474 static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, 3475 struct irdma_cq_poll_info *cur_cqe, 3476 struct ib_wc *entry) 3477 { 3478 int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); 3479 3480 if (ret) 3481 return ret; 3482 3483 irdma_process_cqe(entry, cur_cqe); 3484 3485 return 0; 3486 } 3487 3488 /** 3489 * __irdma_poll_cq - poll cq for completion (kernel apps) 3490 * @iwcq: cq to poll 3491 * @num_entries: number of entries to poll 3492 * @entry: wr of a completed entry 3493 */ 3494 static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) 3495 { 3496 struct list_head *tmp_node, *list_node; 3497 struct irdma_cq_buf *last_buf = NULL; 3498 struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; 3499 struct irdma_cq_buf *cq_buf; 3500 enum irdma_status_code ret; 3501 struct irdma_device *iwdev; 3502 struct irdma_cq_uk *ukcq; 3503 bool cq_new_cqe = false; 3504 int resized_bufs = 0; 3505 int npolled = 0; 3506 3507 iwdev = to_iwdev(iwcq->ibcq.device); 3508 ukcq = &iwcq->sc_cq.cq_uk; 3509 3510 /* go through the list of previously resized CQ buffers */ 3511 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 3512 cq_buf = container_of(list_node, struct irdma_cq_buf, list); 3513 while (npolled < num_entries) { 3514 ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); 3515 if (!ret) { 3516 ++npolled; 3517 cq_new_cqe = true; 3518 continue; 3519 } 3520 if (ret == IRDMA_ERR_Q_EMPTY) 3521 break; 3522 /* QP using the CQ is destroyed. Skip reporting this CQE */ 3523 if (ret == IRDMA_ERR_Q_DESTROYED) { 3524 cq_new_cqe = true; 3525 continue; 3526 } 3527 goto error; 3528 } 3529 3530 /* save the resized CQ buffer which received the last cqe */ 3531 if (cq_new_cqe) 3532 last_buf = cq_buf; 3533 cq_new_cqe = false; 3534 } 3535 3536 /* check the current CQ for new cqes */ 3537 while (npolled < num_entries) { 3538 ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); 3539 if (!ret) { 3540 ++npolled; 3541 cq_new_cqe = true; 3542 continue; 3543 } 3544 3545 if (ret == IRDMA_ERR_Q_EMPTY) 3546 break; 3547 /* QP using the CQ is destroyed. Skip reporting this CQE */ 3548 if (ret == IRDMA_ERR_Q_DESTROYED) { 3549 cq_new_cqe = true; 3550 continue; 3551 } 3552 goto error; 3553 } 3554 3555 if (cq_new_cqe) 3556 /* all previous CQ resizes are complete */ 3557 resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); 3558 else if (last_buf) 3559 /* only CQ resizes up to the last_buf are complete */ 3560 resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); 3561 if (resized_bufs) 3562 /* report to the HW the number of complete CQ resizes */ 3563 irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); 3564 3565 return npolled; 3566 error: 3567 ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n", 3568 __func__, ret); 3569 3570 return -EINVAL; 3571 } 3572 3573 /** 3574 * irdma_poll_cq - poll cq for completion (kernel apps) 3575 * @ibcq: cq to poll 3576 * @num_entries: number of entries to poll 3577 * @entry: wr of a completed entry 3578 */ 3579 static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, 3580 struct ib_wc *entry) 3581 { 3582 struct irdma_cq *iwcq; 3583 unsigned long flags; 3584 int ret; 3585 3586 iwcq = to_iwcq(ibcq); 3587 3588 spin_lock_irqsave(&iwcq->lock, flags); 3589 ret = __irdma_poll_cq(iwcq, num_entries, entry); 3590 spin_unlock_irqrestore(&iwcq->lock, flags); 3591 3592 return ret; 3593 } 3594 3595 /** 3596 * irdma_req_notify_cq - arm cq kernel application 3597 * @ibcq: cq to arm 3598 * @notify_flags: notofication flags 3599 */ 3600 static int irdma_req_notify_cq(struct ib_cq *ibcq, 3601 enum ib_cq_notify_flags notify_flags) 3602 { 3603 struct irdma_cq *iwcq; 3604 struct irdma_cq_uk *ukcq; 3605 unsigned long flags; 3606 enum irdma_cmpl_notify cq_notify = IRDMA_CQ_COMPL_EVENT; 3607 3608 iwcq = to_iwcq(ibcq); 3609 ukcq = &iwcq->sc_cq.cq_uk; 3610 if (notify_flags == IB_CQ_SOLICITED) 3611 cq_notify = IRDMA_CQ_COMPL_SOLICITED; 3612 3613 spin_lock_irqsave(&iwcq->lock, flags); 3614 irdma_uk_cq_request_notification(ukcq, cq_notify); 3615 spin_unlock_irqrestore(&iwcq->lock, flags); 3616 3617 return 0; 3618 } 3619 3620 static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, 3621 struct ib_port_immutable *immutable) 3622 { 3623 struct ib_port_attr attr; 3624 int err; 3625 3626 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 3627 err = ib_query_port(ibdev, port_num, &attr); 3628 if (err) 3629 return err; 3630 3631 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 3632 immutable->pkey_tbl_len = attr.pkey_tbl_len; 3633 immutable->gid_tbl_len = attr.gid_tbl_len; 3634 3635 return 0; 3636 } 3637 3638 static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num, 3639 struct ib_port_immutable *immutable) 3640 { 3641 struct ib_port_attr attr; 3642 int err; 3643 3644 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 3645 err = ib_query_port(ibdev, port_num, &attr); 3646 if (err) 3647 return err; 3648 immutable->gid_tbl_len = attr.gid_tbl_len; 3649 3650 return 0; 3651 } 3652 3653 static const char *const irdma_hw_stat_names[] = { 3654 /* 32bit names */ 3655 [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors", 3656 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards", 3657 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts", 3658 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes", 3659 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards", 3660 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts", 3661 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes", 3662 [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs", 3663 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors", 3664 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors", 3665 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled", 3666 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored", 3667 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent", 3668 3669 /* 64bit names */ 3670 [IRDMA_HW_STAT_INDEX_IP4RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3671 "ip4InOctets", 3672 [IRDMA_HW_STAT_INDEX_IP4RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3673 "ip4InPkts", 3674 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] = 3675 "ip4InReasmRqd", 3676 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3677 "ip4InMcastOctets", 3678 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3679 "ip4InMcastPkts", 3680 [IRDMA_HW_STAT_INDEX_IP4TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3681 "ip4OutOctets", 3682 [IRDMA_HW_STAT_INDEX_IP4TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3683 "ip4OutPkts", 3684 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] = 3685 "ip4OutSegRqd", 3686 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3687 "ip4OutMcastOctets", 3688 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3689 "ip4OutMcastPkts", 3690 [IRDMA_HW_STAT_INDEX_IP6RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3691 "ip6InOctets", 3692 [IRDMA_HW_STAT_INDEX_IP6RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3693 "ip6InPkts", 3694 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] = 3695 "ip6InReasmRqd", 3696 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3697 "ip6InMcastOctets", 3698 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3699 "ip6InMcastPkts", 3700 [IRDMA_HW_STAT_INDEX_IP6TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3701 "ip6OutOctets", 3702 [IRDMA_HW_STAT_INDEX_IP6TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3703 "ip6OutPkts", 3704 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32] = 3705 "ip6OutSegRqd", 3706 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3707 "ip6OutMcastOctets", 3708 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3709 "ip6OutMcastPkts", 3710 [IRDMA_HW_STAT_INDEX_TCPRXSEGS + IRDMA_HW_STAT_INDEX_MAX_32] = 3711 "tcpInSegs", 3712 [IRDMA_HW_STAT_INDEX_TCPTXSEG + IRDMA_HW_STAT_INDEX_MAX_32] = 3713 "tcpOutSegs", 3714 [IRDMA_HW_STAT_INDEX_RDMARXRDS + IRDMA_HW_STAT_INDEX_MAX_32] = 3715 "iwInRdmaReads", 3716 [IRDMA_HW_STAT_INDEX_RDMARXSNDS + IRDMA_HW_STAT_INDEX_MAX_32] = 3717 "iwInRdmaSends", 3718 [IRDMA_HW_STAT_INDEX_RDMARXWRS + IRDMA_HW_STAT_INDEX_MAX_32] = 3719 "iwInRdmaWrites", 3720 [IRDMA_HW_STAT_INDEX_RDMATXRDS + IRDMA_HW_STAT_INDEX_MAX_32] = 3721 "iwOutRdmaReads", 3722 [IRDMA_HW_STAT_INDEX_RDMATXSNDS + IRDMA_HW_STAT_INDEX_MAX_32] = 3723 "iwOutRdmaSends", 3724 [IRDMA_HW_STAT_INDEX_RDMATXWRS + IRDMA_HW_STAT_INDEX_MAX_32] = 3725 "iwOutRdmaWrites", 3726 [IRDMA_HW_STAT_INDEX_RDMAVBND + IRDMA_HW_STAT_INDEX_MAX_32] = 3727 "iwRdmaBnd", 3728 [IRDMA_HW_STAT_INDEX_RDMAVINV + IRDMA_HW_STAT_INDEX_MAX_32] = 3729 "iwRdmaInv", 3730 [IRDMA_HW_STAT_INDEX_UDPRXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3731 "RxUDP", 3732 [IRDMA_HW_STAT_INDEX_UDPTXPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3733 "TxUDP", 3734 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS + IRDMA_HW_STAT_INDEX_MAX_32] = 3735 "RxECNMrkd", 3736 }; 3737 3738 static void irdma_get_dev_fw_str(struct ib_device *dev, char *str) 3739 { 3740 struct irdma_device *iwdev = to_iwdev(dev); 3741 3742 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", 3743 irdma_fw_major_ver(&iwdev->rf->sc_dev), 3744 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 3745 } 3746 3747 /** 3748 * irdma_alloc_hw_port_stats - Allocate a hw stats structure 3749 * @ibdev: device pointer from stack 3750 * @port_num: port number 3751 */ 3752 static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev, 3753 u32 port_num) 3754 { 3755 int num_counters = IRDMA_HW_STAT_INDEX_MAX_32 + 3756 IRDMA_HW_STAT_INDEX_MAX_64; 3757 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 3758 3759 BUILD_BUG_ON(ARRAY_SIZE(irdma_hw_stat_names) != 3760 (IRDMA_HW_STAT_INDEX_MAX_32 + IRDMA_HW_STAT_INDEX_MAX_64)); 3761 3762 return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters, 3763 lifespan); 3764 } 3765 3766 /** 3767 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 3768 * @ibdev: device pointer from stack 3769 * @stats: stats pointer from stack 3770 * @port_num: port number 3771 * @index: which hw counter the stack is requesting we update 3772 */ 3773 static int irdma_get_hw_stats(struct ib_device *ibdev, 3774 struct rdma_hw_stats *stats, u32 port_num, 3775 int index) 3776 { 3777 struct irdma_device *iwdev = to_iwdev(ibdev); 3778 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 3779 3780 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 3781 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 3782 else 3783 irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat); 3784 3785 memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats)); 3786 3787 return stats->num_counters; 3788 } 3789 3790 /** 3791 * irdma_query_gid - Query port GID 3792 * @ibdev: device pointer from stack 3793 * @port: port number 3794 * @index: Entry index 3795 * @gid: Global ID 3796 */ 3797 static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index, 3798 union ib_gid *gid) 3799 { 3800 struct irdma_device *iwdev = to_iwdev(ibdev); 3801 3802 memset(gid->raw, 0, sizeof(gid->raw)); 3803 ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); 3804 3805 return 0; 3806 } 3807 3808 /** 3809 * mcast_list_add - Add a new mcast item to list 3810 * @rf: RDMA PCI function 3811 * @new_elem: pointer to element to add 3812 */ 3813 static void mcast_list_add(struct irdma_pci_f *rf, 3814 struct mc_table_list *new_elem) 3815 { 3816 list_add(&new_elem->list, &rf->mc_qht_list.list); 3817 } 3818 3819 /** 3820 * mcast_list_del - Remove an mcast item from list 3821 * @mc_qht_elem: pointer to mcast table list element 3822 */ 3823 static void mcast_list_del(struct mc_table_list *mc_qht_elem) 3824 { 3825 if (mc_qht_elem) 3826 list_del(&mc_qht_elem->list); 3827 } 3828 3829 /** 3830 * mcast_list_lookup_ip - Search mcast list for address 3831 * @rf: RDMA PCI function 3832 * @ip_mcast: pointer to mcast IP address 3833 */ 3834 static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf, 3835 u32 *ip_mcast) 3836 { 3837 struct mc_table_list *mc_qht_el; 3838 struct list_head *pos, *q; 3839 3840 list_for_each_safe (pos, q, &rf->mc_qht_list.list) { 3841 mc_qht_el = list_entry(pos, struct mc_table_list, list); 3842 if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, 3843 sizeof(mc_qht_el->mc_info.dest_ip))) 3844 return mc_qht_el; 3845 } 3846 3847 return NULL; 3848 } 3849 3850 /** 3851 * irdma_mcast_cqp_op - perform a mcast cqp operation 3852 * @iwdev: irdma device 3853 * @mc_grp_ctx: mcast group info 3854 * @op: operation 3855 * 3856 * returns error status 3857 */ 3858 static int irdma_mcast_cqp_op(struct irdma_device *iwdev, 3859 struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) 3860 { 3861 struct cqp_cmds_info *cqp_info; 3862 struct irdma_cqp_request *cqp_request; 3863 enum irdma_status_code status; 3864 3865 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3866 if (!cqp_request) 3867 return -ENOMEM; 3868 3869 cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; 3870 cqp_info = &cqp_request->info; 3871 cqp_info->cqp_cmd = op; 3872 cqp_info->post_sq = 1; 3873 cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; 3874 cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; 3875 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3876 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3877 if (status) 3878 return -ENOMEM; 3879 3880 return 0; 3881 } 3882 3883 /** 3884 * irdma_mcast_mac - Get the multicast MAC for an IP address 3885 * @ip_addr: IPv4 or IPv6 address 3886 * @mac: pointer to result MAC address 3887 * @ipv4: flag indicating IPv4 or IPv6 3888 * 3889 */ 3890 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4) 3891 { 3892 u8 *ip = (u8 *)ip_addr; 3893 3894 if (ipv4) { 3895 unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00, 3896 0x00, 0x00}; 3897 3898 mac4[3] = ip[2] & 0x7F; 3899 mac4[4] = ip[1]; 3900 mac4[5] = ip[0]; 3901 ether_addr_copy(mac, mac4); 3902 } else { 3903 unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00, 3904 0x00, 0x00}; 3905 3906 mac6[2] = ip[3]; 3907 mac6[3] = ip[2]; 3908 mac6[4] = ip[1]; 3909 mac6[5] = ip[0]; 3910 ether_addr_copy(mac, mac6); 3911 } 3912 } 3913 3914 /** 3915 * irdma_attach_mcast - attach a qp to a multicast group 3916 * @ibqp: ptr to qp 3917 * @ibgid: pointer to global ID 3918 * @lid: local ID 3919 * 3920 * returns error status 3921 */ 3922 static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 3923 { 3924 struct irdma_qp *iwqp = to_iwqp(ibqp); 3925 struct irdma_device *iwdev = iwqp->iwdev; 3926 struct irdma_pci_f *rf = iwdev->rf; 3927 struct mc_table_list *mc_qht_elem; 3928 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 3929 unsigned long flags; 3930 u32 ip_addr[4] = {}; 3931 u32 mgn; 3932 u32 no_mgs; 3933 int ret = 0; 3934 bool ipv4; 3935 u16 vlan_id; 3936 union { 3937 struct sockaddr saddr; 3938 struct sockaddr_in saddr_in; 3939 struct sockaddr_in6 saddr_in6; 3940 } sgid_addr; 3941 unsigned char dmac[ETH_ALEN]; 3942 3943 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 3944 3945 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { 3946 irdma_copy_ip_ntohl(ip_addr, 3947 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 3948 irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL); 3949 ipv4 = false; 3950 ibdev_dbg(&iwdev->ibdev, 3951 "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, 3952 ip_addr); 3953 irdma_mcast_mac(ip_addr, dmac, false); 3954 } else { 3955 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 3956 ipv4 = true; 3957 vlan_id = irdma_get_vlan_ipv4(ip_addr); 3958 irdma_mcast_mac(ip_addr, dmac, true); 3959 ibdev_dbg(&iwdev->ibdev, 3960 "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n", 3961 ibqp->qp_num, ip_addr, dmac); 3962 } 3963 3964 spin_lock_irqsave(&rf->qh_list_lock, flags); 3965 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 3966 if (!mc_qht_elem) { 3967 struct irdma_dma_mem *dma_mem_mc; 3968 3969 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 3970 mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL); 3971 if (!mc_qht_elem) 3972 return -ENOMEM; 3973 3974 mc_qht_elem->mc_info.ipv4_valid = ipv4; 3975 memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, 3976 sizeof(mc_qht_elem->mc_info.dest_ip)); 3977 ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, 3978 &mgn, &rf->next_mcg); 3979 if (ret) { 3980 kfree(mc_qht_elem); 3981 return -ENOMEM; 3982 } 3983 3984 mc_qht_elem->mc_info.mgn = mgn; 3985 dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; 3986 dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX, 3987 IRDMA_HW_PAGE_SIZE); 3988 dma_mem_mc->va = dma_alloc_coherent(rf->hw.device, 3989 dma_mem_mc->size, 3990 &dma_mem_mc->pa, 3991 GFP_KERNEL); 3992 if (!dma_mem_mc->va) { 3993 irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); 3994 kfree(mc_qht_elem); 3995 return -ENOMEM; 3996 } 3997 3998 mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; 3999 memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, 4000 sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); 4001 mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; 4002 mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; 4003 if (vlan_id < VLAN_N_VID) 4004 mc_qht_elem->mc_grp_ctx.vlan_valid = true; 4005 mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->vsi.fcn_id; 4006 mc_qht_elem->mc_grp_ctx.qs_handle = 4007 iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; 4008 ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); 4009 4010 spin_lock_irqsave(&rf->qh_list_lock, flags); 4011 mcast_list_add(rf, mc_qht_elem); 4012 } else { 4013 if (mc_qht_elem->mc_grp_ctx.no_of_mgs == 4014 IRDMA_MAX_MGS_PER_CTX) { 4015 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4016 return -ENOMEM; 4017 } 4018 } 4019 4020 mcg_info.qp_id = iwqp->ibqp.qp_num; 4021 no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; 4022 irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4023 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4024 4025 /* Only if there is a change do we need to modify or create */ 4026 if (!no_mgs) { 4027 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4028 IRDMA_OP_MC_CREATE); 4029 } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4030 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4031 IRDMA_OP_MC_MODIFY); 4032 } else { 4033 return 0; 4034 } 4035 4036 if (ret) 4037 goto error; 4038 4039 return 0; 4040 4041 error: 4042 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4043 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4044 mcast_list_del(mc_qht_elem); 4045 dma_free_coherent(rf->hw.device, 4046 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4047 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4048 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4049 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4050 irdma_free_rsrc(rf, rf->allocated_mcgs, 4051 mc_qht_elem->mc_grp_ctx.mg_id); 4052 kfree(mc_qht_elem); 4053 } 4054 4055 return ret; 4056 } 4057 4058 /** 4059 * irdma_detach_mcast - detach a qp from a multicast group 4060 * @ibqp: ptr to qp 4061 * @ibgid: pointer to global ID 4062 * @lid: local ID 4063 * 4064 * returns error status 4065 */ 4066 static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 4067 { 4068 struct irdma_qp *iwqp = to_iwqp(ibqp); 4069 struct irdma_device *iwdev = iwqp->iwdev; 4070 struct irdma_pci_f *rf = iwdev->rf; 4071 u32 ip_addr[4] = {}; 4072 struct mc_table_list *mc_qht_elem; 4073 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 4074 int ret; 4075 unsigned long flags; 4076 union { 4077 struct sockaddr saddr; 4078 struct sockaddr_in saddr_in; 4079 struct sockaddr_in6 saddr_in6; 4080 } sgid_addr; 4081 4082 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 4083 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) 4084 irdma_copy_ip_ntohl(ip_addr, 4085 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4086 else 4087 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4088 4089 spin_lock_irqsave(&rf->qh_list_lock, flags); 4090 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 4091 if (!mc_qht_elem) { 4092 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4093 ibdev_dbg(&iwdev->ibdev, 4094 "VERBS: address not found MCG\n"); 4095 return 0; 4096 } 4097 4098 mcg_info.qp_id = iwqp->ibqp.qp_num; 4099 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4100 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4101 mcast_list_del(mc_qht_elem); 4102 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4103 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4104 IRDMA_OP_MC_DESTROY); 4105 if (ret) { 4106 ibdev_dbg(&iwdev->ibdev, 4107 "VERBS: failed MC_DESTROY MCG\n"); 4108 spin_lock_irqsave(&rf->qh_list_lock, flags); 4109 mcast_list_add(rf, mc_qht_elem); 4110 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4111 return -EAGAIN; 4112 } 4113 4114 dma_free_coherent(rf->hw.device, 4115 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4116 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4117 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4118 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4119 irdma_free_rsrc(rf, rf->allocated_mcgs, 4120 mc_qht_elem->mc_grp_ctx.mg_id); 4121 kfree(mc_qht_elem); 4122 } else { 4123 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4124 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4125 IRDMA_OP_MC_MODIFY); 4126 if (ret) { 4127 ibdev_dbg(&iwdev->ibdev, 4128 "VERBS: failed Modify MCG\n"); 4129 return ret; 4130 } 4131 } 4132 4133 return 0; 4134 } 4135 4136 /** 4137 * irdma_create_ah - create address handle 4138 * @ibah: address handle 4139 * @attr: address handle attributes 4140 * @udata: User data 4141 * 4142 * returns 0 on success, error otherwise 4143 */ 4144 static int irdma_create_ah(struct ib_ah *ibah, 4145 struct rdma_ah_init_attr *attr, 4146 struct ib_udata *udata) 4147 { 4148 struct irdma_pd *pd = to_iwpd(ibah->pd); 4149 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 4150 struct rdma_ah_attr *ah_attr = attr->ah_attr; 4151 const struct ib_gid_attr *sgid_attr; 4152 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 4153 struct irdma_pci_f *rf = iwdev->rf; 4154 struct irdma_sc_ah *sc_ah; 4155 u32 ah_id = 0; 4156 struct irdma_ah_info *ah_info; 4157 struct irdma_create_ah_resp uresp; 4158 union { 4159 struct sockaddr saddr; 4160 struct sockaddr_in saddr_in; 4161 struct sockaddr_in6 saddr_in6; 4162 } sgid_addr, dgid_addr; 4163 int err; 4164 u8 dmac[ETH_ALEN]; 4165 4166 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah_id, 4167 &rf->next_ah); 4168 if (err) 4169 return err; 4170 4171 ah->pd = pd; 4172 sc_ah = &ah->sc_ah; 4173 sc_ah->ah_info.ah_idx = ah_id; 4174 sc_ah->ah_info.vsi = &iwdev->vsi; 4175 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 4176 ah->sgid_index = ah_attr->grh.sgid_index; 4177 sgid_attr = ah_attr->grh.sgid_attr; 4178 memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid)); 4179 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid_attr->gid); 4180 rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid); 4181 ah->av.attrs = *ah_attr; 4182 ah->av.net_type = rdma_gid_attr_network_type(sgid_attr); 4183 ah->av.sgid_addr.saddr = sgid_addr.saddr; 4184 ah->av.dgid_addr.saddr = dgid_addr.saddr; 4185 ah_info = &sc_ah->ah_info; 4186 ah_info->ah_idx = ah_id; 4187 ah_info->pd_idx = pd->sc_pd.pd_id; 4188 if (ah_attr->ah_flags & IB_AH_GRH) { 4189 ah_info->flow_label = ah_attr->grh.flow_label; 4190 ah_info->hop_ttl = ah_attr->grh.hop_limit; 4191 ah_info->tc_tos = ah_attr->grh.traffic_class; 4192 } 4193 4194 ether_addr_copy(dmac, ah_attr->roce.dmac); 4195 if (rdma_gid_attr_network_type(sgid_attr) == RDMA_NETWORK_IPV4) { 4196 ah_info->ipv4_valid = true; 4197 ah_info->dest_ip_addr[0] = 4198 ntohl(dgid_addr.saddr_in.sin_addr.s_addr); 4199 ah_info->src_ip_addr[0] = 4200 ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4201 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 4202 ah_info->dest_ip_addr[0]); 4203 if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) { 4204 ah_info->do_lpbk = true; 4205 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true); 4206 } 4207 } else { 4208 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 4209 dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4210 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 4211 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4212 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 4213 ah_info->dest_ip_addr); 4214 if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr)) { 4215 ah_info->do_lpbk = true; 4216 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false); 4217 } 4218 } 4219 4220 err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag, 4221 ah_info->mac_addr); 4222 if (err) 4223 goto error; 4224 4225 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, 4226 ah_info->ipv4_valid, dmac); 4227 4228 if (ah_info->dst_arpindex == -1) { 4229 err = -EINVAL; 4230 goto error; 4231 } 4232 4233 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb) 4234 ah_info->vlan_tag = 0; 4235 4236 if (ah_info->vlan_tag < VLAN_N_VID) { 4237 ah_info->insert_vlan_tag = true; 4238 ah_info->vlan_tag |= 4239 rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; 4240 } 4241 4242 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, 4243 attr->flags & RDMA_CREATE_AH_SLEEPABLE, 4244 irdma_gsi_ud_qp_ah_cb, sc_ah); 4245 4246 if (err) { 4247 ibdev_dbg(&iwdev->ibdev, 4248 "VERBS: CQP-OP Create AH fail"); 4249 goto error; 4250 } 4251 4252 if (!(attr->flags & RDMA_CREATE_AH_SLEEPABLE)) { 4253 int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD; 4254 4255 do { 4256 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 4257 mdelay(1); 4258 } while (!sc_ah->ah_info.ah_valid && --cnt); 4259 4260 if (!cnt) { 4261 ibdev_dbg(&iwdev->ibdev, 4262 "VERBS: CQP create AH timed out"); 4263 err = -ETIMEDOUT; 4264 goto error; 4265 } 4266 } 4267 4268 if (udata) { 4269 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 4270 err = ib_copy_to_udata(udata, &uresp, 4271 min(sizeof(uresp), udata->outlen)); 4272 } 4273 return 0; 4274 4275 error: 4276 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); 4277 4278 return err; 4279 } 4280 4281 /** 4282 * irdma_destroy_ah - Destroy address handle 4283 * @ibah: pointer to address handle 4284 * @ah_flags: flags for sleepable 4285 */ 4286 static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 4287 { 4288 struct irdma_device *iwdev = to_iwdev(ibah->device); 4289 struct irdma_ah *ah = to_iwah(ibah); 4290 4291 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 4292 false, NULL, ah); 4293 4294 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 4295 ah->sc_ah.ah_info.ah_idx); 4296 4297 return 0; 4298 } 4299 4300 /** 4301 * irdma_query_ah - Query address handle 4302 * @ibah: pointer to address handle 4303 * @ah_attr: address handle attributes 4304 */ 4305 static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 4306 { 4307 struct irdma_ah *ah = to_iwah(ibah); 4308 4309 memset(ah_attr, 0, sizeof(*ah_attr)); 4310 if (ah->av.attrs.ah_flags & IB_AH_GRH) { 4311 ah_attr->ah_flags = IB_AH_GRH; 4312 ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; 4313 ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; 4314 ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; 4315 ah_attr->grh.sgid_index = ah->sgid_index; 4316 ah_attr->grh.sgid_index = ah->sgid_index; 4317 memcpy(&ah_attr->grh.dgid, &ah->dgid, 4318 sizeof(ah_attr->grh.dgid)); 4319 } 4320 4321 return 0; 4322 } 4323 4324 static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, 4325 u32 port_num) 4326 { 4327 return IB_LINK_LAYER_ETHERNET; 4328 } 4329 4330 static __be64 irdma_mac_to_guid(struct net_device *ndev) 4331 { 4332 unsigned char *mac = ndev->dev_addr; 4333 __be64 guid; 4334 unsigned char *dst = (unsigned char *)&guid; 4335 4336 dst[0] = mac[0] ^ 2; 4337 dst[1] = mac[1]; 4338 dst[2] = mac[2]; 4339 dst[3] = 0xff; 4340 dst[4] = 0xfe; 4341 dst[5] = mac[3]; 4342 dst[6] = mac[4]; 4343 dst[7] = mac[5]; 4344 4345 return guid; 4346 } 4347 4348 static const struct ib_device_ops irdma_roce_dev_ops = { 4349 .attach_mcast = irdma_attach_mcast, 4350 .create_ah = irdma_create_ah, 4351 .create_user_ah = irdma_create_ah, 4352 .destroy_ah = irdma_destroy_ah, 4353 .detach_mcast = irdma_detach_mcast, 4354 .get_link_layer = irdma_get_link_layer, 4355 .get_port_immutable = irdma_roce_port_immutable, 4356 .modify_qp = irdma_modify_qp_roce, 4357 .query_ah = irdma_query_ah, 4358 .query_pkey = irdma_query_pkey, 4359 }; 4360 4361 static const struct ib_device_ops irdma_iw_dev_ops = { 4362 .modify_qp = irdma_modify_qp, 4363 .get_port_immutable = irdma_iw_port_immutable, 4364 .query_gid = irdma_query_gid, 4365 }; 4366 4367 static const struct ib_device_ops irdma_dev_ops = { 4368 .owner = THIS_MODULE, 4369 .driver_id = RDMA_DRIVER_IRDMA, 4370 .uverbs_abi_ver = IRDMA_ABI_VER, 4371 4372 .alloc_hw_port_stats = irdma_alloc_hw_port_stats, 4373 .alloc_mr = irdma_alloc_mr, 4374 .alloc_mw = irdma_alloc_mw, 4375 .alloc_pd = irdma_alloc_pd, 4376 .alloc_ucontext = irdma_alloc_ucontext, 4377 .create_cq = irdma_create_cq, 4378 .create_qp = irdma_create_qp, 4379 .dealloc_driver = irdma_ib_dealloc_device, 4380 .dealloc_mw = irdma_dealloc_mw, 4381 .dealloc_pd = irdma_dealloc_pd, 4382 .dealloc_ucontext = irdma_dealloc_ucontext, 4383 .dereg_mr = irdma_dereg_mr, 4384 .destroy_cq = irdma_destroy_cq, 4385 .destroy_qp = irdma_destroy_qp, 4386 .disassociate_ucontext = irdma_disassociate_ucontext, 4387 .get_dev_fw_str = irdma_get_dev_fw_str, 4388 .get_dma_mr = irdma_get_dma_mr, 4389 .get_hw_stats = irdma_get_hw_stats, 4390 .map_mr_sg = irdma_map_mr_sg, 4391 .mmap = irdma_mmap, 4392 .mmap_free = irdma_mmap_free, 4393 .poll_cq = irdma_poll_cq, 4394 .post_recv = irdma_post_recv, 4395 .post_send = irdma_post_send, 4396 .query_device = irdma_query_device, 4397 .query_port = irdma_query_port, 4398 .query_qp = irdma_query_qp, 4399 .reg_user_mr = irdma_reg_user_mr, 4400 .req_notify_cq = irdma_req_notify_cq, 4401 .resize_cq = irdma_resize_cq, 4402 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd), 4403 INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext), 4404 INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), 4405 INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), 4406 INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), 4407 }; 4408 4409 /** 4410 * irdma_init_roce_device - initialization of roce rdma device 4411 * @iwdev: irdma device 4412 */ 4413 static void irdma_init_roce_device(struct irdma_device *iwdev) 4414 { 4415 iwdev->ibdev.node_type = RDMA_NODE_IB_CA; 4416 iwdev->ibdev.node_guid = irdma_mac_to_guid(iwdev->netdev); 4417 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); 4418 } 4419 4420 /** 4421 * irdma_init_iw_device - initialization of iwarp rdma device 4422 * @iwdev: irdma device 4423 */ 4424 static int irdma_init_iw_device(struct irdma_device *iwdev) 4425 { 4426 struct net_device *netdev = iwdev->netdev; 4427 4428 iwdev->ibdev.node_type = RDMA_NODE_RNIC; 4429 ether_addr_copy((u8 *)&iwdev->ibdev.node_guid, netdev->dev_addr); 4430 iwdev->ibdev.ops.iw_add_ref = irdma_qp_add_ref; 4431 iwdev->ibdev.ops.iw_rem_ref = irdma_qp_rem_ref; 4432 iwdev->ibdev.ops.iw_get_qp = irdma_get_qp; 4433 iwdev->ibdev.ops.iw_connect = irdma_connect; 4434 iwdev->ibdev.ops.iw_accept = irdma_accept; 4435 iwdev->ibdev.ops.iw_reject = irdma_reject; 4436 iwdev->ibdev.ops.iw_create_listen = irdma_create_listen; 4437 iwdev->ibdev.ops.iw_destroy_listen = irdma_destroy_listen; 4438 memcpy(iwdev->ibdev.iw_ifname, netdev->name, 4439 sizeof(iwdev->ibdev.iw_ifname)); 4440 ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops); 4441 4442 return 0; 4443 } 4444 4445 /** 4446 * irdma_init_rdma_device - initialization of rdma device 4447 * @iwdev: irdma device 4448 */ 4449 static int irdma_init_rdma_device(struct irdma_device *iwdev) 4450 { 4451 struct pci_dev *pcidev = iwdev->rf->pcidev; 4452 int ret; 4453 4454 if (iwdev->roce_mode) { 4455 irdma_init_roce_device(iwdev); 4456 } else { 4457 ret = irdma_init_iw_device(iwdev); 4458 if (ret) 4459 return ret; 4460 } 4461 iwdev->ibdev.phys_port_cnt = 1; 4462 iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; 4463 iwdev->ibdev.dev.parent = &pcidev->dev; 4464 ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops); 4465 4466 return 0; 4467 } 4468 4469 /** 4470 * irdma_port_ibevent - indicate port event 4471 * @iwdev: irdma device 4472 */ 4473 void irdma_port_ibevent(struct irdma_device *iwdev) 4474 { 4475 struct ib_event event; 4476 4477 event.device = &iwdev->ibdev; 4478 event.element.port_num = 1; 4479 event.event = 4480 iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; 4481 ib_dispatch_event(&event); 4482 } 4483 4484 /** 4485 * irdma_ib_unregister_device - unregister rdma device from IB 4486 * core 4487 * @iwdev: irdma device 4488 */ 4489 void irdma_ib_unregister_device(struct irdma_device *iwdev) 4490 { 4491 iwdev->iw_status = 0; 4492 irdma_port_ibevent(iwdev); 4493 ib_unregister_device(&iwdev->ibdev); 4494 } 4495 4496 /** 4497 * irdma_ib_register_device - register irdma device to IB core 4498 * @iwdev: irdma device 4499 */ 4500 int irdma_ib_register_device(struct irdma_device *iwdev) 4501 { 4502 int ret; 4503 4504 ret = irdma_init_rdma_device(iwdev); 4505 if (ret) 4506 return ret; 4507 4508 ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1); 4509 if (ret) 4510 goto error; 4511 dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX); 4512 ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device); 4513 if (ret) 4514 goto error; 4515 4516 iwdev->iw_status = 1; 4517 irdma_port_ibevent(iwdev); 4518 4519 return 0; 4520 4521 error: 4522 if (ret) 4523 ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n"); 4524 4525 return ret; 4526 } 4527 4528 /** 4529 * irdma_ib_dealloc_device 4530 * @ibdev: ib device 4531 * 4532 * callback from ibdev dealloc_driver to deallocate resources 4533 * unber irdma device 4534 */ 4535 void irdma_ib_dealloc_device(struct ib_device *ibdev) 4536 { 4537 struct irdma_device *iwdev = to_iwdev(ibdev); 4538 4539 irdma_rt_deinit_hw(iwdev); 4540 irdma_ctrl_deinit_hw(iwdev->rf); 4541 kfree(iwdev->rf); 4542 } 4543