1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2018 - 2026 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "irdma_main.h" 36 37 #define IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000) 38 39 static u16 kc_rdma_flow_label_to_udp_sport(u32 fl) { 40 u32 fl_low = fl & 0x03FFF; 41 u32 fl_high = fl & 0xFC000; 42 43 fl_low ^= fl_high >> 14; 44 45 return (u16)(fl_low | IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN); 46 } 47 48 #define IRDMA_GRH_FLOWLABEL_MASK (0x000FFFFF) 49 50 static u32 kc_rdma_calc_flow_label(u32 lqpn, u32 rqpn) { 51 u64 fl = (u64)lqpn * rqpn; 52 53 fl ^= fl >> 20; 54 fl ^= fl >> 40; 55 56 return (u32)(fl & IRDMA_GRH_FLOWLABEL_MASK); 57 } 58 59 u16 60 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) 61 { 62 if (!fl) 63 fl = kc_rdma_calc_flow_label(lqpn, rqpn); 64 return kc_rdma_flow_label_to_udp_sport(fl); 65 } 66 67 void 68 irdma_get_dev_fw_str(struct ib_device *dev, 69 char *str, 70 size_t str_len) 71 { 72 struct irdma_device *iwdev = to_iwdev(dev); 73 74 snprintf(str, str_len, "%u.%u", 75 irdma_fw_major_ver(&iwdev->rf->sc_dev), 76 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 77 } 78 79 int 80 irdma_add_gid(struct ib_device *device, 81 u8 port_num, 82 unsigned int index, 83 const union ib_gid *gid, 84 const struct ib_gid_attr *attr, 85 void **context) 86 { 87 return 0; 88 } 89 90 int 91 irdma_del_gid(struct ib_device *device, 92 u8 port_num, 93 unsigned int index, 94 void **context) 95 { 96 return 0; 97 } 98 99 /** 100 * irdma_alloc_mr - register stag for fast memory registration 101 * @pd: ibpd pointer 102 * @mr_type: memory for stag registrion 103 * @max_num_sg: man number of pages 104 * @udata: user data 105 */ 106 struct ib_mr * 107 irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 108 u32 max_num_sg, struct ib_udata *udata) 109 { 110 struct irdma_device *iwdev = to_iwdev(pd->device); 111 struct irdma_pble_alloc *palloc; 112 struct irdma_pbl *iwpbl; 113 struct irdma_mr *iwmr; 114 int status; 115 u32 stag; 116 int err_code = -ENOMEM; 117 118 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 119 if (!iwmr) 120 return ERR_PTR(-ENOMEM); 121 122 stag = irdma_create_stag(iwdev); 123 if (!stag) { 124 err_code = -ENOMEM; 125 goto err; 126 } 127 128 iwmr->stag = stag; 129 iwmr->ibmr.rkey = stag; 130 iwmr->ibmr.lkey = stag; 131 iwmr->ibmr.pd = pd; 132 iwmr->ibmr.device = pd->device; 133 iwpbl = &iwmr->iwpbl; 134 iwpbl->iwmr = iwmr; 135 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 136 palloc = &iwpbl->pble_alloc; 137 iwmr->page_cnt = max_num_sg; 138 /* Assume system PAGE_SIZE as the sg page sizes are unknown. */ 139 iwmr->len = max_num_sg * PAGE_SIZE; 140 status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 141 false); 142 if (status) 143 goto err_get_pble; 144 145 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 146 if (err_code) 147 goto err_alloc_stag; 148 149 iwpbl->pbl_allocated = true; 150 151 return &iwmr->ibmr; 152 err_alloc_stag: 153 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 154 err_get_pble: 155 irdma_free_stag(iwdev, stag); 156 err: 157 kfree(iwmr); 158 159 return ERR_PTR(err_code); 160 } 161 162 #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8) 163 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) 164 /** 165 * irdma_alloc_ucontext - Allocate the user context data structure 166 * @uctx: context 167 * @udata: user data 168 * 169 * This keeps track of all objects associated with a particular 170 * user-mode client. 171 */ 172 int 173 irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 174 { 175 struct ib_device *ibdev = uctx->device; 176 struct irdma_device *iwdev = to_iwdev(ibdev); 177 struct irdma_alloc_ucontext_req req = {0}; 178 struct irdma_alloc_ucontext_resp uresp = {0}; 179 struct irdma_ucontext *ucontext = to_ucontext(uctx); 180 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 181 182 if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || 183 udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) 184 return -EINVAL; 185 186 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 187 return -EINVAL; 188 189 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 190 goto ver_error; 191 192 ucontext->iwdev = iwdev; 193 ucontext->abi_ver = req.userspace_ver; 194 195 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR) 196 ucontext->use_raw_attrs = true; 197 198 /* GEN_1 support for libi40iw */ 199 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { 200 if (uk_attrs->hw_rev != IRDMA_GEN_1) 201 return -EOPNOTSUPP; 202 203 ucontext->legacy_mode = true; 204 uresp.max_qps = iwdev->rf->max_qp; 205 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 206 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 207 uresp.kernel_ver = req.userspace_ver; 208 if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) 209 return -EFAULT; 210 } else { 211 u64 bar_off; 212 213 uresp.kernel_ver = IRDMA_ABI_VER; 214 uresp.feature_flags = uk_attrs->feature_flags; 215 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 216 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 217 uresp.max_hw_inline = uk_attrs->max_hw_inline; 218 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 219 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 220 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 221 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 222 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 223 uresp.hw_rev = uk_attrs->hw_rev; 224 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; 225 uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size; 226 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE; 227 228 bar_off = 229 (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 230 ucontext->db_mmap_entry = 231 irdma_user_mmap_entry_insert(ucontext, bar_off, 232 IRDMA_MMAP_IO_NC, 233 &uresp.db_mmap_key); 234 if (!ucontext->db_mmap_entry) { 235 return -ENOMEM; 236 } 237 238 if (ib_copy_to_udata(udata, &uresp, 239 min(sizeof(uresp), udata->outlen))) { 240 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 241 return -EFAULT; 242 } 243 } 244 245 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 246 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 247 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 248 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 249 INIT_LIST_HEAD(&ucontext->vma_list); 250 mutex_init(&ucontext->vma_list_mutex); 251 252 return 0; 253 254 ver_error: 255 irdma_dev_err(&iwdev->ibdev, 256 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 257 req.userspace_ver, IRDMA_ABI_VER); 258 return -EINVAL; 259 } 260 261 262 /** 263 * irdma_dealloc_ucontext - deallocate the user context data structure 264 * @context: user context created during alloc 265 */ 266 void 267 irdma_dealloc_ucontext(struct ib_ucontext *context) 268 { 269 struct irdma_ucontext *ucontext = to_ucontext(context); 270 271 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 272 273 return; 274 } 275 276 277 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) 278 /** 279 * irdma_alloc_pd - allocate protection domain 280 * @pd: protection domain 281 * @udata: user data 282 */ 283 int 284 irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 285 { 286 struct irdma_pd *iwpd = to_iwpd(pd); 287 struct irdma_device *iwdev = to_iwdev(pd->device); 288 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 289 struct irdma_pci_f *rf = iwdev->rf; 290 struct irdma_alloc_pd_resp uresp = {0}; 291 struct irdma_sc_pd *sc_pd; 292 u32 pd_id = 0; 293 int err; 294 295 if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) 296 return -EINVAL; 297 298 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 299 &rf->next_pd); 300 if (err) 301 return err; 302 303 sc_pd = &iwpd->sc_pd; 304 if (udata) { 305 struct irdma_ucontext *ucontext = 306 rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); 307 308 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 309 uresp.pd_id = pd_id; 310 if (ib_copy_to_udata(udata, &uresp, 311 min(sizeof(uresp), udata->outlen))) { 312 err = -EFAULT; 313 goto error; 314 } 315 } else { 316 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 317 } 318 319 spin_lock_init(&iwpd->udqp_list_lock); 320 INIT_LIST_HEAD(&iwpd->udqp_list); 321 322 return 0; 323 324 error: 325 326 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 327 328 return err; 329 } 330 331 332 void 333 irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 334 { 335 struct irdma_pd *iwpd = to_iwpd(ibpd); 336 struct irdma_device *iwdev = to_iwdev(ibpd->device); 337 338 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 339 } 340 341 342 343 /** 344 * irdma_find_qp_update_qs - update QS handle for UD QPs 345 * @rf: RDMA PCI function 346 * @pd: protection domain object 347 * @user_pri: selected user priority 348 */ 349 static void 350 irdma_find_qp_update_qs(struct irdma_pci_f *rf, 351 struct irdma_pd *pd, u8 user_pri) 352 { 353 struct irdma_qp *iwqp; 354 struct list_head *tmp_node, *list_node; 355 struct irdma_udqs_work *work; 356 unsigned long flags; 357 bool qs_change; 358 359 spin_lock_irqsave(&pd->udqp_list_lock, flags); 360 list_for_each_safe(list_node, tmp_node, &pd->udqp_list) { 361 qs_change = true; 362 iwqp = list_entry(list_node, struct irdma_qp, ud_list_elem); 363 irdma_qp_add_ref(&iwqp->ibqp); 364 /* check if qs_handle needs to be changed */ 365 if (iwqp->sc_qp.qs_handle == iwqp->sc_qp.vsi->qos[user_pri].qs_handle) { 366 if (iwqp->ctx_info.user_pri == user_pri) { 367 /* qs_handle and user_pri don't change */ 368 irdma_qp_rem_ref(&iwqp->ibqp); 369 continue; 370 } 371 qs_change = false; 372 } 373 /* perform qp qos change */ 374 work = kzalloc(sizeof(*work), GFP_ATOMIC); 375 if (!work) { 376 irdma_qp_rem_ref(&iwqp->ibqp); 377 spin_unlock_irqrestore(&pd->udqp_list_lock, flags); 378 return; 379 } 380 work->iwqp = iwqp; 381 work->user_prio = user_pri; 382 work->qs_change = qs_change; 383 INIT_WORK(&work->work, irdma_udqp_qs_worker); 384 if (qs_change) 385 irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND); 386 queue_work(rf->iwdev->cleanup_wq, &work->work); 387 } 388 spin_unlock_irqrestore(&pd->udqp_list_lock, flags); 389 } 390 391 static void 392 irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info, 393 const struct ib_gid_attr *sgid_attr, 394 union irdma_sockaddr *sgid_addr, 395 union irdma_sockaddr *dgid_addr, 396 u8 *dmac, u8 net_type) 397 { 398 if (net_type == RDMA_NETWORK_IPV4) { 399 ah_info->ipv4_valid = true; 400 ah_info->dest_ip_addr[0] = 401 ntohl(dgid_addr->saddr_in.sin_addr.s_addr); 402 ah_info->src_ip_addr[0] = 403 ntohl(sgid_addr->saddr_in.sin_addr.s_addr); 404 CURVNET_SET_QUIET(vnet); 405 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 406 ah_info->dest_ip_addr[0]); 407 CURVNET_RESTORE(); 408 if (ipv4_is_multicast(dgid_addr->saddr_in.sin_addr.s_addr)) { 409 irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac); 410 } 411 } else { 412 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 413 dgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32); 414 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 415 sgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32); 416 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 417 ah_info->dest_ip_addr); 418 if (rdma_is_multicast_addr(&dgid_addr->saddr_in6.sin6_addr)) { 419 irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac); 420 } 421 } 422 } 423 424 static inline u8 irdma_roce_get_vlan_prio(if_t ndev, u8 prio) 425 { 426 return prio; 427 } 428 429 static int 430 irdma_create_ah_vlan_tag(struct irdma_device *iwdev, 431 struct irdma_pd *pd, 432 struct irdma_ah_info *ah_info, 433 const struct ib_gid_attr *sgid_attr, 434 u8 *dmac) 435 { 436 u16 vlan_prio; 437 438 if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev)) 439 ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev); 440 else 441 ah_info->vlan_tag = VLAN_N_VID; 442 443 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, dmac); 444 445 if (ah_info->dst_arpindex == -1) 446 return -EINVAL; 447 448 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) 449 ah_info->vlan_tag = 0; 450 451 if (ah_info->vlan_tag < VLAN_N_VID) { 452 ah_info->insert_vlan_tag = true; 453 vlan_prio = (u16)irdma_roce_get_vlan_prio(sgid_attr->ndev, 454 rt_tos2priority(ah_info->tc_tos)); 455 ah_info->vlan_tag |= vlan_prio << VLAN_PRIO_SHIFT; 456 irdma_find_qp_update_qs(iwdev->rf, pd, vlan_prio); 457 } 458 if (iwdev->roce_dcqcn_en) { 459 ah_info->tc_tos &= ~ECN_CODE_PT_MASK; 460 ah_info->tc_tos |= ECN_CODE_PT_VAL; 461 } 462 463 return 0; 464 } 465 466 static int 467 irdma_create_ah_wait(struct irdma_pci_f *rf, 468 struct irdma_sc_ah *sc_ah, bool sleep) 469 { 470 int ret; 471 472 if (!sleep) { 473 bool timeout = false; 474 u64 start = get_jiffies_64(); 475 u64 completed_ops = atomic64_read(&rf->sc_dev.cqp->completed_ops); 476 struct irdma_cqp_request *cqp_request = 477 sc_ah->ah_info.cqp_request; 478 const u64 timeout_jiffies = 479 msecs_to_jiffies(rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms * 480 CQP_TIMEOUT_THRESHOLD); 481 482 /* 483 * NOTE: irdma_check_cqp_progress is not used here because it relies on a notion of a cycle count, but 484 * we want to avoid unnecessary delays. We are in an atomic context here, so we might as well check in 485 * a tight loop. 486 */ 487 while (!READ_ONCE(cqp_request->request_done)) { 488 u64 tmp; 489 u64 curr_jiffies; 490 491 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 492 493 curr_jiffies = get_jiffies_64(); 494 tmp = atomic64_read(&rf->sc_dev.cqp->completed_ops); 495 if (tmp != completed_ops) { 496 /* CQP is progressing. Reset timer. */ 497 completed_ops = tmp; 498 start = curr_jiffies; 499 } 500 501 if ((curr_jiffies - start) > timeout_jiffies) { 502 timeout = true; 503 break; 504 } 505 } 506 507 if (!timeout && !cqp_request->compl_info.op_ret_val) { 508 irdma_put_cqp_request(&rf->cqp, cqp_request); 509 sc_ah->ah_info.ah_valid = true; 510 } else { 511 ret = timeout ? -ETIMEDOUT : -EINVAL; 512 irdma_dev_err(&rf->iwdev->ibdev, "CQP create AH error ret = %d opt_ret_val = %d", 513 ret, cqp_request->compl_info.op_ret_val); 514 irdma_put_cqp_request(&rf->cqp, cqp_request); 515 if (timeout && !rf->reset) { 516 rf->reset = true; 517 rf->gen_ops.request_reset(rf); 518 } 519 return ret; 520 } 521 } 522 523 return 0; 524 } 525 526 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) 527 528 static int 529 irdma_create_sleepable_ah(struct ib_ah *ib_ah, 530 struct ib_ah_attr *attr, u32 flags, 531 struct ib_udata *udata) 532 { 533 struct irdma_pd *pd = to_iwpd(ib_ah->pd); 534 struct irdma_ah *ah = container_of(ib_ah, struct irdma_ah, ibah); 535 struct irdma_device *iwdev = to_iwdev(ib_ah->pd->device); 536 union ib_gid sgid; 537 struct ib_gid_attr sgid_attr; 538 struct irdma_pci_f *rf = iwdev->rf; 539 struct irdma_sc_ah *sc_ah; 540 u32 ah_id = 0; 541 struct irdma_ah_info *ah_info; 542 struct irdma_create_ah_resp uresp = {}; 543 union irdma_sockaddr sgid_addr, dgid_addr; 544 int err; 545 u8 dmac[ETHER_ADDR_LEN]; 546 bool sleep = (flags & RDMA_CREATE_AH_SLEEPABLE) != 0; 547 548 if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) 549 return -EINVAL; 550 551 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, 552 rf->max_ah, &ah_id, &rf->next_ah); 553 554 if (err) 555 return err; 556 557 ah->pd = pd; 558 sc_ah = &ah->sc_ah; 559 sc_ah->ah_info.ah_idx = ah_id; 560 sc_ah->ah_info.vsi = &iwdev->vsi; 561 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 562 ah->sgid_index = attr->grh.sgid_index; 563 memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid)); 564 rcu_read_lock(); 565 err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num, 566 attr->grh.sgid_index, &sgid, &sgid_attr); 567 rcu_read_unlock(); 568 if (err) { 569 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, 570 "GID lookup at idx=%d with port=%d failed\n", 571 attr->grh.sgid_index, attr->port_num); 572 err = -EINVAL; 573 goto err_gid_l2; 574 } 575 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); 576 rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); 577 ah->av.attrs = *attr; 578 ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); 579 580 if (sgid_attr.ndev) 581 dev_put(sgid_attr.ndev); 582 583 ah_info = &sc_ah->ah_info; 584 ah_info->ah_idx = ah_id; 585 ah_info->pd_idx = pd->sc_pd.pd_id; 586 ether_addr_copy(ah_info->mac_addr, if_getlladdr(iwdev->netdev)); 587 588 if (attr->ah_flags & IB_AH_GRH) { 589 ah_info->flow_label = attr->grh.flow_label; 590 ah_info->hop_ttl = attr->grh.hop_limit; 591 ah_info->tc_tos = attr->grh.traffic_class; 592 } 593 594 ether_addr_copy(dmac, attr->dmac); 595 596 irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr, 597 dmac, ah->av.net_type); 598 599 err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac); 600 if (err) 601 goto err_gid_l2; 602 603 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, 604 sleep, NULL, sc_ah); 605 if (err) { 606 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP-OP Create AH fail"); 607 goto err_gid_l2; 608 } 609 610 err = irdma_create_ah_wait(rf, sc_ah, sleep); 611 if (err) 612 goto err_gid_l2; 613 614 if (udata) { 615 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 616 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); 617 if (err) { 618 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, 619 IRDMA_OP_AH_DESTROY, false, NULL, ah); 620 goto err_gid_l2; 621 } 622 } 623 624 return 0; 625 err_gid_l2: 626 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); 627 628 return err; 629 } 630 631 /** 632 * irdma_create_ah - create address handle 633 * @ib_ah: ptr to AH 634 * @attr: address handle attributes 635 * @flags: AH flags to wait 636 * @udata: user data 637 * 638 * returns 0 on success, error otherwise 639 */ 640 int 641 irdma_create_ah(struct ib_ah *ib_ah, 642 struct ib_ah_attr *attr, u32 flags, 643 struct ib_udata *udata) 644 { 645 return irdma_create_sleepable_ah(ib_ah, attr, flags, udata); 646 } 647 648 void 649 irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr) 650 { 651 ether_addr_copy(dmac, attr->dmac); 652 } 653 654 int 655 irdma_create_ah_stub(struct ib_ah *ib_ah, 656 struct ib_ah_attr *attr, u32 flags, 657 struct ib_udata *udata) 658 { 659 return -ENOSYS; 660 } 661 662 void 663 irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags) 664 { 665 return; 666 } 667 668 669 /** 670 * irdma_free_qp_rsrc - free up memory resources for qp 671 * @iwqp: qp ptr (user or kernel) 672 */ 673 void 674 irdma_free_qp_rsrc(struct irdma_qp *iwqp) 675 { 676 struct irdma_device *iwdev = iwqp->iwdev; 677 struct irdma_pci_f *rf = iwdev->rf; 678 u32 qp_num = iwqp->ibqp.qp_num; 679 680 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); 681 irdma_dealloc_push_page(rf, iwqp); 682 if (iwqp->sc_qp.vsi) { 683 irdma_qp_rem_qos(&iwqp->sc_qp); 684 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, 685 iwqp->sc_qp.user_pri); 686 } 687 688 if (qp_num > 2) 689 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 690 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem); 691 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem); 692 kfree(iwqp->kqp.sig_trk_mem); 693 iwqp->kqp.sig_trk_mem = NULL; 694 kfree(iwqp->kqp.sq_wrid_mem); 695 kfree(iwqp->kqp.rq_wrid_mem); 696 kfree(iwqp->sg_list); 697 kfree(iwqp); 698 } 699 700 /** 701 * irdma_create_qp - create qp 702 * @ibpd: ptr of pd 703 * @init_attr: attributes for qp 704 * @udata: user data for create qp 705 */ 706 struct ib_qp * 707 irdma_create_qp(struct ib_pd *ibpd, 708 struct ib_qp_init_attr *init_attr, 709 struct ib_udata *udata) 710 { 711 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx) 712 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd) 713 struct irdma_pd *iwpd = to_iwpd(ibpd); 714 struct irdma_device *iwdev = to_iwdev(ibpd->device); 715 struct irdma_pci_f *rf = iwdev->rf; 716 struct irdma_qp *iwqp; 717 struct irdma_create_qp_resp uresp = {0}; 718 u32 qp_num = 0; 719 int ret; 720 int err_code; 721 struct irdma_sc_qp *qp; 722 struct irdma_sc_dev *dev = &rf->sc_dev; 723 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 724 struct irdma_qp_init_info init_info = {{0}}; 725 struct irdma_qp_host_ctx_info *ctx_info; 726 u32 next_qp = 0; 727 unsigned long flags; 728 729 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 730 if (err_code) 731 return ERR_PTR(err_code); 732 733 if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN || 734 udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN)) 735 return ERR_PTR(-EINVAL); 736 737 init_info.vsi = &iwdev->vsi; 738 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 739 init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr; 740 init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr; 741 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 742 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 743 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 744 745 iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); 746 if (!iwqp) 747 return ERR_PTR(-ENOMEM); 748 749 iwqp->sg_list = kcalloc(uk_attrs->max_hw_wq_frags, sizeof(*iwqp->sg_list), 750 GFP_KERNEL); 751 if (!iwqp->sg_list) { 752 kfree(iwqp); 753 return ERR_PTR(-ENOMEM); 754 } 755 756 qp = &iwqp->sc_qp; 757 qp->qp_uk.back_qp = iwqp; 758 qp->qp_uk.lock = &iwqp->lock; 759 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 760 761 iwqp->iwdev = iwdev; 762 iwqp->q2_ctx_mem.size = IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE; 763 iwqp->q2_ctx_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->q2_ctx_mem, 764 iwqp->q2_ctx_mem.size, 765 256); 766 if (!iwqp->q2_ctx_mem.va) { 767 kfree(iwqp->sg_list); 768 kfree(iwqp); 769 return ERR_PTR(-ENOMEM); 770 } 771 772 init_info.q2 = iwqp->q2_ctx_mem.va; 773 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 774 init_info.host_ctx = (__le64 *) (init_info.q2 + IRDMA_Q2_BUF_SIZE); 775 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 776 777 if (init_attr->qp_type == IB_QPT_GSI) 778 qp_num = 1; 779 else if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) 780 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 781 &qp_num, &next_qp); 782 else 783 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 784 &qp_num, &rf->next_qp); 785 if (err_code) 786 goto error; 787 788 iwqp->iwpd = iwpd; 789 iwqp->ibqp.qp_num = qp_num; 790 qp = &iwqp->sc_qp; 791 iwqp->iwscq = to_iwcq(init_attr->send_cq); 792 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 793 iwqp->host_ctx.va = init_info.host_ctx; 794 iwqp->host_ctx.pa = init_info.host_ctx_pa; 795 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 796 797 init_info.pd = &iwpd->sc_pd; 798 init_info.qp_uk_init_info.qp_id = qp_num; 799 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 800 init_info.qp_uk_init_info.first_sq_wq = 1; 801 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 802 init_waitqueue_head(&iwqp->waitq); 803 init_waitqueue_head(&iwqp->mod_qp_waitq); 804 805 spin_lock_init(&iwqp->dwork_flush_lock); 806 807 if (udata) { 808 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_user_flush_worker); 809 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 810 err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr); 811 } else { 812 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_kern_flush_worker); 813 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 814 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 815 } 816 817 if (err_code) { 818 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "setup qp failed\n"); 819 goto error; 820 } 821 822 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 823 if (init_attr->qp_type == IB_QPT_RC) { 824 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 825 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 826 IRDMA_WRITE_WITH_IMM | 827 IRDMA_ROCE; 828 } else { 829 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 830 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 831 IRDMA_ROCE; 832 } 833 } else { 834 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 835 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 836 } 837 838 ret = irdma_sc_qp_init(qp, &init_info); 839 if (ret) { 840 err_code = -EPROTO; 841 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "qp_init fail\n"); 842 goto error; 843 } 844 845 ctx_info = &iwqp->ctx_info; 846 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 847 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 848 849 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 850 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 851 else 852 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 853 854 err_code = irdma_cqp_create_qp_cmd(iwqp); 855 if (err_code) 856 goto error; 857 858 atomic_set(&iwqp->refcnt, 1); 859 spin_lock_init(&iwqp->lock); 860 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 861 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; 862 rf->qp_table[qp_num] = iwqp; 863 864 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 865 if (dev->ws_add(&iwdev->vsi, 0)) { 866 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 867 err_code = -EINVAL; 868 goto error; 869 } 870 871 irdma_qp_add_qos(&iwqp->sc_qp); 872 spin_lock_irqsave(&iwpd->udqp_list_lock, flags); 873 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) 874 list_add_tail(&iwqp->ud_list_elem, &iwpd->udqp_list); 875 spin_unlock_irqrestore(&iwpd->udqp_list_lock, flags); 876 } 877 878 if (udata) { 879 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 880 if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) { 881 uresp.lsmm = 1; 882 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 883 } else { 884 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) { 885 uresp.lsmm = 1; 886 if (qp->qp_uk.start_wqe_idx) { 887 uresp.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX; 888 uresp.start_wqe_idx = qp->qp_uk.start_wqe_idx; 889 } 890 } 891 } 892 uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size; 893 uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size; 894 uresp.qp_id = qp_num; 895 uresp.qp_caps = qp->qp_uk.qp_caps; 896 897 err_code = ib_copy_to_udata(udata, &uresp, 898 min(sizeof(uresp), udata->outlen)); 899 if (err_code) { 900 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); 901 irdma_destroy_qp(&iwqp->ibqp, udata); 902 return ERR_PTR(err_code); 903 } 904 } 905 906 init_completion(&iwqp->free_qp); 907 return &iwqp->ibqp; 908 909 error: 910 irdma_free_qp_rsrc(iwqp); 911 912 return ERR_PTR(err_code); 913 } 914 915 /** 916 * irdma_destroy_qp - destroy qp 917 * @ibqp: qp's ib pointer also to get to device's qp address 918 * @udata: user data 919 */ 920 int 921 irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 922 { 923 struct irdma_qp *iwqp = to_iwqp(ibqp); 924 struct irdma_device *iwdev = iwqp->iwdev; 925 unsigned long flags; 926 927 if (iwqp->sc_qp.qp_uk.destroy_pending) 928 goto free_rsrc; 929 iwqp->sc_qp.qp_uk.destroy_pending = true; 930 931 spin_lock_irqsave(&iwqp->iwpd->udqp_list_lock, flags); 932 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) 933 list_del(&iwqp->ud_list_elem); 934 spin_unlock_irqrestore(&iwqp->iwpd->udqp_list_lock, flags); 935 936 if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE) 937 irdma_modify_qp_to_err(&iwqp->sc_qp); 938 939 if (!iwqp->user_mode) { 940 if (iwqp->iwscq) { 941 irdma_clean_cqes(iwqp, iwqp->iwscq); 942 if (iwqp->iwrcq != iwqp->iwscq) 943 irdma_clean_cqes(iwqp, iwqp->iwrcq); 944 } 945 } 946 irdma_qp_rem_ref(&iwqp->ibqp); 947 wait_for_completion(&iwqp->free_qp); 948 irdma_free_lsmm_rsrc(iwqp); 949 if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) 950 return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE; 951 free_rsrc: 952 irdma_remove_push_mmap_entries(iwqp); 953 irdma_free_qp_rsrc(iwqp); 954 955 return 0; 956 } 957 958 /** 959 * irdma_create_cq - create cq 960 * @ibcq: CQ allocated 961 * @attr: attributes for cq 962 * @udata: user data 963 */ 964 int 965 irdma_create_cq(struct ib_cq *ibcq, 966 const struct ib_cq_init_attr *attr, 967 struct ib_udata *udata) 968 { 969 #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf) 970 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) 971 struct ib_device *ibdev = ibcq->device; 972 struct irdma_device *iwdev = to_iwdev(ibdev); 973 struct irdma_pci_f *rf = iwdev->rf; 974 struct irdma_cq *iwcq = to_iwcq(ibcq); 975 u32 cq_num = 0; 976 struct irdma_sc_cq *cq; 977 struct irdma_sc_dev *dev = &rf->sc_dev; 978 struct irdma_cq_init_info info = {0}; 979 int status; 980 struct irdma_cqp_request *cqp_request; 981 struct cqp_cmds_info *cqp_info; 982 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 983 unsigned long flags; 984 int err_code; 985 int entries = attr->cqe; 986 987 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 988 if (err_code) 989 return err_code; 990 991 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || 992 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) 993 return -EINVAL; 994 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 995 &rf->next_cq); 996 if (err_code) 997 return err_code; 998 cq = &iwcq->sc_cq; 999 cq->back_cq = iwcq; 1000 atomic_set(&iwcq->refcnt, 1); 1001 spin_lock_init(&iwcq->lock); 1002 INIT_LIST_HEAD(&iwcq->resize_list); 1003 INIT_LIST_HEAD(&iwcq->cmpl_generated); 1004 info.dev = dev; 1005 ukinfo->cq_size = max_t(int, entries, 4); 1006 ukinfo->cq_id = cq_num; 1007 iwcq->cq_num = cq_num; 1008 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 1009 atomic_set(&iwcq->armed, 0); 1010 if (attr->comp_vector < rf->ceqs_count) 1011 info.ceq_id = attr->comp_vector; 1012 info.ceq_id_valid = true; 1013 info.ceqe_mask = 1; 1014 info.type = IRDMA_CQ_TYPE_IWARP; 1015 info.vsi = &iwdev->vsi; 1016 1017 if (udata) { 1018 struct irdma_ucontext *ucontext; 1019 struct irdma_create_cq_req req = {0}; 1020 struct irdma_cq_mr *cqmr; 1021 struct irdma_pbl *iwpbl; 1022 struct irdma_pbl *iwpbl_shadow; 1023 struct irdma_cq_mr *cqmr_shadow; 1024 1025 iwcq->user_mode = true; 1026 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); 1027 1028 if (ib_copy_from_udata(&req, udata, 1029 min(sizeof(req), udata->inlen))) { 1030 err_code = -EFAULT; 1031 goto cq_free_rsrc; 1032 } 1033 1034 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1035 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 1036 &ucontext->cq_reg_mem_list); 1037 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1038 if (!iwpbl) { 1039 err_code = -EPROTO; 1040 goto cq_free_rsrc; 1041 } 1042 cqmr = &iwpbl->cq_mr; 1043 1044 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 1045 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { 1046 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1047 iwpbl_shadow = irdma_get_pbl((unsigned long)req.user_shadow_area, 1048 &ucontext->cq_reg_mem_list); 1049 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1050 1051 if (!iwpbl_shadow) { 1052 err_code = -EPROTO; 1053 goto cq_free_rsrc; 1054 } 1055 cqmr_shadow = &iwpbl_shadow->cq_mr; 1056 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 1057 cqmr->split = true; 1058 } else { 1059 info.shadow_area_pa = cqmr->shadow; 1060 } 1061 if (iwpbl->pbl_allocated) { 1062 info.virtual_map = true; 1063 info.pbl_chunk_size = 1; 1064 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 1065 } else { 1066 info.cq_base_pa = cqmr->cq_pbl.addr; 1067 } 1068 } else { 1069 /* Kmode allocations */ 1070 int rsize; 1071 1072 if (entries < 1 || entries > rf->max_cqe) { 1073 err_code = -EINVAL; 1074 goto cq_free_rsrc; 1075 } 1076 1077 entries++; 1078 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) 1079 entries *= 2; 1080 ukinfo->cq_size = entries; 1081 1082 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 1083 iwcq->kmem.size = round_up(rsize, IRDMA_HW_PAGE_SIZE); 1084 iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem, 1085 iwcq->kmem.size, IRDMA_HW_PAGE_SIZE); 1086 if (!iwcq->kmem.va) { 1087 err_code = -ENOMEM; 1088 goto cq_free_rsrc; 1089 } 1090 1091 iwcq->kmem_shadow.size = IRDMA_SHADOW_AREA_SIZE << 3; 1092 iwcq->kmem_shadow.va = irdma_allocate_dma_mem(dev->hw, 1093 &iwcq->kmem_shadow, 1094 iwcq->kmem_shadow.size, 1095 64); 1096 1097 if (!iwcq->kmem_shadow.va) { 1098 err_code = -ENOMEM; 1099 goto cq_kmem_free; 1100 } 1101 info.shadow_area_pa = iwcq->kmem_shadow.pa; 1102 ukinfo->shadow_area = iwcq->kmem_shadow.va; 1103 ukinfo->cq_base = iwcq->kmem.va; 1104 info.cq_base_pa = iwcq->kmem.pa; 1105 } 1106 1107 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 1108 (u32)IRDMA_MAX_CQ_READ_THRESH); 1109 if (irdma_sc_cq_init(cq, &info)) { 1110 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "init cq fail\n"); 1111 err_code = -EPROTO; 1112 goto cq_kmem_free; 1113 } 1114 1115 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1116 if (!cqp_request) { 1117 err_code = -ENOMEM; 1118 goto cq_kmem_free; 1119 } 1120 cqp_info = &cqp_request->info; 1121 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1122 cqp_info->post_sq = 1; 1123 cqp_info->in.u.cq_create.cq = cq; 1124 cqp_info->in.u.cq_create.check_overflow = true; 1125 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1126 cqp_info->create = true; 1127 status = irdma_handle_cqp_op(rf, cqp_request); 1128 irdma_put_cqp_request(&rf->cqp, cqp_request); 1129 if (status) { 1130 err_code = -ENOMEM; 1131 goto cq_kmem_free; 1132 } 1133 1134 if (udata) { 1135 struct irdma_create_cq_resp resp = {0}; 1136 1137 resp.cq_id = info.cq_uk_init_info.cq_id; 1138 resp.cq_size = info.cq_uk_init_info.cq_size; 1139 if (ib_copy_to_udata(udata, &resp, 1140 min(sizeof(resp), udata->outlen))) { 1141 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy to user data\n"); 1142 err_code = -EPROTO; 1143 goto cq_destroy; 1144 } 1145 } 1146 1147 WRITE_ONCE(rf->cq_table[cq_num], iwcq); 1148 init_completion(&iwcq->free_cq); 1149 1150 return 0; 1151 cq_destroy: 1152 irdma_cq_wq_destroy(rf, cq); 1153 cq_kmem_free: 1154 if (!iwcq->user_mode) { 1155 irdma_free_dma_mem(dev->hw, &iwcq->kmem); 1156 irdma_free_dma_mem(dev->hw, &iwcq->kmem_shadow); 1157 } 1158 cq_free_rsrc: 1159 irdma_free_rsrc(rf, rf->allocated_cqs, cq_num); 1160 return err_code; 1161 } 1162 1163 /** 1164 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 1165 * @iwmr: iwmr for IB's user page addresses 1166 * @pbl: ple pointer to save 1 level or 0 level pble 1167 * @level: indicated level 0, 1 or 2 1168 */ 1169 1170 void 1171 irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 1172 enum irdma_pble_level level) 1173 { 1174 struct ib_umem *region = iwmr->region; 1175 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 1176 int chunk_pages, entry, i; 1177 struct scatterlist *sg; 1178 u64 pg_addr = 0; 1179 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 1180 struct irdma_pble_info *pinfo; 1181 u32 idx = 0; 1182 u32 pbl_cnt = 0; 1183 1184 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 1185 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) { 1186 chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size); 1187 if (iwmr->type == IRDMA_MEMREG_TYPE_QP && !iwpbl->qp_mr.sq_page) 1188 iwpbl->qp_mr.sq_page = sg_page(sg); 1189 for (i = 0; i < chunk_pages; i++) { 1190 pg_addr = sg_dma_address(sg) + (i * iwmr->page_size); 1191 if ((entry + i) == 0) 1192 *pbl = pg_addr & iwmr->page_msk; 1193 else if (!(pg_addr & ~iwmr->page_msk)) 1194 *pbl = pg_addr; 1195 else 1196 continue; 1197 if (++pbl_cnt == palloc->total_cnt) 1198 break; 1199 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 1200 } 1201 } 1202 } 1203 1204 /** 1205 * irdma_destroy_ah - Destroy address handle 1206 * @ibah: pointer to address handle 1207 * @ah_flags: destroy flags 1208 */ 1209 1210 void 1211 irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 1212 { 1213 struct irdma_device *iwdev = to_iwdev(ibah->device); 1214 struct irdma_ah *ah = to_iwah(ibah); 1215 1216 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 1217 false, NULL, ah); 1218 1219 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 1220 ah->sc_ah.ah_info.ah_idx); 1221 } 1222 1223 1224 int 1225 irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 1226 { 1227 struct irdma_mr *iwmr = to_iwmr(ib_mr); 1228 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 1229 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 1230 int ret; 1231 1232 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 1233 if (iwmr->region) { 1234 struct irdma_ucontext *ucontext; 1235 1236 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); 1237 1238 irdma_del_memlist(iwmr, ucontext); 1239 } 1240 goto done; 1241 } 1242 1243 ret = irdma_hwdereg_mr(ib_mr); 1244 if (ret) 1245 return ret; 1246 1247 irdma_free_stag(iwdev, iwmr->stag); 1248 done: 1249 if (iwpbl->pbl_allocated) 1250 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 1251 1252 if (iwmr->region) 1253 ib_umem_release(iwmr->region); 1254 1255 kfree(iwmr); 1256 1257 return 0; 1258 } 1259 1260 /** 1261 * irdma_reg_user_mr - Register a user memory region 1262 * @pd: ptr of pd 1263 * @start: virtual start address 1264 * @len: length of mr 1265 * @virt: virtual address 1266 * @access: access of mr 1267 * @udata: user data 1268 */ 1269 struct ib_mr * 1270 irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, 1271 u64 virt, int access, 1272 struct ib_udata *udata) 1273 { 1274 #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) 1275 struct irdma_device *iwdev = to_iwdev(pd->device); 1276 struct irdma_mem_reg_req req = {}; 1277 struct ib_umem *region; 1278 struct irdma_mr *iwmr; 1279 int err; 1280 1281 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 1282 return ERR_PTR(-EINVAL); 1283 1284 if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) 1285 return ERR_PTR(-EINVAL); 1286 1287 region = ib_umem_get(pd->uobject->context, start, len, access, 0); 1288 1289 if (IS_ERR(region)) { 1290 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, 1291 "Failed to create ib_umem region err=%ld\n", 1292 PTR_ERR(region)); 1293 return (struct ib_mr *)region; 1294 } 1295 1296 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { 1297 ib_umem_release(region); 1298 return ERR_PTR(-EFAULT); 1299 } 1300 1301 iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type); 1302 if (IS_ERR(iwmr)) { 1303 ib_umem_release(region); 1304 return (struct ib_mr *)iwmr; 1305 } 1306 1307 switch (req.reg_type) { 1308 case IRDMA_MEMREG_TYPE_QP: 1309 err = irdma_reg_user_mr_type_qp(req, udata, iwmr); 1310 if (err) 1311 goto error; 1312 1313 break; 1314 case IRDMA_MEMREG_TYPE_CQ: 1315 err = irdma_reg_user_mr_type_cq(req, udata, iwmr); 1316 if (err) 1317 goto error; 1318 1319 break; 1320 case IRDMA_MEMREG_TYPE_MEM: 1321 err = irdma_reg_user_mr_type_mem(iwmr, access, true); 1322 if (err) 1323 goto error; 1324 1325 break; 1326 default: 1327 err = -EINVAL; 1328 goto error; 1329 } 1330 1331 return &iwmr->ibmr; 1332 1333 error: 1334 ib_umem_release(region); 1335 irdma_free_iwmr(iwmr); 1336 1337 return ERR_PTR(err); 1338 } 1339 1340 /* 1341 * irdma_rereg_user_mr - Re-Register a user memory region @ibmr: ib mem to access iwarp mr pointer @flags: bit mask to 1342 * indicate which of the attr's of MR modified @start: virtual start address @len: length of mr @virt: virtual address 1343 * @new access flags: bit mask of access flags @new_pd: ptr of pd @udata: user data 1344 */ 1345 int 1346 irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len, 1347 u64 virt, int new_access, struct ib_pd *new_pd, 1348 struct ib_udata *udata) 1349 { 1350 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 1351 struct irdma_mr *iwmr = to_iwmr(ib_mr); 1352 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 1353 int ret; 1354 1355 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 1356 return -EINVAL; 1357 1358 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) 1359 return -EOPNOTSUPP; 1360 1361 ret = irdma_hwdereg_mr(ib_mr); 1362 if (ret) 1363 return ret; 1364 1365 if (flags & IB_MR_REREG_ACCESS) 1366 iwmr->access = new_access; 1367 1368 if (flags & IB_MR_REREG_PD) { 1369 iwmr->ibmr.pd = new_pd; 1370 iwmr->ibmr.device = new_pd->device; 1371 } 1372 1373 if (flags & IB_MR_REREG_TRANS) { 1374 if (iwpbl->pbl_allocated) { 1375 irdma_free_pble(iwdev->rf->pble_rsrc, 1376 &iwpbl->pble_alloc); 1377 iwpbl->pbl_allocated = false; 1378 } 1379 if (iwmr->region) { 1380 ib_umem_release(iwmr->region); 1381 iwmr->region = NULL; 1382 } 1383 1384 ib_mr = irdma_rereg_mr_trans(iwmr, start, len, virt, udata); 1385 if (IS_ERR(ib_mr)) 1386 return PTR_ERR(ib_mr); 1387 1388 } else { 1389 ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access); 1390 if (ret) 1391 return ret; 1392 } 1393 1394 return 0; 1395 } 1396 1397 int 1398 kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr, 1399 u16 *vlan_id) 1400 { 1401 int ret; 1402 union ib_gid sgid; 1403 struct ib_gid_attr sgid_attr; 1404 struct irdma_av *av = &iwqp->roce_ah.av; 1405 1406 ret = ib_get_cached_gid(iwqp->ibqp.device, attr->ah_attr.port_num, 1407 attr->ah_attr.grh.sgid_index, &sgid, 1408 &sgid_attr); 1409 if (ret) 1410 return ret; 1411 1412 if (sgid_attr.ndev) { 1413 *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev); 1414 ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, if_getlladdr(sgid_attr.ndev)); 1415 } 1416 1417 av->net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); 1418 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid); 1419 dev_put(sgid_attr.ndev); 1420 iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri; 1421 1422 return 0; 1423 } 1424 1425 /** 1426 * irdma_destroy_cq - destroy cq 1427 * @ib_cq: cq pointer 1428 * @udata: user data 1429 */ 1430 void 1431 irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1432 { 1433 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1434 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1435 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1436 struct irdma_sc_dev *dev = cq->dev; 1437 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1438 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1439 unsigned long flags; 1440 1441 spin_lock_irqsave(&iwcq->lock, flags); 1442 if (!list_empty(&iwcq->cmpl_generated)) 1443 irdma_remove_cmpls_list(iwcq); 1444 if (!list_empty(&iwcq->resize_list)) 1445 irdma_process_resize_list(iwcq, iwdev, NULL); 1446 spin_unlock_irqrestore(&iwcq->lock, flags); 1447 1448 irdma_cq_rem_ref(ib_cq); 1449 wait_for_completion(&iwcq->free_cq); 1450 1451 irdma_cq_wq_destroy(iwdev->rf, cq); 1452 1453 spin_lock_irqsave(&iwceq->ce_lock, flags); 1454 irdma_sc_cleanup_ceqes(cq, ceq); 1455 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1456 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1457 } 1458 1459 /** 1460 * kc_set_loc_seq_num_mss - Set local seq number and mss 1461 * @cm_node: cm node info 1462 */ 1463 void 1464 kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node) 1465 { 1466 struct timespec ts; 1467 1468 getnanotime(&ts); 1469 cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec; 1470 if (cm_node->iwdev->vsi.mtu > 1500 && 1471 2 * cm_node->iwdev->vsi.mtu > cm_node->iwdev->rcv_wnd) 1472 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? 1473 (1500 - IRDMA_MTU_TO_MSS_IPV4) : 1474 (1500 - IRDMA_MTU_TO_MSS_IPV6); 1475 else 1476 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? 1477 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4) : 1478 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6); 1479 } 1480 1481 /** 1482 * irdma_disassociate_ucontext - Disassociate user context 1483 * @context: ib user context 1484 */ 1485 void 1486 irdma_disassociate_ucontext(struct ib_ucontext *context) 1487 { 1488 } 1489 1490 struct ib_device * 1491 ib_device_get_by_netdev(if_t netdev, int driver_id) 1492 { 1493 struct irdma_device *iwdev; 1494 struct irdma_handler *hdl; 1495 unsigned long flags; 1496 1497 spin_lock_irqsave(&irdma_handler_lock, flags); 1498 list_for_each_entry(hdl, &irdma_handlers, list) { 1499 iwdev = hdl->iwdev; 1500 if (netdev == iwdev->netdev) { 1501 spin_unlock_irqrestore(&irdma_handler_lock, 1502 flags); 1503 return &iwdev->ibdev; 1504 } 1505 } 1506 spin_unlock_irqrestore(&irdma_handler_lock, flags); 1507 1508 return NULL; 1509 } 1510 1511 void 1512 ib_unregister_device_put(struct ib_device *device) 1513 { 1514 ib_unregister_device(device); 1515 } 1516 1517 /** 1518 * irdma_query_gid_roce - Query port GID for Roce 1519 * @ibdev: device pointer from stack 1520 * @port: port number 1521 * @index: Entry index 1522 * @gid: Global ID 1523 */ 1524 int 1525 irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index, 1526 union ib_gid *gid) 1527 { 1528 int ret; 1529 1530 ret = ib_get_cached_gid(ibdev, port, index, gid, NULL); 1531 if (ret == -EAGAIN) { 1532 memcpy(gid, &zgid, sizeof(*gid)); 1533 return 0; 1534 } 1535 1536 return ret; 1537 } 1538 1539 /** 1540 * irdma_modify_port - modify port attributes 1541 * @ibdev: device pointer from stack 1542 * @port: port number for query 1543 * @mask: Property mask 1544 * @props: returning device attributes 1545 */ 1546 int 1547 irdma_modify_port(struct ib_device *ibdev, u8 port, int mask, 1548 struct ib_port_modify *props) 1549 { 1550 if (port > 1) 1551 return -EINVAL; 1552 1553 return 0; 1554 } 1555 1556 /** 1557 * irdma_query_pkey - Query partition key 1558 * @ibdev: device pointer from stack 1559 * @port: port number 1560 * @index: index of pkey 1561 * @pkey: pointer to store the pkey 1562 */ 1563 int 1564 irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, 1565 u16 *pkey) 1566 { 1567 if (index >= IRDMA_PKEY_TBL_SZ) 1568 return -EINVAL; 1569 1570 *pkey = IRDMA_DEFAULT_PKEY; 1571 return 0; 1572 } 1573 1574 int 1575 irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num, 1576 struct ib_port_immutable *immutable) 1577 { 1578 struct ib_port_attr attr; 1579 int err; 1580 1581 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 1582 err = ib_query_port(ibdev, port_num, &attr); 1583 if (err) 1584 return err; 1585 1586 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 1587 immutable->pkey_tbl_len = attr.pkey_tbl_len; 1588 immutable->gid_tbl_len = attr.gid_tbl_len; 1589 1590 return 0; 1591 } 1592 1593 int 1594 irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num, 1595 struct ib_port_immutable *immutable) 1596 { 1597 struct ib_port_attr attr; 1598 int err; 1599 1600 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 1601 err = ib_query_port(ibdev, port_num, &attr); 1602 if (err) 1603 return err; 1604 immutable->gid_tbl_len = 1; 1605 1606 return 0; 1607 } 1608 1609 /** 1610 * irdma_query_port - get port attributes 1611 * @ibdev: device pointer from stack 1612 * @port: port number for query 1613 * @props: returning device attributes 1614 */ 1615 int 1616 irdma_query_port(struct ib_device *ibdev, u8 port, 1617 struct ib_port_attr *props) 1618 { 1619 struct irdma_device *iwdev = to_iwdev(ibdev); 1620 if_t netdev = iwdev->netdev; 1621 1622 /* no need to zero out pros here. done by caller */ 1623 1624 props->max_mtu = IB_MTU_4096; 1625 props->active_mtu = min(props->max_mtu, iboe_get_mtu(if_getmtu(netdev))); 1626 props->lid = 1; 1627 props->lmc = 0; 1628 props->sm_lid = 0; 1629 props->sm_sl = 0; 1630 if ((if_getlinkstate(netdev) == LINK_STATE_UP) && (if_getdrvflags(netdev) & IFF_DRV_RUNNING)) { 1631 props->state = IB_PORT_ACTIVE; 1632 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 1633 } else { 1634 props->state = IB_PORT_DOWN; 1635 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 1636 } 1637 ib_get_eth_speed(ibdev, port, (u16 *)& props->active_speed, &props->active_width); 1638 1639 if (rdma_protocol_roce(ibdev, 1)) { 1640 props->gid_tbl_len = 32; 1641 props->port_cap_flags |= IB_PORT_IP_BASED_GIDS; 1642 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 1643 } else { 1644 props->gid_tbl_len = 1; 1645 } 1646 props->qkey_viol_cntr = 0; 1647 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 1648 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 1649 1650 return 0; 1651 } 1652 1653 static const char *const irdma_hw_stat_names[] = { 1654 /* gen1 - 32-bit */ 1655 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards", 1656 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts", 1657 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes", 1658 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards", 1659 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts", 1660 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes", 1661 [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors", 1662 /* gen1 - 64-bit */ 1663 [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = "ip4InOctets", 1664 [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = "ip4InPkts", 1665 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = "ip4InReasmRqd", 1666 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = "ip4InMcastPkts", 1667 [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = "ip4OutOctets", 1668 [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = "ip4OutPkts", 1669 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = "ip4OutSegRqd", 1670 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = "ip4OutMcastPkts", 1671 [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = "ip6InOctets", 1672 [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = "ip6InPkts", 1673 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = "ip6InReasmRqd", 1674 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = "ip6InMcastPkts", 1675 [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = "ip6OutOctets", 1676 [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = "ip6OutPkts", 1677 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = "ip6OutSegRqd", 1678 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = "ip6OutMcastPkts", 1679 [IRDMA_HW_STAT_INDEX_RDMARXRDS] = "InRdmaReads", 1680 [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = "InRdmaSends", 1681 [IRDMA_HW_STAT_INDEX_RDMARXWRS] = "InRdmaWrites", 1682 [IRDMA_HW_STAT_INDEX_RDMATXRDS] = "OutRdmaReads", 1683 [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = "OutRdmaSends", 1684 [IRDMA_HW_STAT_INDEX_RDMATXWRS] = "OutRdmaWrites", 1685 [IRDMA_HW_STAT_INDEX_RDMAVBND] = "RdmaBnd", 1686 [IRDMA_HW_STAT_INDEX_RDMAVINV] = "RdmaInv", 1687 1688 /* gen2 - 32-bit */ 1689 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled", 1690 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored", 1691 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent", 1692 /* gen2 - 64-bit */ 1693 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = "ip4InMcastOctets", 1694 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = "ip4OutMcastOctets", 1695 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = "ip6InMcastOctets", 1696 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = "ip6OutMcastOctets", 1697 [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = "RxUDP", 1698 [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = "TxUDP", 1699 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = "RxECNMrkd", 1700 [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "RetransSegs", 1701 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "InOptErrors", 1702 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "InProtoErrors", 1703 [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = "InSegs", 1704 [IRDMA_HW_STAT_INDEX_TCPTXSEG] = "OutSegs", 1705 }; 1706 1707 /** 1708 * irdma_alloc_hw_stats - Allocate a hw stats structure 1709 * @ibdev: device pointer from stack 1710 * @port_num: port number 1711 */ 1712 struct rdma_hw_stats * 1713 irdma_alloc_hw_stats(struct ib_device *ibdev, 1714 u8 port_num) 1715 { 1716 struct irdma_device *iwdev = to_iwdev(ibdev); 1717 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1718 1719 int num_counters = dev->hw_attrs.max_stat_idx; 1720 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 1721 1722 return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters, 1723 lifespan); 1724 } 1725 1726 /** 1727 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 1728 * @ibdev: device pointer from stack 1729 * @stats: stats pointer from stack 1730 * @port_num: port number 1731 * @index: which hw counter the stack is requesting we update 1732 */ 1733 int 1734 irdma_get_hw_stats(struct ib_device *ibdev, 1735 struct rdma_hw_stats *stats, u8 port_num, 1736 int index) 1737 { 1738 struct irdma_device *iwdev = to_iwdev(ibdev); 1739 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 1740 1741 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 1742 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 1743 1744 memcpy(&stats->value[0], hw_stats, sizeof(u64)* stats->num_counters); 1745 1746 return stats->num_counters; 1747 } 1748 1749 /** 1750 * irdma_query_gid - Query port GID 1751 * @ibdev: device pointer from stack 1752 * @port: port number 1753 * @index: Entry index 1754 * @gid: Global ID 1755 */ 1756 int 1757 irdma_query_gid(struct ib_device *ibdev, u8 port, int index, 1758 union ib_gid *gid) 1759 { 1760 struct irdma_device *iwdev = to_iwdev(ibdev); 1761 1762 memset(gid->raw, 0, sizeof(gid->raw)); 1763 ether_addr_copy(gid->raw, if_getlladdr(iwdev->netdev)); 1764 1765 return 0; 1766 } 1767 1768 enum rdma_link_layer 1769 irdma_get_link_layer(struct ib_device *ibdev, 1770 u8 port_num) 1771 { 1772 return IB_LINK_LAYER_ETHERNET; 1773 } 1774 1775 inline enum ib_mtu 1776 ib_mtu_int_to_enum(int mtu) 1777 { 1778 if (mtu >= 4096) 1779 return IB_MTU_4096; 1780 else if (mtu >= 2048) 1781 return IB_MTU_2048; 1782 else if (mtu >= 1024) 1783 return IB_MTU_1024; 1784 else if (mtu >= 512) 1785 return IB_MTU_512; 1786 else 1787 return IB_MTU_256; 1788 } 1789 1790 inline void 1791 kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev) 1792 { 1793 iwdev->ibdev.uverbs_cmd_mask |= 1794 BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) | 1795 BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) | 1796 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) | 1797 BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST); 1798 } 1799 1800 inline void 1801 kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev) 1802 { 1803 iwdev->ibdev.uverbs_cmd_mask = 1804 BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) | 1805 BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) | 1806 BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) | 1807 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) | 1808 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) | 1809 BIT_ULL(IB_USER_VERBS_CMD_REG_MR) | 1810 BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) | 1811 BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) | 1812 BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 1813 BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) | 1814 BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ) | 1815 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) | 1816 BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | 1817 BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) | 1818 BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) | 1819 BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) | 1820 BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) | 1821 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) | 1822 BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) | 1823 BIT_ULL(IB_USER_VERBS_CMD_POST_SEND); 1824 iwdev->ibdev.uverbs_ex_cmd_mask = 1825 BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP) | 1826 BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE); 1827 1828 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 1829 iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ); 1830 } 1831 1832 static void 1833 ib_get_width_and_speed(u32 netdev_speed, u32 lanes, 1834 u16 *speed, u8 *width) 1835 { 1836 if (!lanes) { 1837 if (netdev_speed <= SPEED_1000) { 1838 *width = IB_WIDTH_1X; 1839 *speed = IB_SPEED_SDR; 1840 } else if (netdev_speed <= SPEED_10000) { 1841 *width = IB_WIDTH_1X; 1842 *speed = IB_SPEED_FDR10; 1843 } else if (netdev_speed <= SPEED_20000) { 1844 *width = IB_WIDTH_4X; 1845 *speed = IB_SPEED_DDR; 1846 } else if (netdev_speed <= SPEED_25000) { 1847 *width = IB_WIDTH_1X; 1848 *speed = IB_SPEED_EDR; 1849 } else if (netdev_speed <= SPEED_40000) { 1850 *width = IB_WIDTH_4X; 1851 *speed = IB_SPEED_FDR10; 1852 } else if (netdev_speed <= SPEED_50000) { 1853 *width = IB_WIDTH_2X; 1854 *speed = IB_SPEED_EDR; 1855 } else if (netdev_speed <= SPEED_100000) { 1856 *width = IB_WIDTH_4X; 1857 *speed = IB_SPEED_EDR; 1858 } else if (netdev_speed <= SPEED_200000) { 1859 *width = IB_WIDTH_4X; 1860 *speed = IB_SPEED_HDR; 1861 } else { 1862 *width = IB_WIDTH_4X; 1863 *speed = IB_SPEED_NDR; 1864 } 1865 1866 return; 1867 } 1868 1869 switch (lanes) { 1870 case 1: 1871 *width = IB_WIDTH_1X; 1872 break; 1873 case 2: 1874 *width = IB_WIDTH_2X; 1875 break; 1876 case 4: 1877 *width = IB_WIDTH_4X; 1878 break; 1879 case 8: 1880 *width = IB_WIDTH_8X; 1881 break; 1882 case 12: 1883 *width = IB_WIDTH_12X; 1884 break; 1885 default: 1886 *width = IB_WIDTH_1X; 1887 } 1888 1889 switch (netdev_speed / lanes) { 1890 case SPEED_2500: 1891 *speed = IB_SPEED_SDR; 1892 break; 1893 case SPEED_5000: 1894 *speed = IB_SPEED_DDR; 1895 break; 1896 case SPEED_10000: 1897 *speed = IB_SPEED_FDR10; 1898 break; 1899 case SPEED_14000: 1900 *speed = IB_SPEED_FDR; 1901 break; 1902 case SPEED_25000: 1903 *speed = IB_SPEED_EDR; 1904 break; 1905 case SPEED_50000: 1906 *speed = IB_SPEED_HDR; 1907 break; 1908 case SPEED_100000: 1909 *speed = IB_SPEED_NDR; 1910 break; 1911 default: 1912 *speed = IB_SPEED_SDR; 1913 } 1914 } 1915 1916 int 1917 ib_get_eth_speed(struct ib_device *ibdev, u32 port_num, u16 *speed, u8 *width) 1918 { 1919 if_t netdev = ibdev->get_netdev(ibdev, port_num); 1920 u32 netdev_speed, lanes; 1921 1922 if (!netdev) 1923 return -ENODEV; 1924 1925 netdev_speed = (u32)if_getbaudrate(netdev); 1926 dev_put(netdev); 1927 lanes = 0; 1928 1929 ib_get_width_and_speed(netdev_speed, lanes, speed, width); 1930 1931 return 0; 1932 } 1933 1934 u64 1935 irdma_mac_to_u64(const u8 *eth_add) 1936 { 1937 int idx; 1938 u64 u64_eth_add; 1939 1940 for (idx = 0, u64_eth_add = 0; idx < ETHER_ADDR_LEN; idx++) 1941 u64_eth_add = u64_eth_add << 8 | eth_add[idx]; 1942 1943 return u64_eth_add; 1944 } 1945