1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2018 - 2022 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 /*$FreeBSD$*/ 35 36 #include "irdma_main.h" 37 38 #define IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000) 39 40 static u16 kc_rdma_flow_label_to_udp_sport(u32 fl) { 41 u32 fl_low = fl & 0x03FFF; 42 u32 fl_high = fl & 0xFC000; 43 44 fl_low ^= fl_high >> 14; 45 46 return (u16)(fl_low | IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN); 47 } 48 49 #define IRDMA_GRH_FLOWLABEL_MASK (0x000FFFFF) 50 51 static u32 kc_rdma_calc_flow_label(u32 lqpn, u32 rqpn) { 52 u64 fl = (u64)lqpn * rqpn; 53 54 fl ^= fl >> 20; 55 fl ^= fl >> 40; 56 57 return (u32)(fl & IRDMA_GRH_FLOWLABEL_MASK); 58 } 59 60 u16 61 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) 62 { 63 if (!fl) 64 fl = kc_rdma_calc_flow_label(lqpn, rqpn); 65 return kc_rdma_flow_label_to_udp_sport(fl); 66 } 67 68 void 69 irdma_get_dev_fw_str(struct ib_device *dev, 70 char *str, 71 size_t str_len) 72 { 73 struct irdma_device *iwdev = to_iwdev(dev); 74 75 snprintf(str, str_len, "%u.%u", 76 irdma_fw_major_ver(&iwdev->rf->sc_dev), 77 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 78 } 79 80 int 81 irdma_add_gid(struct ib_device *device, 82 u8 port_num, 83 unsigned int index, 84 const union ib_gid *gid, 85 const struct ib_gid_attr *attr, 86 void **context) 87 { 88 return 0; 89 } 90 91 int 92 irdma_del_gid(struct ib_device *device, 93 u8 port_num, 94 unsigned int index, 95 void **context) 96 { 97 return 0; 98 } 99 100 #if __FreeBSD_version >= 1400026 101 /** 102 * irdma_alloc_mr - register stag for fast memory registration 103 * @pd: ibpd pointer 104 * @mr_type: memory for stag registrion 105 * @max_num_sg: man number of pages 106 * @udata: user data 107 */ 108 struct ib_mr * 109 irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 110 u32 max_num_sg, struct ib_udata *udata) 111 { 112 #else 113 /** 114 * irdma_alloc_mr - register stag for fast memory registration 115 * @pd: ibpd pointer 116 * @mr_type: memory for stag registrion 117 * @max_num_sg: man number of pages 118 */ 119 struct ib_mr * 120 irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 121 u32 max_num_sg) 122 { 123 #endif 124 struct irdma_device *iwdev = to_iwdev(pd->device); 125 struct irdma_pble_alloc *palloc; 126 struct irdma_pbl *iwpbl; 127 struct irdma_mr *iwmr; 128 int status; 129 u32 stag; 130 int err_code = -ENOMEM; 131 132 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 133 if (!iwmr) 134 return ERR_PTR(-ENOMEM); 135 136 stag = irdma_create_stag(iwdev); 137 if (!stag) { 138 err_code = -ENOMEM; 139 goto err; 140 } 141 142 iwmr->stag = stag; 143 iwmr->ibmr.rkey = stag; 144 iwmr->ibmr.lkey = stag; 145 iwmr->ibmr.pd = pd; 146 iwmr->ibmr.device = pd->device; 147 iwpbl = &iwmr->iwpbl; 148 iwpbl->iwmr = iwmr; 149 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 150 palloc = &iwpbl->pble_alloc; 151 iwmr->page_cnt = max_num_sg; 152 /* Assume system PAGE_SIZE as the sg page sizes are unknown. */ 153 iwmr->len = max_num_sg * PAGE_SIZE; 154 status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 155 false); 156 if (status) 157 goto err_get_pble; 158 159 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 160 if (err_code) 161 goto err_alloc_stag; 162 163 iwpbl->pbl_allocated = true; 164 165 return &iwmr->ibmr; 166 err_alloc_stag: 167 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 168 err_get_pble: 169 irdma_free_stag(iwdev, stag); 170 err: 171 kfree(iwmr); 172 173 return ERR_PTR(err_code); 174 } 175 176 #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8) 177 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) 178 #if __FreeBSD_version >= 1400026 179 /** 180 * irdma_alloc_ucontext - Allocate the user context data structure 181 * @uctx: context 182 * @udata: user data 183 * 184 * This keeps track of all objects associated with a particular 185 * user-mode client. 186 */ 187 int 188 irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 189 { 190 struct ib_device *ibdev = uctx->device; 191 struct irdma_device *iwdev = to_iwdev(ibdev); 192 struct irdma_alloc_ucontext_req req = {0}; 193 struct irdma_alloc_ucontext_resp uresp = {0}; 194 struct irdma_ucontext *ucontext = to_ucontext(uctx); 195 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 196 197 if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || 198 udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) 199 return -EINVAL; 200 201 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 202 return -EINVAL; 203 204 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 205 goto ver_error; 206 207 ucontext->iwdev = iwdev; 208 ucontext->abi_ver = req.userspace_ver; 209 210 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR) 211 ucontext->use_raw_attrs = true; 212 213 /* GEN_1 support for libi40iw */ 214 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { 215 if (uk_attrs->hw_rev != IRDMA_GEN_1) 216 return -EOPNOTSUPP; 217 218 ucontext->legacy_mode = true; 219 uresp.max_qps = iwdev->rf->max_qp; 220 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 221 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 222 uresp.kernel_ver = req.userspace_ver; 223 if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) 224 return -EFAULT; 225 } else { 226 u64 bar_off; 227 228 uresp.kernel_ver = IRDMA_ABI_VER; 229 uresp.feature_flags = uk_attrs->feature_flags; 230 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 231 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 232 uresp.max_hw_inline = uk_attrs->max_hw_inline; 233 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 234 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 235 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 236 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 237 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 238 uresp.hw_rev = uk_attrs->hw_rev; 239 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; 240 241 bar_off = 242 (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 243 ucontext->db_mmap_entry = 244 irdma_user_mmap_entry_insert(ucontext, bar_off, 245 IRDMA_MMAP_IO_NC, 246 &uresp.db_mmap_key); 247 if (!ucontext->db_mmap_entry) { 248 return -ENOMEM; 249 } 250 251 if (ib_copy_to_udata(udata, &uresp, 252 min(sizeof(uresp), udata->outlen))) { 253 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 254 return -EFAULT; 255 } 256 } 257 258 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 259 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 260 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 261 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 262 INIT_LIST_HEAD(&ucontext->vma_list); 263 mutex_init(&ucontext->vma_list_mutex); 264 265 return 0; 266 267 ver_error: 268 irdma_dev_err(&iwdev->ibdev, 269 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 270 req.userspace_ver, IRDMA_ABI_VER); 271 return -EINVAL; 272 } 273 #endif 274 275 #if __FreeBSD_version < 1400026 276 /** 277 * irdma_alloc_ucontext - Allocate the user context data structure 278 * @ibdev: ib device pointer 279 * @udata: user data 280 * 281 * This keeps track of all objects associated with a particular 282 * user-mode client. 283 */ 284 struct ib_ucontext * 285 irdma_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata) 286 { 287 struct irdma_device *iwdev = to_iwdev(ibdev); 288 struct irdma_alloc_ucontext_req req = {0}; 289 struct irdma_alloc_ucontext_resp uresp = {0}; 290 struct irdma_ucontext *ucontext; 291 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 292 293 if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || 294 udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) 295 return ERR_PTR(-EINVAL); 296 297 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 298 return ERR_PTR(-EINVAL); 299 300 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 301 goto ver_error; 302 303 ucontext = kzalloc(sizeof(*ucontext), GFP_KERNEL); 304 if (!ucontext) 305 return ERR_PTR(-ENOMEM); 306 307 ucontext->iwdev = iwdev; 308 ucontext->abi_ver = req.userspace_ver; 309 310 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR) 311 ucontext->use_raw_attrs = true; 312 313 /* GEN_1 legacy support with libi40iw */ 314 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { 315 if (uk_attrs->hw_rev != IRDMA_GEN_1) { 316 kfree(ucontext); 317 return ERR_PTR(-EOPNOTSUPP); 318 } 319 320 ucontext->legacy_mode = true; 321 uresp.max_qps = iwdev->rf->max_qp; 322 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 323 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 324 uresp.kernel_ver = req.userspace_ver; 325 if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) { 326 kfree(ucontext); 327 return ERR_PTR(-EFAULT); 328 } 329 } else { 330 u64 bar_off; 331 332 uresp.kernel_ver = IRDMA_ABI_VER; 333 uresp.feature_flags = uk_attrs->feature_flags; 334 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 335 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 336 uresp.max_hw_inline = uk_attrs->max_hw_inline; 337 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 338 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 339 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 340 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 341 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 342 uresp.hw_rev = uk_attrs->hw_rev; 343 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; 344 345 bar_off = 346 (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 347 348 spin_lock_init(&ucontext->mmap_tbl_lock); 349 ucontext->db_mmap_entry = 350 irdma_user_mmap_entry_add_hash(ucontext, bar_off, 351 IRDMA_MMAP_IO_NC, 352 &uresp.db_mmap_key); 353 if (!ucontext->db_mmap_entry) { 354 spin_lock_destroy(&ucontext->mmap_tbl_lock); 355 kfree(ucontext); 356 return ERR_PTR(-ENOMEM); 357 } 358 359 if (ib_copy_to_udata(udata, &uresp, 360 min(sizeof(uresp), udata->outlen))) { 361 irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry); 362 spin_lock_destroy(&ucontext->mmap_tbl_lock); 363 kfree(ucontext); 364 return ERR_PTR(-EFAULT); 365 } 366 } 367 368 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 369 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 370 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 371 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 372 INIT_LIST_HEAD(&ucontext->vma_list); 373 mutex_init(&ucontext->vma_list_mutex); 374 375 return &ucontext->ibucontext; 376 377 ver_error: 378 irdma_dev_err(&iwdev->ibdev, 379 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 380 req.userspace_ver, IRDMA_ABI_VER); 381 return ERR_PTR(-EINVAL); 382 } 383 #endif 384 385 #if __FreeBSD_version >= 1400026 386 /** 387 * irdma_dealloc_ucontext - deallocate the user context data structure 388 * @context: user context created during alloc 389 */ 390 void 391 irdma_dealloc_ucontext(struct ib_ucontext *context) 392 { 393 struct irdma_ucontext *ucontext = to_ucontext(context); 394 395 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 396 397 return; 398 } 399 #endif 400 401 #if __FreeBSD_version < 1400026 402 /** 403 * irdma_dealloc_ucontext - deallocate the user context data structure 404 * @context: user context created during alloc 405 */ 406 int 407 irdma_dealloc_ucontext(struct ib_ucontext *context) 408 { 409 struct irdma_ucontext *ucontext = to_ucontext(context); 410 411 irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry); 412 spin_lock_destroy(&ucontext->mmap_tbl_lock); 413 kfree(ucontext); 414 415 return 0; 416 } 417 #endif 418 419 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) 420 #if __FreeBSD_version >= 1400026 421 /** 422 * irdma_alloc_pd - allocate protection domain 423 * @pd: protection domain 424 * @udata: user data 425 */ 426 int 427 irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 428 { 429 struct irdma_pd *iwpd = to_iwpd(pd); 430 struct irdma_device *iwdev = to_iwdev(pd->device); 431 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 432 struct irdma_pci_f *rf = iwdev->rf; 433 struct irdma_alloc_pd_resp uresp = {0}; 434 struct irdma_sc_pd *sc_pd; 435 u32 pd_id = 0; 436 int err; 437 438 if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) 439 return -EINVAL; 440 441 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 442 &rf->next_pd); 443 if (err) 444 return err; 445 446 sc_pd = &iwpd->sc_pd; 447 if (udata) { 448 struct irdma_ucontext *ucontext = 449 rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); 450 451 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 452 uresp.pd_id = pd_id; 453 if (ib_copy_to_udata(udata, &uresp, 454 min(sizeof(uresp), udata->outlen))) { 455 err = -EFAULT; 456 goto error; 457 } 458 } else { 459 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 460 } 461 462 spin_lock_init(&iwpd->udqp_list_lock); 463 INIT_LIST_HEAD(&iwpd->udqp_list); 464 465 return 0; 466 467 error: 468 469 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 470 471 return err; 472 } 473 #endif 474 475 #if __FreeBSD_version < 1400026 476 /** 477 * irdma_alloc_pd - allocate protection domain 478 * @ibdev: IB device 479 * @context: user context 480 * @udata: user data 481 */ 482 struct ib_pd * 483 irdma_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata) 484 { 485 struct irdma_pd *iwpd; 486 struct irdma_device *iwdev = to_iwdev(ibdev); 487 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 488 struct irdma_pci_f *rf = iwdev->rf; 489 struct irdma_alloc_pd_resp uresp = {0}; 490 struct irdma_sc_pd *sc_pd; 491 u32 pd_id = 0; 492 int err; 493 494 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 495 &rf->next_pd); 496 if (err) 497 return ERR_PTR(err); 498 499 iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL); 500 if (!iwpd) { 501 err = -ENOMEM; 502 goto free_res; 503 } 504 505 sc_pd = &iwpd->sc_pd; 506 if (udata) { 507 struct irdma_ucontext *ucontext = to_ucontext(context); 508 509 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 510 uresp.pd_id = pd_id; 511 if (ib_copy_to_udata(udata, &uresp, 512 min(sizeof(uresp), udata->outlen))) { 513 err = -EFAULT; 514 goto error; 515 } 516 } else { 517 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 518 } 519 520 spin_lock_init(&iwpd->udqp_list_lock); 521 INIT_LIST_HEAD(&iwpd->udqp_list); 522 523 return &iwpd->ibpd; 524 525 error: 526 kfree(iwpd); 527 free_res: 528 529 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 530 531 return ERR_PTR(err); 532 } 533 534 #endif 535 536 #if __FreeBSD_version >= 1400026 537 void 538 irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 539 { 540 struct irdma_pd *iwpd = to_iwpd(ibpd); 541 struct irdma_device *iwdev = to_iwdev(ibpd->device); 542 543 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 544 } 545 546 #endif 547 548 #if __FreeBSD_version < 1400026 549 int 550 irdma_dealloc_pd(struct ib_pd *ibpd) 551 { 552 struct irdma_pd *iwpd = to_iwpd(ibpd); 553 struct irdma_device *iwdev = to_iwdev(ibpd->device); 554 555 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 556 kfree(iwpd); 557 return 0; 558 } 559 #endif 560 561 /** 562 * irdma_find_qp_update_qs - update QS handle for UD QPs 563 * @rf: RDMA PCI function 564 * @pd: protection domain object 565 * @user_pri: selected user priority 566 */ 567 static void 568 irdma_find_qp_update_qs(struct irdma_pci_f *rf, 569 struct irdma_pd *pd, u8 user_pri) 570 { 571 struct irdma_qp *iwqp; 572 struct list_head *tmp_node, *list_node; 573 struct irdma_udqs_work *work; 574 unsigned long flags; 575 bool qs_change; 576 577 spin_lock_irqsave(&pd->udqp_list_lock, flags); 578 list_for_each_safe(list_node, tmp_node, &pd->udqp_list) { 579 qs_change = true; 580 iwqp = list_entry(list_node, struct irdma_qp, ud_list_elem); 581 irdma_qp_add_ref(&iwqp->ibqp); 582 /* check if qs_handle needs to be changed */ 583 if (iwqp->sc_qp.qs_handle == iwqp->sc_qp.vsi->qos[user_pri].qs_handle) { 584 if (iwqp->ctx_info.user_pri == user_pri) { 585 /* qs_handle and user_pri don't change */ 586 irdma_qp_rem_ref(&iwqp->ibqp); 587 continue; 588 } 589 qs_change = false; 590 } 591 /* perform qp qos change */ 592 work = kzalloc(sizeof(*work), GFP_ATOMIC); 593 if (!work) { 594 irdma_qp_rem_ref(&iwqp->ibqp); 595 spin_unlock_irqrestore(&pd->udqp_list_lock, flags); 596 return; 597 } 598 work->iwqp = iwqp; 599 work->user_prio = user_pri; 600 work->qs_change = qs_change; 601 INIT_WORK(&work->work, irdma_udqp_qs_worker); 602 if (qs_change) 603 irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND); 604 queue_work(rf->iwdev->cleanup_wq, &work->work); 605 } 606 spin_unlock_irqrestore(&pd->udqp_list_lock, flags); 607 } 608 609 static void 610 irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info, 611 const struct ib_gid_attr *sgid_attr, 612 struct sockaddr *sgid_addr, struct sockaddr *dgid_addr, 613 u8 *dmac, u8 net_type) 614 { 615 if (net_type == RDMA_NETWORK_IPV4) { 616 ah_info->ipv4_valid = true; 617 ah_info->dest_ip_addr[0] = 618 ntohl(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr); 619 ah_info->src_ip_addr[0] = 620 ntohl(((struct sockaddr_in *)sgid_addr)->sin_addr.s_addr); 621 CURVNET_SET_QUIET(vnet); 622 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 623 ah_info->dest_ip_addr[0]); 624 CURVNET_RESTORE(); 625 if (ipv4_is_multicast(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr)) { 626 irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac); 627 } 628 } else { 629 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 630 ((struct sockaddr_in6 *)dgid_addr)->sin6_addr.__u6_addr.__u6_addr32); 631 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 632 ((struct sockaddr_in6 *)sgid_addr)->sin6_addr.__u6_addr.__u6_addr32); 633 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 634 ah_info->dest_ip_addr); 635 if (rdma_is_multicast_addr(&((struct sockaddr_in6 *)dgid_addr)->sin6_addr)) { 636 irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac); 637 } 638 } 639 } 640 641 static inline u8 irdma_get_vlan_ndev_prio(if_t ndev, u8 prio) 642 { 643 return prio; 644 } 645 646 static int 647 irdma_create_ah_vlan_tag(struct irdma_device *iwdev, 648 struct irdma_pd *pd, 649 struct irdma_ah_info *ah_info, 650 const struct ib_gid_attr *sgid_attr, 651 u8 *dmac) 652 { 653 u16 vlan_prio; 654 655 if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev)) 656 ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev); 657 else 658 ah_info->vlan_tag = VLAN_N_VID; 659 660 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, dmac); 661 662 if (ah_info->dst_arpindex == -1) 663 return -EINVAL; 664 665 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) 666 ah_info->vlan_tag = 0; 667 668 if (ah_info->vlan_tag < VLAN_N_VID) { 669 if_t ndev = sgid_attr->ndev; 670 671 ah_info->insert_vlan_tag = true; 672 vlan_prio = (u16)irdma_get_vlan_ndev_prio(ndev, rt_tos2priority(ah_info->tc_tos)); 673 ah_info->vlan_tag |= vlan_prio << VLAN_PRIO_SHIFT; 674 irdma_find_qp_update_qs(iwdev->rf, pd, vlan_prio); 675 } 676 if (iwdev->roce_dcqcn_en) { 677 ah_info->tc_tos &= ~ECN_CODE_PT_MASK; 678 ah_info->tc_tos |= ECN_CODE_PT_VAL; 679 } 680 681 return 0; 682 } 683 684 static int 685 irdma_create_ah_wait(struct irdma_pci_f *rf, 686 struct irdma_sc_ah *sc_ah, bool sleep) 687 { 688 if (!sleep) { 689 int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms * 690 CQP_TIMEOUT_THRESHOLD; 691 692 do { 693 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 694 mdelay(1); 695 } while (!sc_ah->ah_info.ah_valid && --cnt); 696 697 if (!cnt) 698 return -ETIMEDOUT; 699 } 700 return 0; 701 } 702 703 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) 704 705 #if __FreeBSD_version >= 1400026 706 /** 707 * irdma_create_ah - create address handle 708 * @ib_ah: ptr to AH 709 * @attr: address handle attributes 710 * @flags: AH flags to wait 711 * @udata: user data 712 * 713 * returns 0 on success, error otherwise 714 */ 715 int 716 irdma_create_ah(struct ib_ah *ib_ah, 717 struct ib_ah_attr *attr, u32 flags, 718 struct ib_udata *udata) 719 { 720 struct irdma_pd *pd = to_iwpd(ib_ah->pd); 721 struct irdma_ah *ah = container_of(ib_ah, struct irdma_ah, ibah); 722 struct irdma_device *iwdev = to_iwdev(ib_ah->pd->device); 723 union ib_gid sgid; 724 struct ib_gid_attr sgid_attr; 725 struct irdma_pci_f *rf = iwdev->rf; 726 struct irdma_sc_ah *sc_ah; 727 u32 ah_id = 0; 728 struct irdma_ah_info *ah_info; 729 struct irdma_create_ah_resp uresp; 730 union { 731 struct sockaddr saddr; 732 struct sockaddr_in saddr_in; 733 struct sockaddr_in6 saddr_in6; 734 } sgid_addr, dgid_addr; 735 int err; 736 u8 dmac[ETH_ALEN]; 737 bool sleep = (flags & RDMA_CREATE_AH_SLEEPABLE) != 0; 738 739 if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) 740 return -EINVAL; 741 742 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, 743 rf->max_ah, &ah_id, &rf->next_ah); 744 745 if (err) 746 return err; 747 748 ah->pd = pd; 749 sc_ah = &ah->sc_ah; 750 sc_ah->ah_info.ah_idx = ah_id; 751 sc_ah->ah_info.vsi = &iwdev->vsi; 752 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 753 ah->sgid_index = attr->grh.sgid_index; 754 memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid)); 755 rcu_read_lock(); 756 err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num, 757 attr->grh.sgid_index, &sgid, &sgid_attr); 758 rcu_read_unlock(); 759 if (err) { 760 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, 761 "GID lookup at idx=%d with port=%d failed\n", 762 attr->grh.sgid_index, attr->port_num); 763 err = -EINVAL; 764 goto err_gid_l2; 765 } 766 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); 767 rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); 768 ah->av.attrs = *attr; 769 ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr, 770 sgid_attr.gid_type, 771 &sgid); 772 773 if (sgid_attr.ndev) 774 dev_put(sgid_attr.ndev); 775 776 ah->av.sgid_addr.saddr = sgid_addr.saddr; 777 ah->av.dgid_addr.saddr = dgid_addr.saddr; 778 ah_info = &sc_ah->ah_info; 779 ah_info->ah_idx = ah_id; 780 ah_info->pd_idx = pd->sc_pd.pd_id; 781 ether_addr_copy(ah_info->mac_addr, if_getlladdr(iwdev->netdev)); 782 783 if (attr->ah_flags & IB_AH_GRH) { 784 ah_info->flow_label = attr->grh.flow_label; 785 ah_info->hop_ttl = attr->grh.hop_limit; 786 ah_info->tc_tos = attr->grh.traffic_class; 787 } 788 789 ether_addr_copy(dmac, attr->dmac); 790 791 irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, 792 dmac, ah->av.net_type); 793 794 err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac); 795 if (err) 796 goto err_gid_l2; 797 798 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, 799 sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); 800 if (err) { 801 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP-OP Create AH fail"); 802 goto err_gid_l2; 803 } 804 805 err = irdma_create_ah_wait(rf, sc_ah, sleep); 806 if (err) { 807 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out"); 808 goto err_gid_l2; 809 } 810 811 if (udata) { 812 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 813 err = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 814 if (err) { 815 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, 816 IRDMA_OP_AH_DESTROY, false, NULL, ah); 817 goto err_gid_l2; 818 } 819 } 820 821 return 0; 822 err_gid_l2: 823 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); 824 825 return err; 826 } 827 #endif 828 829 void 830 irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr) 831 { 832 ether_addr_copy(dmac, attr->dmac); 833 } 834 835 #if __FreeBSD_version < 1400026 836 struct ib_ah * 837 irdma_create_ah_stub(struct ib_pd *ibpd, 838 struct ib_ah_attr *attr, 839 struct ib_udata *udata) 840 #else 841 int 842 irdma_create_ah_stub(struct ib_ah *ib_ah, 843 struct ib_ah_attr *attr, u32 flags, 844 struct ib_udata *udata) 845 #endif 846 { 847 #if __FreeBSD_version >= 1400026 848 return -ENOSYS; 849 #else 850 return ERR_PTR(-ENOSYS); 851 #endif 852 } 853 854 #if __FreeBSD_version >= 1400026 855 void 856 irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags) 857 { 858 return; 859 } 860 #else 861 int 862 irdma_destroy_ah_stub(struct ib_ah *ibah) 863 { 864 return -ENOSYS; 865 } 866 #endif 867 868 #if __FreeBSD_version < 1400026 869 /** 870 * irdma_create_ah - create address handle 871 * @ibpd: ptr to pd 872 * @attr: address handle attributes 873 * @udata: user data 874 * 875 * returns a pointer to an address handle 876 */ 877 struct ib_ah * 878 irdma_create_ah(struct ib_pd *ibpd, 879 struct ib_ah_attr *attr, 880 struct ib_udata *udata) 881 { 882 struct irdma_pd *pd = to_iwpd(ibpd); 883 struct irdma_device *iwdev = to_iwdev(ibpd->device); 884 struct irdma_ah *ah; 885 union ib_gid sgid; 886 struct ib_gid_attr sgid_attr; 887 struct irdma_pci_f *rf = iwdev->rf; 888 struct irdma_sc_ah *sc_ah; 889 u32 ah_id = 0; 890 struct irdma_ah_info *ah_info; 891 struct irdma_create_ah_resp uresp; 892 union { 893 struct sockaddr saddr; 894 struct sockaddr_in saddr_in; 895 struct sockaddr_in6 saddr_in6; 896 } sgid_addr, dgid_addr; 897 int err; 898 u8 dmac[ETH_ALEN]; 899 bool sleep = udata ? true : false; 900 901 if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) 902 return ERR_PTR(-EINVAL); 903 904 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, 905 rf->max_ah, &ah_id, &rf->next_ah); 906 907 if (err) 908 return ERR_PTR(err); 909 910 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 911 if (!ah) { 912 irdma_free_rsrc(rf, rf->allocated_ahs, ah_id); 913 return ERR_PTR(-ENOMEM); 914 } 915 916 ah->pd = pd; 917 sc_ah = &ah->sc_ah; 918 sc_ah->ah_info.ah_idx = ah_id; 919 sc_ah->ah_info.vsi = &iwdev->vsi; 920 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 921 ah->sgid_index = attr->grh.sgid_index; 922 memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid)); 923 rcu_read_lock(); 924 err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num, 925 attr->grh.sgid_index, &sgid, &sgid_attr); 926 rcu_read_unlock(); 927 if (err) { 928 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, 929 "GID lookup at idx=%d with port=%d failed\n", 930 attr->grh.sgid_index, attr->port_num); 931 err = -EINVAL; 932 goto err_gid_l2; 933 } 934 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); 935 rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); 936 ah->av.attrs = *attr; 937 ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr, 938 sgid_attr.gid_type, 939 &sgid); 940 941 if (sgid_attr.ndev) 942 dev_put(sgid_attr.ndev); 943 944 ah->av.sgid_addr.saddr = sgid_addr.saddr; 945 ah->av.dgid_addr.saddr = dgid_addr.saddr; 946 ah_info = &sc_ah->ah_info; 947 ah_info->ah_idx = ah_id; 948 ah_info->pd_idx = pd->sc_pd.pd_id; 949 950 ether_addr_copy(ah_info->mac_addr, if_getlladdr(iwdev->netdev)); 951 if (attr->ah_flags & IB_AH_GRH) { 952 ah_info->flow_label = attr->grh.flow_label; 953 ah_info->hop_ttl = attr->grh.hop_limit; 954 ah_info->tc_tos = attr->grh.traffic_class; 955 } 956 957 if (udata) 958 ib_resolve_eth_dmac(ibpd->device, attr); 959 irdma_ether_copy(dmac, attr); 960 961 irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, 962 dmac, ah->av.net_type); 963 964 err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac); 965 if (err) 966 goto err_gid_l2; 967 968 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, 969 sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); 970 if (err) { 971 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail"); 972 goto err_gid_l2; 973 } 974 975 err = irdma_create_ah_wait(rf, sc_ah, sleep); 976 if (err) { 977 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out"); 978 goto err_gid_l2; 979 } 980 981 if (udata) { 982 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 983 err = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 984 if (err) { 985 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, 986 IRDMA_OP_AH_DESTROY, false, NULL, ah); 987 goto err_gid_l2; 988 } 989 } 990 991 return &ah->ibah; 992 err_gid_l2: 993 kfree(ah); 994 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); 995 996 return ERR_PTR(err); 997 } 998 #endif 999 1000 /** 1001 * irdma_free_qp_rsrc - free up memory resources for qp 1002 * @iwqp: qp ptr (user or kernel) 1003 */ 1004 void 1005 irdma_free_qp_rsrc(struct irdma_qp *iwqp) 1006 { 1007 struct irdma_device *iwdev = iwqp->iwdev; 1008 struct irdma_pci_f *rf = iwdev->rf; 1009 u32 qp_num = iwqp->ibqp.qp_num; 1010 1011 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); 1012 irdma_dealloc_push_page(rf, &iwqp->sc_qp); 1013 if (iwqp->sc_qp.vsi) { 1014 irdma_qp_rem_qos(&iwqp->sc_qp); 1015 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, 1016 iwqp->sc_qp.user_pri); 1017 } 1018 1019 if (qp_num > 2) 1020 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 1021 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem); 1022 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem); 1023 kfree(iwqp->kqp.sig_trk_mem); 1024 iwqp->kqp.sig_trk_mem = NULL; 1025 kfree(iwqp->kqp.sq_wrid_mem); 1026 kfree(iwqp->kqp.rq_wrid_mem); 1027 kfree(iwqp->sg_list); 1028 kfree(iwqp); 1029 } 1030 1031 /** 1032 * irdma_create_qp - create qp 1033 * @ibpd: ptr of pd 1034 * @init_attr: attributes for qp 1035 * @udata: user data for create qp 1036 */ 1037 struct ib_qp * 1038 irdma_create_qp(struct ib_pd *ibpd, 1039 struct ib_qp_init_attr *init_attr, 1040 struct ib_udata *udata) 1041 { 1042 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx) 1043 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd) 1044 struct irdma_pd *iwpd = to_iwpd(ibpd); 1045 struct irdma_device *iwdev = to_iwdev(ibpd->device); 1046 struct irdma_pci_f *rf = iwdev->rf; 1047 struct irdma_qp *iwqp; 1048 struct irdma_create_qp_resp uresp = {0}; 1049 u32 qp_num = 0; 1050 int ret; 1051 int err_code; 1052 struct irdma_sc_qp *qp; 1053 struct irdma_sc_dev *dev = &rf->sc_dev; 1054 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 1055 struct irdma_qp_init_info init_info = {{0}}; 1056 struct irdma_qp_host_ctx_info *ctx_info; 1057 unsigned long flags; 1058 1059 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 1060 if (err_code) 1061 return ERR_PTR(err_code); 1062 1063 if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN || 1064 udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN)) 1065 return ERR_PTR(-EINVAL); 1066 1067 init_info.vsi = &iwdev->vsi; 1068 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 1069 init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr; 1070 init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr; 1071 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 1072 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 1073 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 1074 1075 iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); 1076 if (!iwqp) 1077 return ERR_PTR(-ENOMEM); 1078 1079 iwqp->sg_list = kcalloc(uk_attrs->max_hw_wq_frags, sizeof(*iwqp->sg_list), 1080 GFP_KERNEL); 1081 if (!iwqp->sg_list) { 1082 kfree(iwqp); 1083 return ERR_PTR(-ENOMEM); 1084 } 1085 1086 qp = &iwqp->sc_qp; 1087 qp->qp_uk.back_qp = iwqp; 1088 qp->qp_uk.lock = &iwqp->lock; 1089 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1090 1091 iwqp->iwdev = iwdev; 1092 iwqp->q2_ctx_mem.size = IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE; 1093 iwqp->q2_ctx_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->q2_ctx_mem, 1094 iwqp->q2_ctx_mem.size, 1095 256); 1096 if (!iwqp->q2_ctx_mem.va) { 1097 kfree(iwqp->sg_list); 1098 kfree(iwqp); 1099 return ERR_PTR(-ENOMEM); 1100 } 1101 1102 init_info.q2 = iwqp->q2_ctx_mem.va; 1103 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 1104 init_info.host_ctx = (__le64 *) (init_info.q2 + IRDMA_Q2_BUF_SIZE); 1105 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 1106 1107 if (init_attr->qp_type == IB_QPT_GSI) 1108 qp_num = 1; 1109 else 1110 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 1111 &qp_num, &rf->next_qp); 1112 if (err_code) 1113 goto error; 1114 1115 iwqp->iwpd = iwpd; 1116 iwqp->ibqp.qp_num = qp_num; 1117 qp = &iwqp->sc_qp; 1118 iwqp->iwscq = to_iwcq(init_attr->send_cq); 1119 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 1120 iwqp->host_ctx.va = init_info.host_ctx; 1121 iwqp->host_ctx.pa = init_info.host_ctx_pa; 1122 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 1123 1124 init_info.pd = &iwpd->sc_pd; 1125 init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; 1126 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 1127 init_info.qp_uk_init_info.first_sq_wq = 1; 1128 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 1129 init_waitqueue_head(&iwqp->waitq); 1130 init_waitqueue_head(&iwqp->mod_qp_waitq); 1131 1132 if (udata) { 1133 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 1134 err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr); 1135 } else { 1136 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); 1137 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 1138 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 1139 } 1140 1141 if (err_code) { 1142 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "setup qp failed\n"); 1143 goto error; 1144 } 1145 1146 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 1147 if (init_attr->qp_type == IB_QPT_RC) { 1148 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 1149 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 1150 IRDMA_WRITE_WITH_IMM | 1151 IRDMA_ROCE; 1152 } else { 1153 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 1154 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 1155 IRDMA_ROCE; 1156 } 1157 } else { 1158 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 1159 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 1160 } 1161 1162 ret = irdma_sc_qp_init(qp, &init_info); 1163 if (ret) { 1164 err_code = -EPROTO; 1165 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "qp_init fail\n"); 1166 goto error; 1167 } 1168 1169 ctx_info = &iwqp->ctx_info; 1170 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1171 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1172 1173 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1174 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 1175 else 1176 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 1177 1178 err_code = irdma_cqp_create_qp_cmd(iwqp); 1179 if (err_code) 1180 goto error; 1181 1182 atomic_set(&iwqp->refcnt, 1); 1183 spin_lock_init(&iwqp->lock); 1184 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 1185 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; 1186 rf->qp_table[qp_num] = iwqp; 1187 1188 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 1189 if (dev->ws_add(&iwdev->vsi, 0)) { 1190 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 1191 err_code = -EINVAL; 1192 goto error; 1193 } 1194 1195 irdma_qp_add_qos(&iwqp->sc_qp); 1196 spin_lock_irqsave(&iwpd->udqp_list_lock, flags); 1197 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) 1198 list_add_tail(&iwqp->ud_list_elem, &iwpd->udqp_list); 1199 spin_unlock_irqrestore(&iwpd->udqp_list_lock, flags); 1200 } 1201 1202 if (udata) { 1203 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 1204 if (udata->outlen < sizeof(uresp)) { 1205 uresp.lsmm = 1; 1206 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 1207 } else { 1208 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 1209 uresp.lsmm = 1; 1210 } 1211 uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size; 1212 uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size; 1213 uresp.qp_id = qp_num; 1214 uresp.qp_caps = qp->qp_uk.qp_caps; 1215 1216 err_code = ib_copy_to_udata(udata, &uresp, 1217 min(sizeof(uresp), udata->outlen)); 1218 if (err_code) { 1219 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); 1220 kc_irdma_destroy_qp(&iwqp->ibqp, udata); 1221 return ERR_PTR(err_code); 1222 } 1223 } 1224 1225 init_completion(&iwqp->free_qp); 1226 return &iwqp->ibqp; 1227 1228 error: 1229 irdma_free_qp_rsrc(iwqp); 1230 1231 return ERR_PTR(err_code); 1232 } 1233 1234 /** 1235 * irdma_destroy_qp - destroy qp 1236 * @ibqp: qp's ib pointer also to get to device's qp address 1237 * @udata: user data 1238 */ 1239 #if __FreeBSD_version >= 1400026 1240 int 1241 irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1242 #else 1243 int 1244 irdma_destroy_qp(struct ib_qp *ibqp) 1245 #endif 1246 { 1247 struct irdma_qp *iwqp = to_iwqp(ibqp); 1248 struct irdma_device *iwdev = iwqp->iwdev; 1249 unsigned long flags; 1250 1251 if (iwqp->sc_qp.qp_uk.destroy_pending) 1252 goto free_rsrc; 1253 iwqp->sc_qp.qp_uk.destroy_pending = true; 1254 1255 spin_lock_irqsave(&iwqp->iwpd->udqp_list_lock, flags); 1256 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) 1257 list_del(&iwqp->ud_list_elem); 1258 spin_unlock_irqrestore(&iwqp->iwpd->udqp_list_lock, flags); 1259 1260 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) 1261 irdma_modify_qp_to_err(&iwqp->sc_qp); 1262 1263 irdma_qp_rem_ref(&iwqp->ibqp); 1264 wait_for_completion(&iwqp->free_qp); 1265 irdma_free_lsmm_rsrc(iwqp); 1266 if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) 1267 return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE; 1268 free_rsrc: 1269 if (!iwqp->user_mode) { 1270 if (iwqp->iwscq) { 1271 irdma_clean_cqes(iwqp, iwqp->iwscq); 1272 if (iwqp->iwrcq != iwqp->iwscq) 1273 irdma_clean_cqes(iwqp, iwqp->iwrcq); 1274 } 1275 } 1276 irdma_remove_push_mmap_entries(iwqp); 1277 irdma_free_qp_rsrc(iwqp); 1278 1279 return 0; 1280 } 1281 1282 /** 1283 * irdma_create_cq - create cq 1284 * @ibcq: CQ allocated 1285 * @attr: attributes for cq 1286 * @udata: user data 1287 */ 1288 #if __FreeBSD_version >= 1400026 1289 int 1290 irdma_create_cq(struct ib_cq *ibcq, 1291 const struct ib_cq_init_attr *attr, 1292 struct ib_udata *udata) 1293 #else 1294 struct ib_cq * 1295 irdma_create_cq(struct ib_device *ibdev, 1296 const struct ib_cq_init_attr *attr, 1297 struct ib_ucontext *context, 1298 struct ib_udata *udata) 1299 #endif 1300 { 1301 #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf) 1302 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) 1303 #if __FreeBSD_version >= 1400026 1304 struct ib_device *ibdev = ibcq->device; 1305 #endif 1306 struct irdma_device *iwdev = to_iwdev(ibdev); 1307 struct irdma_pci_f *rf = iwdev->rf; 1308 #if __FreeBSD_version >= 1400026 1309 struct irdma_cq *iwcq = to_iwcq(ibcq); 1310 #else 1311 struct irdma_cq *iwcq; 1312 #endif 1313 u32 cq_num = 0; 1314 struct irdma_sc_cq *cq; 1315 struct irdma_sc_dev *dev = &rf->sc_dev; 1316 struct irdma_cq_init_info info = {0}; 1317 int status; 1318 struct irdma_cqp_request *cqp_request; 1319 struct cqp_cmds_info *cqp_info; 1320 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 1321 unsigned long flags; 1322 int err_code; 1323 int entries = attr->cqe; 1324 bool cqe_64byte_ena; 1325 1326 #if __FreeBSD_version >= 1400026 1327 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 1328 if (err_code) 1329 return err_code; 1330 1331 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || 1332 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) 1333 return -EINVAL; 1334 #else 1335 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 1336 if (err_code) 1337 return ERR_PTR(err_code); 1338 1339 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || 1340 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) 1341 return ERR_PTR(-EINVAL); 1342 1343 iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL); 1344 if (!iwcq) 1345 return ERR_PTR(-ENOMEM); 1346 #endif 1347 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 1348 &rf->next_cq); 1349 if (err_code) 1350 #if __FreeBSD_version >= 1400026 1351 return err_code; 1352 #else 1353 goto error; 1354 #endif 1355 cq = &iwcq->sc_cq; 1356 cq->back_cq = iwcq; 1357 atomic_set(&iwcq->refcnt, 1); 1358 spin_lock_init(&iwcq->lock); 1359 INIT_LIST_HEAD(&iwcq->resize_list); 1360 INIT_LIST_HEAD(&iwcq->cmpl_generated); 1361 info.dev = dev; 1362 ukinfo->cq_size = max(entries, 4); 1363 ukinfo->cq_id = cq_num; 1364 cqe_64byte_ena = (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE) ? true : false; 1365 ukinfo->avoid_mem_cflct = cqe_64byte_ena; 1366 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 1367 atomic_set(&iwcq->armed, 0); 1368 if (attr->comp_vector < rf->ceqs_count) 1369 info.ceq_id = attr->comp_vector; 1370 info.ceq_id_valid = true; 1371 info.ceqe_mask = 1; 1372 info.type = IRDMA_CQ_TYPE_IWARP; 1373 info.vsi = &iwdev->vsi; 1374 1375 if (udata) { 1376 struct irdma_ucontext *ucontext; 1377 struct irdma_create_cq_req req = {0}; 1378 struct irdma_cq_mr *cqmr; 1379 struct irdma_pbl *iwpbl; 1380 struct irdma_pbl *iwpbl_shadow; 1381 struct irdma_cq_mr *cqmr_shadow; 1382 1383 iwcq->user_mode = true; 1384 #if __FreeBSD_version >= 1400026 1385 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); 1386 #else 1387 ucontext = to_ucontext(context); 1388 #endif 1389 1390 if (ib_copy_from_udata(&req, udata, 1391 min(sizeof(req), udata->inlen))) { 1392 err_code = -EFAULT; 1393 goto cq_free_rsrc; 1394 } 1395 1396 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1397 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 1398 &ucontext->cq_reg_mem_list); 1399 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1400 if (!iwpbl) { 1401 err_code = -EPROTO; 1402 goto cq_free_rsrc; 1403 } 1404 iwcq->iwpbl = iwpbl; 1405 iwcq->cq_mem_size = 0; 1406 cqmr = &iwpbl->cq_mr; 1407 1408 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 1409 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { 1410 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1411 iwpbl_shadow = irdma_get_pbl((unsigned long)req.user_shadow_area, 1412 &ucontext->cq_reg_mem_list); 1413 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1414 1415 if (!iwpbl_shadow) { 1416 err_code = -EPROTO; 1417 goto cq_free_rsrc; 1418 } 1419 iwcq->iwpbl_shadow = iwpbl_shadow; 1420 cqmr_shadow = &iwpbl_shadow->cq_mr; 1421 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 1422 cqmr->split = true; 1423 } else { 1424 info.shadow_area_pa = cqmr->shadow; 1425 } 1426 if (iwpbl->pbl_allocated) { 1427 info.virtual_map = true; 1428 info.pbl_chunk_size = 1; 1429 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 1430 } else { 1431 info.cq_base_pa = cqmr->cq_pbl.addr; 1432 } 1433 } else { 1434 /* Kmode allocations */ 1435 int rsize; 1436 1437 if (entries < 1 || entries > rf->max_cqe) { 1438 err_code = -EINVAL; 1439 goto cq_free_rsrc; 1440 } 1441 1442 entries++; 1443 if (!cqe_64byte_ena && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1444 entries *= 2; 1445 ukinfo->cq_size = entries; 1446 1447 if (cqe_64byte_ena) 1448 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_extended_cqe); 1449 else 1450 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 1451 iwcq->kmem.size = round_up(rsize, IRDMA_HW_PAGE_SIZE); 1452 iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem, 1453 iwcq->kmem.size, IRDMA_HW_PAGE_SIZE); 1454 if (!iwcq->kmem.va) { 1455 err_code = -ENOMEM; 1456 goto cq_free_rsrc; 1457 } 1458 1459 iwcq->kmem_shadow.size = IRDMA_SHADOW_AREA_SIZE << 3; 1460 iwcq->kmem_shadow.va = irdma_allocate_dma_mem(dev->hw, 1461 &iwcq->kmem_shadow, 1462 iwcq->kmem_shadow.size, 1463 64); 1464 1465 if (!iwcq->kmem_shadow.va) { 1466 err_code = -ENOMEM; 1467 goto cq_free_rsrc; 1468 } 1469 info.shadow_area_pa = iwcq->kmem_shadow.pa; 1470 ukinfo->shadow_area = iwcq->kmem_shadow.va; 1471 ukinfo->cq_base = iwcq->kmem.va; 1472 info.cq_base_pa = iwcq->kmem.pa; 1473 } 1474 1475 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1476 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 1477 (u32)IRDMA_MAX_CQ_READ_THRESH); 1478 if (irdma_sc_cq_init(cq, &info)) { 1479 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "init cq fail\n"); 1480 err_code = -EPROTO; 1481 goto cq_free_rsrc; 1482 } 1483 1484 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1485 if (!cqp_request) { 1486 err_code = -ENOMEM; 1487 goto cq_free_rsrc; 1488 } 1489 cqp_info = &cqp_request->info; 1490 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1491 cqp_info->post_sq = 1; 1492 cqp_info->in.u.cq_create.cq = cq; 1493 cqp_info->in.u.cq_create.check_overflow = true; 1494 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1495 status = irdma_handle_cqp_op(rf, cqp_request); 1496 irdma_put_cqp_request(&rf->cqp, cqp_request); 1497 if (status) { 1498 err_code = -ENOMEM; 1499 goto cq_free_rsrc; 1500 } 1501 1502 if (udata) { 1503 struct irdma_create_cq_resp resp = {0}; 1504 1505 resp.cq_id = info.cq_uk_init_info.cq_id; 1506 resp.cq_size = info.cq_uk_init_info.cq_size; 1507 if (ib_copy_to_udata(udata, &resp, 1508 min(sizeof(resp), udata->outlen))) { 1509 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy to user data\n"); 1510 err_code = -EPROTO; 1511 goto cq_destroy; 1512 } 1513 } 1514 1515 rf->cq_table[cq_num] = iwcq; 1516 init_completion(&iwcq->free_cq); 1517 1518 #if __FreeBSD_version >= 1400026 1519 return 0; 1520 #else 1521 return &iwcq->ibcq; 1522 #endif 1523 cq_destroy: 1524 irdma_cq_wq_destroy(rf, cq); 1525 cq_free_rsrc: 1526 irdma_cq_free_rsrc(rf, iwcq); 1527 #if __FreeBSD_version >= 1400026 1528 return err_code; 1529 #else 1530 error: 1531 kfree(iwcq); 1532 return ERR_PTR(err_code); 1533 #endif 1534 } 1535 1536 /** 1537 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 1538 * @iwmr: iwmr for IB's user page addresses 1539 * @pbl: ple pointer to save 1 level or 0 level pble 1540 * @level: indicated level 0, 1 or 2 1541 */ 1542 1543 void 1544 irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 1545 enum irdma_pble_level level) 1546 { 1547 struct ib_umem *region = iwmr->region; 1548 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 1549 int chunk_pages, entry, i; 1550 struct scatterlist *sg; 1551 u64 pg_addr = 0; 1552 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 1553 struct irdma_pble_info *pinfo; 1554 u32 idx = 0; 1555 u32 pbl_cnt = 0; 1556 1557 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 1558 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) { 1559 chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size); 1560 if (iwmr->type == IRDMA_MEMREG_TYPE_QP && !iwpbl->qp_mr.sq_page) 1561 iwpbl->qp_mr.sq_page = sg_page(sg); 1562 for (i = 0; i < chunk_pages; i++) { 1563 pg_addr = sg_dma_address(sg) + (i * iwmr->page_size); 1564 if ((entry + i) == 0) 1565 *pbl = pg_addr & iwmr->page_msk; 1566 else if (!(pg_addr & ~iwmr->page_msk)) 1567 *pbl = pg_addr; 1568 else 1569 continue; 1570 if (++pbl_cnt == palloc->total_cnt) 1571 break; 1572 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 1573 } 1574 } 1575 } 1576 1577 /** 1578 * irdma_destroy_ah - Destroy address handle 1579 * @ibah: pointer to address handle 1580 * @ah_flags: destroy flags 1581 */ 1582 1583 #if __FreeBSD_version >= 1400026 1584 void 1585 irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 1586 { 1587 struct irdma_device *iwdev = to_iwdev(ibah->device); 1588 struct irdma_ah *ah = to_iwah(ibah); 1589 1590 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 1591 false, NULL, ah); 1592 1593 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 1594 ah->sc_ah.ah_info.ah_idx); 1595 } 1596 #endif 1597 1598 #if __FreeBSD_version < 1400026 1599 int 1600 irdma_destroy_ah(struct ib_ah *ibah) 1601 { 1602 struct irdma_device *iwdev = to_iwdev(ibah->device); 1603 struct irdma_ah *ah = to_iwah(ibah); 1604 1605 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 1606 false, NULL, ah); 1607 1608 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 1609 ah->sc_ah.ah_info.ah_idx); 1610 1611 kfree(ah); 1612 return 0; 1613 } 1614 #endif 1615 1616 #if __FreeBSD_version >= 1400026 1617 int 1618 irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 1619 #else 1620 int 1621 irdma_dereg_mr(struct ib_mr *ib_mr) 1622 #endif 1623 { 1624 struct irdma_mr *iwmr = to_iwmr(ib_mr); 1625 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 1626 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 1627 int ret; 1628 1629 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 1630 if (iwmr->region) { 1631 struct irdma_ucontext *ucontext; 1632 #if __FreeBSD_version >= 1400026 1633 1634 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); 1635 1636 #else 1637 struct ib_pd *ibpd = ib_mr->pd; 1638 1639 ucontext = to_ucontext(ibpd->uobject->context); 1640 #endif 1641 irdma_del_memlist(iwmr, ucontext); 1642 } 1643 goto done; 1644 } 1645 1646 ret = irdma_hwdereg_mr(ib_mr); 1647 if (ret) 1648 return ret; 1649 1650 irdma_free_stag(iwdev, iwmr->stag); 1651 done: 1652 if (iwpbl->pbl_allocated) 1653 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); 1654 1655 if (iwmr->region) 1656 ib_umem_release(iwmr->region); 1657 1658 kfree(iwmr); 1659 1660 return 0; 1661 } 1662 1663 /* 1664 * irdma_rereg_user_mr - Re-Register a user memory region @ibmr: ib mem to access iwarp mr pointer @flags: bit mask to 1665 * indicate which of the attr's of MR modified @start: virtual start address @len: length of mr @virt: virtual address 1666 * @new access flags: bit mask of access flags @new_pd: ptr of pd @udata: user data 1667 */ 1668 int 1669 irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len, 1670 u64 virt, int new_access, struct ib_pd *new_pd, 1671 struct ib_udata *udata) 1672 { 1673 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 1674 struct irdma_mr *iwmr = to_iwmr(ib_mr); 1675 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 1676 int ret; 1677 1678 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 1679 return -EINVAL; 1680 1681 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) 1682 return -EOPNOTSUPP; 1683 1684 ret = irdma_hwdereg_mr(ib_mr); 1685 if (ret) 1686 return ret; 1687 1688 if (flags & IB_MR_REREG_ACCESS) 1689 iwmr->access = new_access; 1690 1691 if (flags & IB_MR_REREG_PD) { 1692 iwmr->ibmr.pd = new_pd; 1693 iwmr->ibmr.device = new_pd->device; 1694 } 1695 1696 if (flags & IB_MR_REREG_TRANS) { 1697 if (iwpbl->pbl_allocated) { 1698 irdma_free_pble(iwdev->rf->pble_rsrc, 1699 &iwpbl->pble_alloc); 1700 iwpbl->pbl_allocated = false; 1701 } 1702 if (iwmr->region) { 1703 ib_umem_release(iwmr->region); 1704 iwmr->region = NULL; 1705 } 1706 1707 ib_mr = irdma_rereg_mr_trans(iwmr, start, len, virt, udata); 1708 if (IS_ERR(ib_mr)) 1709 return PTR_ERR(ib_mr); 1710 1711 } else { 1712 ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access); 1713 if (ret) 1714 return ret; 1715 } 1716 1717 return 0; 1718 } 1719 1720 int 1721 kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr, 1722 u16 *vlan_id) 1723 { 1724 int ret; 1725 union ib_gid sgid; 1726 struct ib_gid_attr sgid_attr; 1727 struct irdma_av *av = &iwqp->roce_ah.av; 1728 1729 ret = ib_get_cached_gid(iwqp->ibqp.device, attr->ah_attr.port_num, 1730 attr->ah_attr.grh.sgid_index, &sgid, 1731 &sgid_attr); 1732 if (ret) 1733 return ret; 1734 1735 if (sgid_attr.ndev) { 1736 *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev); 1737 ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, if_getlladdr(sgid_attr.ndev)); 1738 } 1739 1740 av->net_type = kc_rdma_gid_attr_network_type(sgid_attr, 1741 sgid_attr.gid_type, 1742 &sgid); 1743 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid); 1744 dev_put(sgid_attr.ndev); 1745 iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri; 1746 1747 return 0; 1748 } 1749 1750 #if __FreeBSD_version >= 1400026 1751 /** 1752 * irdma_destroy_cq - destroy cq 1753 * @ib_cq: cq pointer 1754 * @udata: user data 1755 */ 1756 void 1757 irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1758 { 1759 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1760 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1761 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1762 struct irdma_sc_dev *dev = cq->dev; 1763 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1764 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1765 unsigned long flags; 1766 1767 spin_lock_irqsave(&iwcq->lock, flags); 1768 if (!list_empty(&iwcq->cmpl_generated)) 1769 irdma_remove_cmpls_list(iwcq); 1770 if (!list_empty(&iwcq->resize_list)) 1771 irdma_process_resize_list(iwcq, iwdev, NULL); 1772 spin_unlock_irqrestore(&iwcq->lock, flags); 1773 1774 irdma_cq_rem_ref(ib_cq); 1775 wait_for_completion(&iwcq->free_cq); 1776 1777 irdma_cq_wq_destroy(iwdev->rf, cq); 1778 1779 spin_lock_irqsave(&iwceq->ce_lock, flags); 1780 irdma_sc_cleanup_ceqes(cq, ceq); 1781 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1782 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1783 } 1784 1785 #endif 1786 #if __FreeBSD_version < 1400026 1787 /** 1788 * irdma_destroy_cq - destroy cq 1789 * @ib_cq: cq pointer 1790 */ 1791 int 1792 irdma_destroy_cq(struct ib_cq *ib_cq) 1793 { 1794 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1795 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1796 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1797 struct irdma_sc_dev *dev = cq->dev; 1798 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1799 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1800 unsigned long flags; 1801 1802 spin_lock_irqsave(&iwcq->lock, flags); 1803 if (!list_empty(&iwcq->cmpl_generated)) 1804 irdma_remove_cmpls_list(iwcq); 1805 if (!list_empty(&iwcq->resize_list)) 1806 irdma_process_resize_list(iwcq, iwdev, NULL); 1807 spin_unlock_irqrestore(&iwcq->lock, flags); 1808 1809 irdma_cq_rem_ref(ib_cq); 1810 wait_for_completion(&iwcq->free_cq); 1811 1812 irdma_cq_wq_destroy(iwdev->rf, cq); 1813 1814 spin_lock_irqsave(&iwceq->ce_lock, flags); 1815 irdma_sc_cleanup_ceqes(cq, ceq); 1816 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1817 1818 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1819 kfree(iwcq); 1820 1821 return 0; 1822 } 1823 1824 #endif 1825 /** 1826 * irdma_alloc_mw - Allocate memory window 1827 * @pd: Protection domain 1828 * @type: Window type 1829 * @udata: user data pointer 1830 */ 1831 struct ib_mw * 1832 irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, 1833 struct ib_udata *udata) 1834 { 1835 struct irdma_device *iwdev = to_iwdev(pd->device); 1836 struct irdma_mr *iwmr; 1837 int err_code; 1838 u32 stag; 1839 1840 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 1841 if (!iwmr) 1842 return ERR_PTR(-ENOMEM); 1843 1844 stag = irdma_create_stag(iwdev); 1845 if (!stag) { 1846 kfree(iwmr); 1847 return ERR_PTR(-ENOMEM); 1848 } 1849 1850 iwmr->stag = stag; 1851 iwmr->ibmw.rkey = stag; 1852 iwmr->ibmw.pd = pd; 1853 iwmr->ibmw.type = type; 1854 iwmr->ibmw.device = pd->device; 1855 1856 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 1857 if (err_code) { 1858 irdma_free_stag(iwdev, stag); 1859 kfree(iwmr); 1860 return ERR_PTR(err_code); 1861 } 1862 1863 return &iwmr->ibmw; 1864 } 1865 1866 /** 1867 * kc_set_loc_seq_num_mss - Set local seq number and mss 1868 * @cm_node: cm node info 1869 */ 1870 void 1871 kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node) 1872 { 1873 struct timespec ts; 1874 1875 getnanotime(&ts); 1876 cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec; 1877 if (cm_node->iwdev->vsi.mtu > 1500 && 1878 2 * cm_node->iwdev->vsi.mtu > cm_node->iwdev->rcv_wnd) 1879 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? 1880 (1500 - IRDMA_MTU_TO_MSS_IPV4) : 1881 (1500 - IRDMA_MTU_TO_MSS_IPV6); 1882 else 1883 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? 1884 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4) : 1885 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6); 1886 } 1887 1888 #if __FreeBSD_version < 1400026 1889 struct irdma_vma_data { 1890 struct list_head list; 1891 struct vm_area_struct *vma; 1892 struct mutex *vma_list_mutex; /* protect the vma_list */ 1893 }; 1894 1895 /** 1896 * irdma_vma_open - 1897 * @vma: User VMA 1898 */ 1899 static void 1900 irdma_vma_open(struct vm_area_struct *vma) 1901 { 1902 vma->vm_ops = NULL; 1903 } 1904 1905 /** 1906 * irdma_vma_close - Remove vma data from vma list 1907 * @vma: User VMA 1908 */ 1909 static void 1910 irdma_vma_close(struct vm_area_struct *vma) 1911 { 1912 struct irdma_vma_data *vma_data; 1913 1914 vma_data = vma->vm_private_data; 1915 vma->vm_private_data = NULL; 1916 vma_data->vma = NULL; 1917 mutex_lock(vma_data->vma_list_mutex); 1918 list_del(&vma_data->list); 1919 mutex_unlock(vma_data->vma_list_mutex); 1920 kfree(vma_data); 1921 } 1922 1923 static const struct vm_operations_struct irdma_vm_ops = { 1924 .open = irdma_vma_open, 1925 .close = irdma_vma_close 1926 }; 1927 1928 /** 1929 * irdma_set_vma_data - Save vma data in context list 1930 * @vma: User VMA 1931 * @context: ib user context 1932 */ 1933 static int 1934 irdma_set_vma_data(struct vm_area_struct *vma, 1935 struct irdma_ucontext *context) 1936 { 1937 struct list_head *vma_head = &context->vma_list; 1938 struct irdma_vma_data *vma_entry; 1939 1940 vma_entry = kzalloc(sizeof(*vma_entry), GFP_KERNEL); 1941 if (!vma_entry) 1942 return -ENOMEM; 1943 1944 vma->vm_private_data = vma_entry; 1945 vma->vm_ops = &irdma_vm_ops; 1946 1947 vma_entry->vma = vma; 1948 vma_entry->vma_list_mutex = &context->vma_list_mutex; 1949 1950 mutex_lock(&context->vma_list_mutex); 1951 list_add(&vma_entry->list, vma_head); 1952 mutex_unlock(&context->vma_list_mutex); 1953 1954 return 0; 1955 } 1956 1957 /** 1958 * irdma_disassociate_ucontext - Disassociate user context 1959 * @context: ib user context 1960 */ 1961 void 1962 irdma_disassociate_ucontext(struct ib_ucontext *context) 1963 { 1964 struct irdma_ucontext *ucontext = to_ucontext(context); 1965 1966 struct irdma_vma_data *vma_data, *n; 1967 struct vm_area_struct *vma; 1968 1969 mutex_lock(&ucontext->vma_list_mutex); 1970 list_for_each_entry_safe(vma_data, n, &ucontext->vma_list, list) { 1971 vma = vma_data->vma; 1972 zap_vma_ptes(vma, vma->vm_start, PAGE_SIZE); 1973 1974 vma->vm_ops = NULL; 1975 list_del(&vma_data->list); 1976 kfree(vma_data); 1977 } 1978 mutex_unlock(&ucontext->vma_list_mutex); 1979 } 1980 1981 int 1982 rdma_user_mmap_io(struct ib_ucontext *context, struct vm_area_struct *vma, 1983 unsigned long pfn, unsigned long size, pgprot_t prot) 1984 { 1985 if (io_remap_pfn_range(vma, 1986 vma->vm_start, 1987 pfn, 1988 size, 1989 prot)) 1990 return -EAGAIN; 1991 1992 return irdma_set_vma_data(vma, to_ucontext(context)); 1993 } 1994 #else 1995 /** 1996 * irdma_disassociate_ucontext - Disassociate user context 1997 * @context: ib user context 1998 */ 1999 void 2000 irdma_disassociate_ucontext(struct ib_ucontext *context) 2001 { 2002 } 2003 #endif 2004 2005 struct ib_device * 2006 ib_device_get_by_netdev(if_t netdev, int driver_id) 2007 { 2008 struct irdma_device *iwdev; 2009 struct irdma_handler *hdl; 2010 unsigned long flags; 2011 2012 spin_lock_irqsave(&irdma_handler_lock, flags); 2013 list_for_each_entry(hdl, &irdma_handlers, list) { 2014 iwdev = hdl->iwdev; 2015 if (netdev == iwdev->netdev) { 2016 spin_unlock_irqrestore(&irdma_handler_lock, 2017 flags); 2018 return &iwdev->ibdev; 2019 } 2020 } 2021 spin_unlock_irqrestore(&irdma_handler_lock, flags); 2022 2023 return NULL; 2024 } 2025 2026 void 2027 ib_unregister_device_put(struct ib_device *device) 2028 { 2029 ib_unregister_device(device); 2030 } 2031 2032 /** 2033 * irdma_query_gid_roce - Query port GID for Roce 2034 * @ibdev: device pointer from stack 2035 * @port: port number 2036 * @index: Entry index 2037 * @gid: Global ID 2038 */ 2039 int 2040 irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index, 2041 union ib_gid *gid) 2042 { 2043 int ret; 2044 2045 ret = rdma_query_gid(ibdev, port, index, gid); 2046 if (ret == -EAGAIN) { 2047 memcpy(gid, &zgid, sizeof(*gid)); 2048 return 0; 2049 } 2050 2051 return ret; 2052 } 2053 2054 /** 2055 * irdma_modify_port - modify port attributes 2056 * @ibdev: device pointer from stack 2057 * @port: port number for query 2058 * @mask: Property mask 2059 * @props: returning device attributes 2060 */ 2061 int 2062 irdma_modify_port(struct ib_device *ibdev, u8 port, int mask, 2063 struct ib_port_modify *props) 2064 { 2065 if (port > 1) 2066 return -EINVAL; 2067 2068 return 0; 2069 } 2070 2071 /** 2072 * irdma_query_pkey - Query partition key 2073 * @ibdev: device pointer from stack 2074 * @port: port number 2075 * @index: index of pkey 2076 * @pkey: pointer to store the pkey 2077 */ 2078 int 2079 irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, 2080 u16 *pkey) 2081 { 2082 if (index >= IRDMA_PKEY_TBL_SZ) 2083 return -EINVAL; 2084 2085 *pkey = IRDMA_DEFAULT_PKEY; 2086 return 0; 2087 } 2088 2089 int 2090 irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num, 2091 struct ib_port_immutable *immutable) 2092 { 2093 struct ib_port_attr attr; 2094 int err; 2095 2096 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 2097 err = ib_query_port(ibdev, port_num, &attr); 2098 if (err) 2099 return err; 2100 2101 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 2102 immutable->pkey_tbl_len = attr.pkey_tbl_len; 2103 immutable->gid_tbl_len = attr.gid_tbl_len; 2104 2105 return 0; 2106 } 2107 2108 int 2109 irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num, 2110 struct ib_port_immutable *immutable) 2111 { 2112 struct ib_port_attr attr; 2113 int err; 2114 2115 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 2116 err = ib_query_port(ibdev, port_num, &attr); 2117 if (err) 2118 return err; 2119 immutable->gid_tbl_len = 1; 2120 2121 return 0; 2122 } 2123 2124 /** 2125 * irdma_query_port - get port attributes 2126 * @ibdev: device pointer from stack 2127 * @port: port number for query 2128 * @props: returning device attributes 2129 */ 2130 int 2131 irdma_query_port(struct ib_device *ibdev, u8 port, 2132 struct ib_port_attr *props) 2133 { 2134 struct irdma_device *iwdev = to_iwdev(ibdev); 2135 if_t netdev = iwdev->netdev; 2136 2137 /* no need to zero out pros here. done by caller */ 2138 2139 props->max_mtu = IB_MTU_4096; 2140 props->active_mtu = ib_mtu_int_to_enum(if_getmtu(netdev)); 2141 props->lid = 1; 2142 props->lmc = 0; 2143 props->sm_lid = 0; 2144 props->sm_sl = 0; 2145 if ((if_getlinkstate(netdev) == LINK_STATE_UP) && 2146 (if_getdrvflags(netdev) & IFF_DRV_RUNNING)) { 2147 props->state = IB_PORT_ACTIVE; 2148 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 2149 } else { 2150 props->state = IB_PORT_DOWN; 2151 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 2152 } 2153 ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width); 2154 2155 if (rdma_protocol_roce(ibdev, 1)) { 2156 props->gid_tbl_len = 32; 2157 kc_set_props_ip_gid_caps(props); 2158 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 2159 } else { 2160 props->gid_tbl_len = 1; 2161 } 2162 props->qkey_viol_cntr = 0; 2163 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 2164 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 2165 2166 return 0; 2167 } 2168 2169 static const char *const irdma_hw_stat_names[] = { 2170 /* gen1 - 32-bit */ 2171 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards", 2172 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts", 2173 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes", 2174 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards", 2175 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts", 2176 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes", 2177 [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors", 2178 /* gen1 - 64-bit */ 2179 [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = "ip4InOctets", 2180 [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = "ip4InPkts", 2181 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = "ip4InReasmRqd", 2182 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = "ip4InMcastPkts", 2183 [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = "ip4OutOctets", 2184 [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = "ip4OutPkts", 2185 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = "ip4OutSegRqd", 2186 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = "ip4OutMcastPkts", 2187 [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = "ip6InOctets", 2188 [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = "ip6InPkts", 2189 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = "ip6InReasmRqd", 2190 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = "ip6InMcastPkts", 2191 [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = "ip6OutOctets", 2192 [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = "ip6OutPkts", 2193 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = "ip6OutSegRqd", 2194 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = "ip6OutMcastPkts", 2195 [IRDMA_HW_STAT_INDEX_RDMARXRDS] = "InRdmaReads", 2196 [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = "InRdmaSends", 2197 [IRDMA_HW_STAT_INDEX_RDMARXWRS] = "InRdmaWrites", 2198 [IRDMA_HW_STAT_INDEX_RDMATXRDS] = "OutRdmaReads", 2199 [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = "OutRdmaSends", 2200 [IRDMA_HW_STAT_INDEX_RDMATXWRS] = "OutRdmaWrites", 2201 [IRDMA_HW_STAT_INDEX_RDMAVBND] = "RdmaBnd", 2202 [IRDMA_HW_STAT_INDEX_RDMAVINV] = "RdmaInv", 2203 2204 /* gen2 - 32-bit */ 2205 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled", 2206 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored", 2207 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent", 2208 /* gen2 - 64-bit */ 2209 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = "ip4InMcastOctets", 2210 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = "ip4OutMcastOctets", 2211 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = "ip6InMcastOctets", 2212 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = "ip6OutMcastOctets", 2213 [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = "RxUDP", 2214 [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = "TxUDP", 2215 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = "RxECNMrkd", 2216 [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "RetransSegs", 2217 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "InOptErrors", 2218 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "InProtoErrors", 2219 [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = "InSegs", 2220 [IRDMA_HW_STAT_INDEX_TCPTXSEG] = "OutSegs", 2221 }; 2222 2223 /** 2224 * irdma_alloc_hw_stats - Allocate a hw stats structure 2225 * @ibdev: device pointer from stack 2226 * @port_num: port number 2227 */ 2228 struct rdma_hw_stats * 2229 irdma_alloc_hw_stats(struct ib_device *ibdev, 2230 u8 port_num) 2231 { 2232 struct irdma_device *iwdev = to_iwdev(ibdev); 2233 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 2234 2235 int num_counters = dev->hw_attrs.max_stat_idx; 2236 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 2237 2238 return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters, 2239 lifespan); 2240 } 2241 2242 /** 2243 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 2244 * @ibdev: device pointer from stack 2245 * @stats: stats pointer from stack 2246 * @port_num: port number 2247 * @index: which hw counter the stack is requesting we update 2248 */ 2249 int 2250 irdma_get_hw_stats(struct ib_device *ibdev, 2251 struct rdma_hw_stats *stats, u8 port_num, 2252 int index) 2253 { 2254 struct irdma_device *iwdev = to_iwdev(ibdev); 2255 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 2256 2257 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 2258 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 2259 2260 memcpy(&stats->value[0], hw_stats, sizeof(u64)* stats->num_counters); 2261 2262 return stats->num_counters; 2263 } 2264 2265 /** 2266 * irdma_query_gid - Query port GID 2267 * @ibdev: device pointer from stack 2268 * @port: port number 2269 * @index: Entry index 2270 * @gid: Global ID 2271 */ 2272 int 2273 irdma_query_gid(struct ib_device *ibdev, u8 port, int index, 2274 union ib_gid *gid) 2275 { 2276 struct irdma_device *iwdev = to_iwdev(ibdev); 2277 2278 memset(gid->raw, 0, sizeof(gid->raw)); 2279 ether_addr_copy(gid->raw, if_getlladdr(iwdev->netdev)); 2280 2281 return 0; 2282 } 2283 2284 enum rdma_link_layer 2285 irdma_get_link_layer(struct ib_device *ibdev, 2286 u8 port_num) 2287 { 2288 return IB_LINK_LAYER_ETHERNET; 2289 } 2290 2291 inline enum ib_mtu 2292 ib_mtu_int_to_enum(int mtu) 2293 { 2294 if (mtu >= 4096) 2295 return IB_MTU_4096; 2296 else if (mtu >= 2048) 2297 return IB_MTU_2048; 2298 else if (mtu >= 1024) 2299 return IB_MTU_1024; 2300 else if (mtu >= 512) 2301 return IB_MTU_512; 2302 else 2303 return IB_MTU_256; 2304 } 2305 2306 inline void 2307 kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev) 2308 { 2309 iwdev->ibdev.uverbs_cmd_mask |= 2310 BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) | 2311 BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) | 2312 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) | 2313 BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST); 2314 } 2315 2316 inline void 2317 kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev) 2318 { 2319 iwdev->ibdev.uverbs_cmd_mask = 2320 BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) | 2321 BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) | 2322 BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) | 2323 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) | 2324 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) | 2325 BIT_ULL(IB_USER_VERBS_CMD_REG_MR) | 2326 BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) | 2327 BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) | 2328 BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 2329 BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) | 2330 BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ) | 2331 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) | 2332 BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | 2333 BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) | 2334 BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) | 2335 BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) | 2336 BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) | 2337 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) | 2338 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) | 2339 BIT_ULL(IB_USER_VERBS_CMD_BIND_MW) | 2340 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) | 2341 BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) | 2342 BIT_ULL(IB_USER_VERBS_CMD_POST_SEND); 2343 iwdev->ibdev.uverbs_ex_cmd_mask = 2344 BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP) | 2345 BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE); 2346 2347 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 2348 iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ); 2349 } 2350 2351 int 2352 ib_get_eth_speed(struct ib_device *ibdev, u32 port_num, u8 *speed, u8 *width) 2353 { 2354 if_t netdev = ibdev->get_netdev(ibdev, port_num); 2355 u32 netdev_speed; 2356 2357 if (!netdev) 2358 return -ENODEV; 2359 2360 netdev_speed = if_getbaudrate(netdev); 2361 dev_put(netdev); 2362 if (netdev_speed <= SPEED_1000) { 2363 *width = IB_WIDTH_1X; 2364 *speed = IB_SPEED_SDR; 2365 } else if (netdev_speed <= SPEED_10000) { 2366 *width = IB_WIDTH_1X; 2367 *speed = IB_SPEED_FDR10; 2368 } else if (netdev_speed <= SPEED_20000) { 2369 *width = IB_WIDTH_4X; 2370 *speed = IB_SPEED_DDR; 2371 } else if (netdev_speed <= SPEED_25000) { 2372 *width = IB_WIDTH_1X; 2373 *speed = IB_SPEED_EDR; 2374 } else if (netdev_speed <= SPEED_40000) { 2375 *width = IB_WIDTH_4X; 2376 *speed = IB_SPEED_FDR10; 2377 } else { 2378 *width = IB_WIDTH_4X; 2379 *speed = IB_SPEED_EDR; 2380 } 2381 2382 return 0; 2383 } 2384