1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 #include <sys/cdefs.h> 35 #include "opt_inet.h" 36 37 #ifdef TCP_OFFLOAD 38 #include <linux/types.h> 39 #include <linux/kref.h> 40 #include <rdma/ib_umem.h> 41 #include <asm/atomic.h> 42 43 #include <common/t4_msg.h> 44 #include "iw_cxgbe.h" 45 46 #define T4_ULPTX_MIN_IO 32 47 #define C4IW_MAX_INLINE_SIZE 96 48 #define T4_ULPTX_MAX_DMA 1024 49 50 static int 51 mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length) 52 { 53 54 return (is_t5(dev->rdev.adap) && length >= 8*1024*1024*1024ULL); 55 } 56 57 static int 58 _c4iw_write_mem_dma_aligned(struct c4iw_rdev *rdev, u32 addr, u32 len, 59 dma_addr_t data, int wait) 60 { 61 struct adapter *sc = rdev->adap; 62 u8 wr_len; 63 int ret = 0; 64 struct c4iw_wr_wait wr_wait; 65 struct wrqe *wr; 66 67 if (wait) 68 c4iw_init_wr_wait(&wr_wait); 69 wr_len = T4_WRITE_MEM_DMA_LEN; 70 71 wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]); 72 if (wr == NULL) 73 return -ENOMEM; 74 t4_write_mem_dma_wr(sc, wrtod(wr), wr_len, 0, addr, len, data, 75 wait ? (u64)(unsigned long)&wr_wait : 0); 76 t4_wrq_tx(sc, wr); 77 78 if (wait) 79 ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__); 80 return ret; 81 } 82 83 84 static int 85 _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data) 86 { 87 struct adapter *sc = rdev->adap; 88 u8 wr_len, *from_dp; 89 int copy_len, num_wqe, i, ret = 0; 90 struct c4iw_wr_wait wr_wait; 91 struct wrqe *wr; 92 93 CTR3(KTR_IW_CXGBE, "%s addr 0x%x len %u", __func__, addr, len); 94 c4iw_init_wr_wait(&wr_wait); 95 num_wqe = DIV_ROUND_UP(len, T4_MAX_INLINE_SIZE); 96 from_dp = data; 97 for (i = 0; i < num_wqe; i++) { 98 copy_len = min(len, T4_MAX_INLINE_SIZE); 99 wr_len = T4_WRITE_MEM_INLINE_LEN(copy_len); 100 101 wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]); 102 if (wr == NULL) 103 return -ENOMEM; 104 t4_write_mem_inline_wr(sc, wrtod(wr), wr_len, 0, addr, copy_len, 105 from_dp, i == (num_wqe - 1) ? 106 (__force __be64)(unsigned long) &wr_wait : 0); 107 t4_wrq_tx(sc, wr); 108 109 if (from_dp != NULL) 110 from_dp += T4_MAX_INLINE_SIZE; 111 addr += T4_MAX_INLINE_SIZE >> 5; 112 len -= T4_MAX_INLINE_SIZE; 113 } 114 ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__); 115 return ret; 116 } 117 118 static int 119 _c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data) 120 { 121 struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev); 122 u32 remain = len; 123 u32 dmalen; 124 int ret = 0; 125 dma_addr_t daddr; 126 dma_addr_t save; 127 128 daddr = dma_map_single(rhp->ibdev.dma_device, data, len, DMA_TO_DEVICE); 129 if (dma_mapping_error(rhp->ibdev.dma_device, daddr)) 130 return -1; 131 save = daddr; 132 133 while (remain > inline_threshold) { 134 if (remain < T4_ULPTX_MAX_DMA) { 135 if (remain & ~T4_ULPTX_MIN_IO) 136 dmalen = remain & ~(T4_ULPTX_MIN_IO-1); 137 else 138 dmalen = remain; 139 } else 140 dmalen = T4_ULPTX_MAX_DMA; 141 remain -= dmalen; 142 ret = _c4iw_write_mem_dma_aligned(rdev, addr, dmalen, 143 daddr, !remain); 144 if (ret) 145 goto out; 146 addr += dmalen >> 5; 147 data = (u8 *)data + dmalen; 148 daddr = daddr + dmalen; 149 } 150 if (remain) 151 ret = _c4iw_write_mem_inline(rdev, addr, remain, data); 152 out: 153 dma_unmap_single(rhp->ibdev.dma_device, save, len, DMA_TO_DEVICE); 154 return ret; 155 } 156 157 /* 158 * write len bytes of data into addr (32B aligned address) 159 * If data is NULL, clear len byte of memory to zero. 160 */ 161 static int 162 write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, 163 void *data) 164 { 165 if (rdev->adap->params.ulptx_memwrite_dsgl && use_dsgl) { 166 if (len > inline_threshold) { 167 if (_c4iw_write_mem_dma(rdev, addr, len, data)) { 168 log(LOG_ERR, "%s: dma map " 169 "failure (non fatal)\n", __func__); 170 return _c4iw_write_mem_inline(rdev, addr, len, 171 data); 172 } else 173 return 0; 174 } else 175 return _c4iw_write_mem_inline(rdev, addr, len, data); 176 } else 177 return _c4iw_write_mem_inline(rdev, addr, len, data); 178 } 179 180 181 /* 182 * Build and write a TPT entry. 183 * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size, 184 * pbl_size and pbl_addr 185 * OUT: stag index 186 */ 187 static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, 188 u32 *stag, u8 stag_state, u32 pdid, 189 enum fw_ri_stag_type type, enum fw_ri_mem_perms perm, 190 int bind_enabled, u32 zbva, u64 to, 191 u64 len, u8 page_size, u32 pbl_size, u32 pbl_addr) 192 { 193 int err; 194 struct fw_ri_tpte tpt; 195 u32 stag_idx; 196 static atomic_t key; 197 198 if (c4iw_stopped(rdev)) 199 return -EIO; 200 201 stag_state = stag_state > 0; 202 stag_idx = (*stag) >> 8; 203 204 if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) { 205 stag_idx = t4_stag_alloc(rdev->adap, 1); 206 if (stag_idx == T4_STAG_UNSET) { 207 mutex_lock(&rdev->stats.lock); 208 rdev->stats.stag.fail++; 209 mutex_unlock(&rdev->stats.lock); 210 return -ENOMEM; 211 } 212 mutex_lock(&rdev->stats.lock); 213 rdev->stats.stag.cur += 32; 214 if (rdev->stats.stag.cur > rdev->stats.stag.max) 215 rdev->stats.stag.max = rdev->stats.stag.cur; 216 mutex_unlock(&rdev->stats.lock); 217 *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff); 218 } 219 CTR5(KTR_IW_CXGBE, 220 "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x", 221 __func__, stag_state, type, pdid, stag_idx); 222 223 /* write TPT entry */ 224 if (reset_tpt_entry) 225 memset(&tpt, 0, sizeof(tpt)); 226 else { 227 if (page_size > ilog2(C4IW_MAX_PAGE_SIZE) - 12) 228 return -EINVAL; 229 tpt.valid_to_pdid = cpu_to_be32(F_FW_RI_TPTE_VALID | 230 V_FW_RI_TPTE_STAGKEY((*stag & M_FW_RI_TPTE_STAGKEY)) | 231 V_FW_RI_TPTE_STAGSTATE(stag_state) | 232 V_FW_RI_TPTE_STAGTYPE(type) | V_FW_RI_TPTE_PDID(pdid)); 233 tpt.locread_to_qpid = cpu_to_be32(V_FW_RI_TPTE_PERM(perm) | 234 (bind_enabled ? F_FW_RI_TPTE_MWBINDEN : 0) | 235 V_FW_RI_TPTE_ADDRTYPE((zbva ? FW_RI_ZERO_BASED_TO : 236 FW_RI_VA_BASED_TO))| 237 V_FW_RI_TPTE_PS(page_size)); 238 tpt.nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32( 239 V_FW_RI_TPTE_PBLADDR(PBL_OFF(rdev, pbl_addr)>>3)); 240 tpt.len_lo = cpu_to_be32((u32)(len & 0xffffffffUL)); 241 tpt.va_hi = cpu_to_be32((u32)(to >> 32)); 242 tpt.va_lo_fbo = cpu_to_be32((u32)(to & 0xffffffffUL)); 243 tpt.dca_mwbcnt_pstag = cpu_to_be32(0); 244 tpt.len_hi = cpu_to_be32((u32)(len >> 32)); 245 } 246 err = write_adapter_mem(rdev, stag_idx + 247 (rdev->adap->vres.stag.start >> 5), 248 sizeof(tpt), &tpt); 249 250 if (reset_tpt_entry) { 251 t4_stag_free(rdev->adap, stag_idx, 1); 252 mutex_lock(&rdev->stats.lock); 253 rdev->stats.stag.cur -= 32; 254 mutex_unlock(&rdev->stats.lock); 255 } 256 return err; 257 } 258 259 static int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl, 260 u32 pbl_addr, u32 pbl_size) 261 { 262 int err; 263 264 CTR4(KTR_IW_CXGBE, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d", 265 __func__, pbl_addr, rdev->adap->vres.pbl.start, pbl_size); 266 267 err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl); 268 return err; 269 } 270 271 static int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size, 272 u32 pbl_addr) 273 { 274 return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 275 pbl_size, pbl_addr); 276 } 277 278 static int allocate_window(struct c4iw_rdev *rdev, u32 * stag, u32 pdid) 279 { 280 *stag = T4_STAG_UNSET; 281 return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0, 282 0UL, 0, 0, 0, 0); 283 } 284 285 static int deallocate_window(struct c4iw_rdev *rdev, u32 stag) 286 { 287 return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0, 288 0); 289 } 290 291 static int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid, 292 u32 pbl_size, u32 pbl_addr) 293 { 294 *stag = T4_STAG_UNSET; 295 return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_NSMR, 0, 0, 0, 296 0UL, 0, 0, pbl_size, pbl_addr); 297 } 298 299 static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag) 300 { 301 u32 mmid; 302 303 mhp->attr.state = 1; 304 mhp->attr.stag = stag; 305 mmid = stag >> 8; 306 mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 307 CTR3(KTR_IW_CXGBE, "%s mmid 0x%x mhp %p", __func__, mmid, mhp); 308 return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); 309 } 310 311 static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, 312 struct c4iw_mr *mhp, int shift) 313 { 314 u32 stag = T4_STAG_UNSET; 315 int ret; 316 317 ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, 318 FW_RI_STAG_NSMR, mhp->attr.len ? mhp->attr.perms : 0, 319 mhp->attr.mw_bind_enable, mhp->attr.zbva, 320 mhp->attr.va_fbo, mhp->attr.len ? mhp->attr.len : -1, shift - 12, 321 mhp->attr.pbl_size, mhp->attr.pbl_addr); 322 if (ret) 323 return ret; 324 325 ret = finish_mem_reg(mhp, stag); 326 if (ret) 327 dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 328 mhp->attr.pbl_addr); 329 return ret; 330 } 331 332 static int alloc_pbl(struct c4iw_mr *mhp, int npages) 333 { 334 mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev, 335 npages << 3); 336 337 if (!mhp->attr.pbl_addr) 338 return -ENOMEM; 339 340 mhp->attr.pbl_size = npages; 341 342 return 0; 343 } 344 345 struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) 346 { 347 struct c4iw_dev *rhp; 348 struct c4iw_pd *php; 349 struct c4iw_mr *mhp; 350 int ret; 351 u32 stag = T4_STAG_UNSET; 352 353 CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); 354 php = to_c4iw_pd(pd); 355 rhp = php->rhp; 356 357 mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 358 if (!mhp) 359 return ERR_PTR(-ENOMEM); 360 361 mhp->rhp = rhp; 362 mhp->attr.pdid = php->pdid; 363 mhp->attr.perms = c4iw_ib_to_tpt_access(acc); 364 mhp->attr.mw_bind_enable = (acc&IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND; 365 mhp->attr.zbva = 0; 366 mhp->attr.va_fbo = 0; 367 mhp->attr.page_size = 0; 368 mhp->attr.len = ~0ULL; 369 mhp->attr.pbl_size = 0; 370 371 ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid, 372 FW_RI_STAG_NSMR, mhp->attr.perms, 373 mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0); 374 if (ret) 375 goto err1; 376 377 ret = finish_mem_reg(mhp, stag); 378 if (ret) 379 goto err2; 380 return &mhp->ibmr; 381 err2: 382 dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 383 mhp->attr.pbl_addr); 384 err1: 385 kfree(mhp); 386 return ERR_PTR(ret); 387 } 388 389 struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 390 u64 virt, int acc, struct ib_udata *udata) 391 { 392 __be64 *pages; 393 int shift, n, len; 394 int i, k, entry; 395 int err = 0; 396 struct scatterlist *sg; 397 struct c4iw_dev *rhp; 398 struct c4iw_pd *php; 399 struct c4iw_mr *mhp; 400 401 CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); 402 403 if (length == ~0ULL) 404 return ERR_PTR(-EINVAL); 405 406 if ((length + start) < start) 407 return ERR_PTR(-EINVAL); 408 409 php = to_c4iw_pd(pd); 410 rhp = php->rhp; 411 412 if (mr_exceeds_hw_limits(rhp, length)) 413 return ERR_PTR(-EINVAL); 414 415 mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 416 if (!mhp) 417 return ERR_PTR(-ENOMEM); 418 419 mhp->rhp = rhp; 420 421 mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0); 422 if (IS_ERR(mhp->umem)) { 423 err = PTR_ERR(mhp->umem); 424 kfree(mhp); 425 return ERR_PTR(err); 426 } 427 428 shift = ffs(mhp->umem->page_size) - 1; 429 430 n = mhp->umem->nmap; 431 err = alloc_pbl(mhp, n); 432 if (err) 433 goto err; 434 435 pages = (__be64 *) __get_free_page(GFP_KERNEL); 436 if (!pages) { 437 err = -ENOMEM; 438 goto err_pbl; 439 } 440 441 i = n = 0; 442 for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) { 443 len = sg_dma_len(sg) >> shift; 444 for (k = 0; k < len; ++k) { 445 pages[i++] = cpu_to_be64(sg_dma_address(sg) + 446 mhp->umem->page_size * k); 447 if (i == PAGE_SIZE / sizeof *pages) { 448 err = write_pbl(&mhp->rhp->rdev, 449 pages, 450 mhp->attr.pbl_addr + (n << 3), i); 451 if (err) 452 goto pbl_done; 453 n += i; 454 i = 0; 455 456 } 457 } 458 } 459 460 if (i) 461 err = write_pbl(&mhp->rhp->rdev, pages, 462 mhp->attr.pbl_addr + (n << 3), i); 463 464 pbl_done: 465 free_page((unsigned long) pages); 466 if (err) 467 goto err_pbl; 468 469 mhp->attr.pdid = php->pdid; 470 mhp->attr.zbva = 0; 471 mhp->attr.perms = c4iw_ib_to_tpt_access(acc); 472 mhp->attr.va_fbo = virt; 473 mhp->attr.page_size = shift - 12; 474 mhp->attr.len = length; 475 476 err = register_mem(rhp, php, mhp, shift); 477 if (err) 478 goto err_pbl; 479 480 return &mhp->ibmr; 481 482 err_pbl: 483 c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 484 mhp->attr.pbl_size << 3); 485 486 err: 487 ib_umem_release(mhp->umem); 488 kfree(mhp); 489 return ERR_PTR(err); 490 } 491 492 struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, 493 struct ib_udata *udata) 494 { 495 struct c4iw_dev *rhp; 496 struct c4iw_pd *php; 497 struct c4iw_mw *mhp; 498 u32 mmid; 499 u32 stag = 0; 500 int ret; 501 502 if (type != IB_MW_TYPE_1) 503 return ERR_PTR(-EINVAL); 504 505 php = to_c4iw_pd(pd); 506 rhp = php->rhp; 507 mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 508 if (!mhp) 509 return ERR_PTR(-ENOMEM); 510 ret = allocate_window(&rhp->rdev, &stag, php->pdid); 511 if (ret) { 512 kfree(mhp); 513 return ERR_PTR(ret); 514 } 515 mhp->rhp = rhp; 516 mhp->attr.pdid = php->pdid; 517 mhp->attr.type = FW_RI_STAG_MW; 518 mhp->attr.stag = stag; 519 mmid = (stag) >> 8; 520 mhp->ibmw.rkey = stag; 521 if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { 522 deallocate_window(&rhp->rdev, mhp->attr.stag); 523 kfree(mhp); 524 return ERR_PTR(-ENOMEM); 525 } 526 CTR4(KTR_IW_CXGBE, "%s mmid 0x%x mhp %p stag 0x%x", __func__, mmid, mhp, 527 stag); 528 return &(mhp->ibmw); 529 } 530 531 int c4iw_dealloc_mw(struct ib_mw *mw) 532 { 533 struct c4iw_dev *rhp; 534 struct c4iw_mw *mhp; 535 u32 mmid; 536 537 mhp = to_c4iw_mw(mw); 538 rhp = mhp->rhp; 539 mmid = (mw->rkey) >> 8; 540 remove_handle(rhp, &rhp->mmidr, mmid); 541 deallocate_window(&rhp->rdev, mhp->attr.stag); 542 kfree(mhp); 543 CTR4(KTR_IW_CXGBE, "%s ib_mw %p mmid 0x%x ptr %p", __func__, mw, mmid, 544 mhp); 545 return 0; 546 } 547 548 struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, 549 enum ib_mr_type mr_type, 550 u32 max_num_sg, struct ib_udata *udata) 551 { 552 struct c4iw_dev *rhp; 553 struct c4iw_pd *php; 554 struct c4iw_mr *mhp; 555 u32 mmid; 556 u32 stag = 0; 557 int ret = 0; 558 int length = roundup(max_num_sg * sizeof(u64), 32); 559 560 php = to_c4iw_pd(pd); 561 rhp = php->rhp; 562 563 if (__predict_false(c4iw_stopped(&rhp->rdev))) 564 return ERR_PTR(-EIO); 565 566 if (mr_type != IB_MR_TYPE_MEM_REG || 567 max_num_sg > t4_max_fr_depth(&rhp->rdev, use_dsgl)) 568 return ERR_PTR(-EINVAL); 569 570 mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); 571 if (!mhp) { 572 ret = -ENOMEM; 573 goto err; 574 } 575 576 mhp->mpl = dma_alloc_coherent(rhp->ibdev.dma_device, 577 length, &mhp->mpl_addr, GFP_KERNEL); 578 if (!mhp->mpl) { 579 ret = -ENOMEM; 580 goto err_mpl; 581 } 582 mhp->max_mpl_len = length; 583 584 mhp->rhp = rhp; 585 ret = alloc_pbl(mhp, max_num_sg); 586 if (ret) 587 goto err1; 588 mhp->attr.pbl_size = max_num_sg; 589 ret = allocate_stag(&rhp->rdev, &stag, php->pdid, 590 mhp->attr.pbl_size, mhp->attr.pbl_addr); 591 if (ret) 592 goto err2; 593 mhp->attr.pdid = php->pdid; 594 mhp->attr.type = FW_RI_STAG_NSMR; 595 mhp->attr.stag = stag; 596 mhp->attr.state = 0; 597 mmid = (stag) >> 8; 598 mhp->ibmr.rkey = mhp->ibmr.lkey = stag; 599 if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { 600 ret = -ENOMEM; 601 goto err3; 602 } 603 604 PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); 605 return &(mhp->ibmr); 606 err3: 607 dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size, 608 mhp->attr.pbl_addr); 609 err2: 610 c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 611 mhp->attr.pbl_size << 3); 612 err1: 613 dma_free_coherent(rhp->ibdev.dma_device, 614 mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr); 615 err_mpl: 616 kfree(mhp); 617 err: 618 return ERR_PTR(ret); 619 } 620 static int c4iw_set_page(struct ib_mr *ibmr, u64 addr) 621 { 622 struct c4iw_mr *mhp = to_c4iw_mr(ibmr); 623 624 if (unlikely(mhp->mpl_len == mhp->attr.pbl_size)) 625 return -ENOMEM; 626 627 mhp->mpl[mhp->mpl_len++] = addr; 628 629 return 0; 630 } 631 632 int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 633 int sg_nents, unsigned int *sg_offset) 634 { 635 struct c4iw_mr *mhp = to_c4iw_mr(ibmr); 636 637 mhp->mpl_len = 0; 638 639 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, c4iw_set_page); 640 } 641 642 643 int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 644 { 645 struct c4iw_dev *rhp; 646 struct c4iw_mr *mhp; 647 u32 mmid; 648 649 CTR2(KTR_IW_CXGBE, "%s ib_mr %p", __func__, ib_mr); 650 651 mhp = to_c4iw_mr(ib_mr); 652 rhp = mhp->rhp; 653 mmid = mhp->attr.stag >> 8; 654 remove_handle(rhp, &rhp->mmidr, mmid); 655 dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, 656 mhp->attr.pbl_addr); 657 if (mhp->attr.pbl_size) 658 c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, 659 mhp->attr.pbl_size << 3); 660 if (mhp->kva) 661 kfree((void *) (unsigned long) mhp->kva); 662 if (mhp->umem) 663 ib_umem_release(mhp->umem); 664 CTR3(KTR_IW_CXGBE, "%s mmid 0x%x ptr %p", __func__, mmid, mhp); 665 kfree(mhp); 666 return 0; 667 } 668 669 void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey) 670 { 671 struct c4iw_mr *mhp; 672 unsigned long flags; 673 674 spin_lock_irqsave(&rhp->lock, flags); 675 mhp = get_mhp(rhp, rkey >> 8); 676 if (mhp) 677 mhp->attr.state = 0; 678 spin_unlock_irqrestore(&rhp->lock, flags); 679 } 680 #endif 681