1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_inet.h" 38 39 #ifdef TCP_OFFLOAD 40 #include <sys/types.h> 41 #include <sys/malloc.h> 42 #include <sys/socket.h> 43 #include <sys/socketvar.h> 44 #include <sys/sockio.h> 45 #include <sys/taskqueue.h> 46 #include <netinet/in.h> 47 #include <net/route.h> 48 49 #include <netinet/in_systm.h> 50 #include <netinet/in_pcb.h> 51 #include <netinet/ip.h> 52 #include <netinet/ip_var.h> 53 #include <netinet/tcp_var.h> 54 #include <netinet/tcp.h> 55 #include <netinet/tcpip.h> 56 57 #include <netinet/toecore.h> 58 59 struct sge_iq; 60 struct rss_header; 61 struct cpl_set_tcb_rpl; 62 #include <linux/types.h> 63 #include "offload.h" 64 #include "tom/t4_tom.h" 65 66 #include "iw_cxgbe.h" 67 #include "user.h" 68 extern int use_dsgl; 69 static int creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize); 70 static int max_fr_immd = T4_MAX_FR_IMMD;//SYSCTL parameter later... 71 72 static int alloc_ird(struct c4iw_dev *dev, u32 ird) 73 { 74 int ret = 0; 75 76 spin_lock_irq(&dev->lock); 77 if (ird <= dev->avail_ird) 78 dev->avail_ird -= ird; 79 else 80 ret = -ENOMEM; 81 spin_unlock_irq(&dev->lock); 82 83 if (ret) 84 log(LOG_WARNING, "%s: device IRD resources exhausted\n", 85 device_get_nameunit(dev->rdev.adap->dev)); 86 87 return ret; 88 } 89 90 static void free_ird(struct c4iw_dev *dev, int ird) 91 { 92 spin_lock_irq(&dev->lock); 93 dev->avail_ird += ird; 94 spin_unlock_irq(&dev->lock); 95 } 96 97 static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state) 98 { 99 unsigned long flag; 100 spin_lock_irqsave(&qhp->lock, flag); 101 qhp->attr.state = state; 102 spin_unlock_irqrestore(&qhp->lock, flag); 103 } 104 105 static int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, 106 struct c4iw_dev_ucontext *uctx) 107 { 108 struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev); 109 /* 110 * uP clears EQ contexts when the connection exits rdma mode, 111 * so no need to post a RESET WR for these EQs. 112 */ 113 dma_free_coherent(rhp->ibdev.dma_device, 114 wq->rq.memsize, wq->rq.queue, 115 dma_unmap_addr(&wq->rq, mapping)); 116 dma_free_coherent(rhp->ibdev.dma_device, 117 wq->sq.memsize, wq->sq.queue, 118 dma_unmap_addr(&wq->sq, mapping)); 119 c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); 120 kfree(wq->rq.sw_rq); 121 kfree(wq->sq.sw_sq); 122 c4iw_put_qpid(rdev, wq->rq.qid, uctx); 123 c4iw_put_qpid(rdev, wq->sq.qid, uctx); 124 return 0; 125 } 126 127 static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, 128 struct t4_cq *rcq, struct t4_cq *scq, 129 struct c4iw_dev_ucontext *uctx) 130 { 131 struct adapter *sc = rdev->adap; 132 struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev); 133 int user = (uctx != &rdev->uctx); 134 struct fw_ri_res_wr *res_wr; 135 struct fw_ri_res *res; 136 int wr_len; 137 struct c4iw_wr_wait wr_wait; 138 int ret = 0; 139 int eqsize; 140 struct wrqe *wr; 141 u64 sq_bar2_qoffset = 0, rq_bar2_qoffset = 0; 142 143 wq->sq.qid = c4iw_get_qpid(rdev, uctx); 144 if (!wq->sq.qid) 145 return -ENOMEM; 146 147 wq->rq.qid = c4iw_get_qpid(rdev, uctx); 148 if (!wq->rq.qid) { 149 ret = -ENOMEM; 150 goto free_sq_qid; 151 } 152 153 if (!user) { 154 wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq, 155 GFP_KERNEL); 156 if (!wq->sq.sw_sq) { 157 ret = -ENOMEM; 158 goto free_rq_qid; 159 } 160 161 wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq, 162 GFP_KERNEL); 163 if (!wq->rq.sw_rq) { 164 ret = -ENOMEM; 165 goto free_sw_sq; 166 } 167 } 168 169 /* 170 * RQT must be a power of 2 and at least 16 deep. 171 */ 172 wq->rq.rqt_size = roundup_pow_of_two(max_t(u16, wq->rq.size, 16)); 173 wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size); 174 if (!wq->rq.rqt_hwaddr) { 175 ret = -ENOMEM; 176 goto free_sw_rq; 177 } 178 179 /*QP memory, allocate DMAable memory for Send & Receive Queues */ 180 wq->sq.queue = dma_alloc_coherent(rhp->ibdev.dma_device, wq->sq.memsize, 181 &(wq->sq.dma_addr), GFP_KERNEL); 182 if (!wq->sq.queue) { 183 ret = -ENOMEM; 184 goto free_hwaddr; 185 } 186 wq->sq.phys_addr = vtophys(wq->sq.queue); 187 dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr); 188 memset(wq->sq.queue, 0, wq->sq.memsize); 189 190 wq->rq.queue = dma_alloc_coherent(rhp->ibdev.dma_device, 191 wq->rq.memsize, &(wq->rq.dma_addr), GFP_KERNEL); 192 if (!wq->rq.queue) { 193 ret = -ENOMEM; 194 goto free_sq_dma; 195 } 196 wq->rq.phys_addr = vtophys(wq->rq.queue); 197 dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); 198 memset(wq->rq.queue, 0, wq->rq.memsize); 199 200 CTR5(KTR_IW_CXGBE, 201 "%s QP sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx", 202 __func__, 203 wq->sq.queue, (unsigned long long)wq->sq.phys_addr, 204 wq->rq.queue, (unsigned long long)wq->rq.phys_addr); 205 206 /* Doorbell/WC regions, determine the BAR2 queue offset and qid. */ 207 t4_bar2_sge_qregs(rdev->adap, wq->sq.qid, T4_BAR2_QTYPE_EGRESS, user, 208 &sq_bar2_qoffset, &wq->sq.bar2_qid); 209 t4_bar2_sge_qregs(rdev->adap, wq->rq.qid, T4_BAR2_QTYPE_EGRESS, user, 210 &rq_bar2_qoffset, &wq->rq.bar2_qid); 211 212 if (user) { 213 /* Compute BAR2 DB/WC physical address(page-aligned) for 214 * Userspace mapping. 215 */ 216 wq->sq.bar2_pa = (rdev->bar2_pa + sq_bar2_qoffset) & PAGE_MASK; 217 wq->rq.bar2_pa = (rdev->bar2_pa + rq_bar2_qoffset) & PAGE_MASK; 218 CTR3(KTR_IW_CXGBE, 219 "%s BAR2 DB/WC sq base pa 0x%llx rq base pa 0x%llx", 220 __func__, (unsigned long long)wq->sq.bar2_pa, 221 (unsigned long long)wq->rq.bar2_pa); 222 } else { 223 /* Compute BAR2 DB/WC virtual address to access in kernel. */ 224 wq->sq.bar2_va = (void __iomem *)((u64)rdev->bar2_kva + 225 sq_bar2_qoffset); 226 wq->rq.bar2_va = (void __iomem *)((u64)rdev->bar2_kva + 227 rq_bar2_qoffset); 228 CTR3(KTR_IW_CXGBE, "%s BAR2 DB/WC sq base va %p rq base va %p", 229 __func__, (unsigned long long)wq->sq.bar2_va, 230 (unsigned long long)wq->rq.bar2_va); 231 } 232 233 wq->rdev = rdev; 234 wq->rq.msn = 1; 235 236 /* build fw_ri_res_wr */ 237 wr_len = sizeof *res_wr + 2 * sizeof *res; 238 239 wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]); 240 if (wr == NULL) { 241 ret = -ENOMEM; 242 goto free_rq_dma; 243 } 244 res_wr = wrtod(wr); 245 246 memset(res_wr, 0, wr_len); 247 res_wr->op_nres = cpu_to_be32( 248 V_FW_WR_OP(FW_RI_RES_WR) | 249 V_FW_RI_RES_WR_NRES(2) | 250 F_FW_WR_COMPL); 251 res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); 252 res_wr->cookie = (unsigned long) &wr_wait; 253 res = res_wr->res; 254 res->u.sqrq.restype = FW_RI_RES_TYPE_SQ; 255 res->u.sqrq.op = FW_RI_RES_OP_WRITE; 256 257 /* eqsize is the number of 64B entries plus the status page size. */ 258 eqsize = wq->sq.size * T4_SQ_NUM_SLOTS + 259 rdev->hw_queue.t4_eq_status_entries; 260 261 res->u.sqrq.fetchszm_to_iqid = cpu_to_be32( 262 V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */ 263 V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */ 264 V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */ 265 V_FW_RI_RES_WR_IQID(scq->cqid)); 266 res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( 267 V_FW_RI_RES_WR_DCAEN(0) | 268 V_FW_RI_RES_WR_DCACPU(0) | 269 V_FW_RI_RES_WR_FBMIN(2) | 270 V_FW_RI_RES_WR_FBMAX(3) | 271 V_FW_RI_RES_WR_CIDXFTHRESHO(0) | 272 V_FW_RI_RES_WR_CIDXFTHRESH(0) | 273 V_FW_RI_RES_WR_EQSIZE(eqsize)); 274 res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid); 275 res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr); 276 res++; 277 res->u.sqrq.restype = FW_RI_RES_TYPE_RQ; 278 res->u.sqrq.op = FW_RI_RES_OP_WRITE; 279 280 /* eqsize is the number of 64B entries plus the status page size. */ 281 eqsize = wq->rq.size * T4_RQ_NUM_SLOTS + 282 rdev->hw_queue.t4_eq_status_entries; 283 res->u.sqrq.fetchszm_to_iqid = cpu_to_be32( 284 V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */ 285 V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */ 286 V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */ 287 V_FW_RI_RES_WR_IQID(rcq->cqid)); 288 res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( 289 V_FW_RI_RES_WR_DCAEN(0) | 290 V_FW_RI_RES_WR_DCACPU(0) | 291 V_FW_RI_RES_WR_FBMIN(2) | 292 V_FW_RI_RES_WR_FBMAX(3) | 293 V_FW_RI_RES_WR_CIDXFTHRESHO(0) | 294 V_FW_RI_RES_WR_CIDXFTHRESH(0) | 295 V_FW_RI_RES_WR_EQSIZE(eqsize)); 296 res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid); 297 res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr); 298 299 c4iw_init_wr_wait(&wr_wait); 300 301 t4_wrq_tx(sc, wr); 302 ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, 303 NULL, __func__); 304 if (ret) 305 goto free_rq_dma; 306 307 CTR5(KTR_IW_CXGBE, 308 "%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%llx rqudb 0x%llx", 309 __func__, wq->sq.qid, wq->rq.qid, 310 (unsigned long long)wq->sq.bar2_va, 311 (unsigned long long)wq->rq.bar2_va); 312 313 return 0; 314 free_rq_dma: 315 dma_free_coherent(rhp->ibdev.dma_device, 316 wq->rq.memsize, wq->rq.queue, 317 dma_unmap_addr(&wq->rq, mapping)); 318 free_sq_dma: 319 dma_free_coherent(rhp->ibdev.dma_device, 320 wq->sq.memsize, wq->sq.queue, 321 dma_unmap_addr(&wq->sq, mapping)); 322 free_hwaddr: 323 c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); 324 free_sw_rq: 325 kfree(wq->rq.sw_rq); 326 free_sw_sq: 327 kfree(wq->sq.sw_sq); 328 free_rq_qid: 329 c4iw_put_qpid(rdev, wq->rq.qid, uctx); 330 free_sq_qid: 331 c4iw_put_qpid(rdev, wq->sq.qid, uctx); 332 return ret; 333 } 334 335 static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp, 336 struct ib_send_wr *wr, int max, u32 *plenp) 337 { 338 u8 *dstp, *srcp; 339 u32 plen = 0; 340 int i; 341 int rem, len; 342 343 dstp = (u8 *)immdp->data; 344 for (i = 0; i < wr->num_sge; i++) { 345 if ((plen + wr->sg_list[i].length) > max) 346 return -EMSGSIZE; 347 srcp = (u8 *)(unsigned long)wr->sg_list[i].addr; 348 plen += wr->sg_list[i].length; 349 rem = wr->sg_list[i].length; 350 while (rem) { 351 if (dstp == (u8 *)&sq->queue[sq->size]) 352 dstp = (u8 *)sq->queue; 353 if (rem <= (u8 *)&sq->queue[sq->size] - dstp) 354 len = rem; 355 else 356 len = (u8 *)&sq->queue[sq->size] - dstp; 357 memcpy(dstp, srcp, len); 358 dstp += len; 359 srcp += len; 360 rem -= len; 361 } 362 } 363 len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp); 364 if (len) 365 memset(dstp, 0, len); 366 immdp->op = FW_RI_DATA_IMMD; 367 immdp->r1 = 0; 368 immdp->r2 = 0; 369 immdp->immdlen = cpu_to_be32(plen); 370 *plenp = plen; 371 return 0; 372 } 373 374 static int build_isgl(__be64 *queue_start, __be64 *queue_end, 375 struct fw_ri_isgl *isglp, struct ib_sge *sg_list, 376 int num_sge, u32 *plenp) 377 378 { 379 int i; 380 u32 plen = 0; 381 __be64 *flitp = (__be64 *)isglp->sge; 382 383 for (i = 0; i < num_sge; i++) { 384 if ((plen + sg_list[i].length) < plen) 385 return -EMSGSIZE; 386 plen += sg_list[i].length; 387 *flitp = cpu_to_be64(((u64)sg_list[i].lkey << 32) | 388 sg_list[i].length); 389 if (++flitp == queue_end) 390 flitp = queue_start; 391 *flitp = cpu_to_be64(sg_list[i].addr); 392 if (++flitp == queue_end) 393 flitp = queue_start; 394 } 395 *flitp = (__force __be64)0; 396 isglp->op = FW_RI_DATA_ISGL; 397 isglp->r1 = 0; 398 isglp->nsge = cpu_to_be16(num_sge); 399 isglp->r2 = 0; 400 if (plenp) 401 *plenp = plen; 402 return 0; 403 } 404 405 static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, 406 struct ib_send_wr *wr, u8 *len16) 407 { 408 u32 plen; 409 int size; 410 int ret; 411 412 if (wr->num_sge > T4_MAX_SEND_SGE) 413 return -EINVAL; 414 switch (wr->opcode) { 415 case IB_WR_SEND: 416 if (wr->send_flags & IB_SEND_SOLICITED) 417 wqe->send.sendop_pkd = cpu_to_be32( 418 V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE)); 419 else 420 wqe->send.sendop_pkd = cpu_to_be32( 421 V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND)); 422 wqe->send.stag_inv = 0; 423 break; 424 case IB_WR_SEND_WITH_INV: 425 if (wr->send_flags & IB_SEND_SOLICITED) 426 wqe->send.sendop_pkd = cpu_to_be32( 427 V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE_INV)); 428 else 429 wqe->send.sendop_pkd = cpu_to_be32( 430 V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_INV)); 431 wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); 432 break; 433 434 default: 435 return -EINVAL; 436 } 437 wqe->send.r3 = 0; 438 wqe->send.r4 = 0; 439 440 plen = 0; 441 if (wr->num_sge) { 442 if (wr->send_flags & IB_SEND_INLINE) { 443 ret = build_immd(sq, wqe->send.u.immd_src, wr, 444 T4_MAX_SEND_INLINE, &plen); 445 if (ret) 446 return ret; 447 size = sizeof wqe->send + sizeof(struct fw_ri_immd) + 448 plen; 449 } else { 450 ret = build_isgl((__be64 *)sq->queue, 451 (__be64 *)&sq->queue[sq->size], 452 wqe->send.u.isgl_src, 453 wr->sg_list, wr->num_sge, &plen); 454 if (ret) 455 return ret; 456 size = sizeof wqe->send + sizeof(struct fw_ri_isgl) + 457 wr->num_sge * sizeof(struct fw_ri_sge); 458 } 459 } else { 460 wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD; 461 wqe->send.u.immd_src[0].r1 = 0; 462 wqe->send.u.immd_src[0].r2 = 0; 463 wqe->send.u.immd_src[0].immdlen = 0; 464 size = sizeof wqe->send + sizeof(struct fw_ri_immd); 465 plen = 0; 466 } 467 *len16 = DIV_ROUND_UP(size, 16); 468 wqe->send.plen = cpu_to_be32(plen); 469 return 0; 470 } 471 472 static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, 473 struct ib_send_wr *wr, u8 *len16) 474 { 475 u32 plen; 476 int size; 477 int ret; 478 479 if (wr->num_sge > T4_MAX_SEND_SGE) 480 return -EINVAL; 481 wqe->write.immd_data = 0; 482 wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey); 483 wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr); 484 if (wr->num_sge) { 485 if (wr->send_flags & IB_SEND_INLINE) { 486 ret = build_immd(sq, wqe->write.u.immd_src, wr, 487 T4_MAX_WRITE_INLINE, &plen); 488 if (ret) 489 return ret; 490 size = sizeof wqe->write + sizeof(struct fw_ri_immd) + 491 plen; 492 } else { 493 ret = build_isgl((__be64 *)sq->queue, 494 (__be64 *)&sq->queue[sq->size], 495 wqe->write.u.isgl_src, 496 wr->sg_list, wr->num_sge, &plen); 497 if (ret) 498 return ret; 499 size = sizeof wqe->write + sizeof(struct fw_ri_isgl) + 500 wr->num_sge * sizeof(struct fw_ri_sge); 501 } 502 } else { 503 wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD; 504 wqe->write.u.immd_src[0].r1 = 0; 505 wqe->write.u.immd_src[0].r2 = 0; 506 wqe->write.u.immd_src[0].immdlen = 0; 507 size = sizeof wqe->write + sizeof(struct fw_ri_immd); 508 plen = 0; 509 } 510 *len16 = DIV_ROUND_UP(size, 16); 511 wqe->write.plen = cpu_to_be32(plen); 512 return 0; 513 } 514 515 static int build_rdma_read(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) 516 { 517 if (wr->num_sge > 1) 518 return -EINVAL; 519 if (wr->num_sge && wr->sg_list[0].length) { 520 wqe->read.stag_src = cpu_to_be32(rdma_wr(wr)->rkey); 521 wqe->read.to_src_hi = cpu_to_be32((u32)(rdma_wr(wr)->remote_addr 522 >> 32)); 523 wqe->read.to_src_lo = 524 cpu_to_be32((u32)rdma_wr(wr)->remote_addr); 525 wqe->read.stag_sink = cpu_to_be32(wr->sg_list[0].lkey); 526 wqe->read.plen = cpu_to_be32(wr->sg_list[0].length); 527 wqe->read.to_sink_hi = cpu_to_be32((u32)(wr->sg_list[0].addr 528 >> 32)); 529 wqe->read.to_sink_lo = cpu_to_be32((u32)(wr->sg_list[0].addr)); 530 } else { 531 wqe->read.stag_src = cpu_to_be32(2); 532 wqe->read.to_src_hi = 0; 533 wqe->read.to_src_lo = 0; 534 wqe->read.stag_sink = cpu_to_be32(2); 535 wqe->read.plen = 0; 536 wqe->read.to_sink_hi = 0; 537 wqe->read.to_sink_lo = 0; 538 } 539 wqe->read.r2 = 0; 540 wqe->read.r5 = 0; 541 *len16 = DIV_ROUND_UP(sizeof wqe->read, 16); 542 return 0; 543 } 544 545 static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, 546 struct ib_recv_wr *wr, u8 *len16) 547 { 548 int ret; 549 550 ret = build_isgl((__be64 *)qhp->wq.rq.queue, 551 (__be64 *)&qhp->wq.rq.queue[qhp->wq.rq.size], 552 &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); 553 if (ret) 554 return ret; 555 *len16 = DIV_ROUND_UP(sizeof wqe->recv + 556 wr->num_sge * sizeof(struct fw_ri_sge), 16); 557 return 0; 558 } 559 560 static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr, 561 u8 *len16) 562 { 563 wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); 564 wqe->inv.r2 = 0; 565 *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16); 566 return 0; 567 } 568 569 static void free_qp_work(struct work_struct *work) 570 { 571 struct c4iw_ucontext *ucontext; 572 struct c4iw_qp *qhp; 573 struct c4iw_dev *rhp; 574 575 qhp = container_of(work, struct c4iw_qp, free_work); 576 ucontext = qhp->ucontext; 577 rhp = qhp->rhp; 578 579 CTR3(KTR_IW_CXGBE, "%s qhp %p ucontext %p\n", __func__, 580 qhp, ucontext); 581 destroy_qp(&rhp->rdev, &qhp->wq, 582 ucontext ? &ucontext->uctx : &rhp->rdev.uctx); 583 584 if (ucontext) 585 c4iw_put_ucontext(ucontext); 586 kfree(qhp); 587 } 588 589 static void queue_qp_free(struct kref *kref) 590 { 591 struct c4iw_qp *qhp; 592 593 qhp = container_of(kref, struct c4iw_qp, kref); 594 CTR2(KTR_IW_CXGBE, "%s qhp %p", __func__, qhp); 595 queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work); 596 } 597 598 void c4iw_qp_add_ref(struct ib_qp *qp) 599 { 600 CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, qp); 601 kref_get(&to_c4iw_qp(qp)->kref); 602 } 603 604 void c4iw_qp_rem_ref(struct ib_qp *qp) 605 { 606 CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, qp); 607 kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free); 608 } 609 610 static void complete_sq_drain_wr(struct c4iw_qp *qhp, struct ib_send_wr *wr) 611 { 612 struct t4_cqe cqe = {}; 613 struct c4iw_cq *schp; 614 unsigned long flag; 615 struct t4_cq *cq; 616 617 schp = to_c4iw_cq(qhp->ibqp.send_cq); 618 cq = &schp->cq; 619 620 PDBG("%s drain sq id %u\n", __func__, qhp->wq.sq.qid); 621 cqe.u.drain_cookie = wr->wr_id; 622 cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | 623 V_CQE_OPCODE(C4IW_DRAIN_OPCODE) | 624 V_CQE_TYPE(1) | 625 V_CQE_SWCQE(1) | 626 V_CQE_QPID(qhp->wq.sq.qid)); 627 628 spin_lock_irqsave(&schp->lock, flag); 629 cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); 630 cq->sw_queue[cq->sw_pidx] = cqe; 631 t4_swcq_produce(cq); 632 spin_unlock_irqrestore(&schp->lock, flag); 633 634 spin_lock_irqsave(&schp->comp_handler_lock, flag); 635 (*schp->ibcq.comp_handler)(&schp->ibcq, 636 schp->ibcq.cq_context); 637 spin_unlock_irqrestore(&schp->comp_handler_lock, flag); 638 } 639 640 static void complete_rq_drain_wr(struct c4iw_qp *qhp, struct ib_recv_wr *wr) 641 { 642 struct t4_cqe cqe = {}; 643 struct c4iw_cq *rchp; 644 unsigned long flag; 645 struct t4_cq *cq; 646 647 rchp = to_c4iw_cq(qhp->ibqp.recv_cq); 648 cq = &rchp->cq; 649 650 PDBG("%s drain rq id %u\n", __func__, qhp->wq.sq.qid); 651 cqe.u.drain_cookie = wr->wr_id; 652 cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | 653 V_CQE_OPCODE(C4IW_DRAIN_OPCODE) | 654 V_CQE_TYPE(0) | 655 V_CQE_SWCQE(1) | 656 V_CQE_QPID(qhp->wq.sq.qid)); 657 658 spin_lock_irqsave(&rchp->lock, flag); 659 cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); 660 cq->sw_queue[cq->sw_pidx] = cqe; 661 t4_swcq_produce(cq); 662 spin_unlock_irqrestore(&rchp->lock, flag); 663 664 spin_lock_irqsave(&rchp->comp_handler_lock, flag); 665 (*rchp->ibcq.comp_handler)(&rchp->ibcq, 666 rchp->ibcq.cq_context); 667 spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); 668 } 669 670 static void build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr *fr, 671 struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16) 672 { 673 __be64 *p = (__be64 *)fr->pbl; 674 675 fr->r2 = cpu_to_be32(0); 676 fr->stag = cpu_to_be32(mhp->ibmr.rkey); 677 678 fr->tpte.valid_to_pdid = cpu_to_be32(F_FW_RI_TPTE_VALID | 679 V_FW_RI_TPTE_STAGKEY((mhp->ibmr.rkey & M_FW_RI_TPTE_STAGKEY)) | 680 V_FW_RI_TPTE_STAGSTATE(1) | 681 V_FW_RI_TPTE_STAGTYPE(FW_RI_STAG_NSMR) | 682 V_FW_RI_TPTE_PDID(mhp->attr.pdid)); 683 fr->tpte.locread_to_qpid = cpu_to_be32( 684 V_FW_RI_TPTE_PERM(c4iw_ib_to_tpt_access(wr->access)) | 685 V_FW_RI_TPTE_ADDRTYPE(FW_RI_VA_BASED_TO) | 686 V_FW_RI_TPTE_PS(ilog2(wr->mr->page_size) - 12)); 687 fr->tpte.nosnoop_pbladdr = cpu_to_be32(V_FW_RI_TPTE_PBLADDR( 688 PBL_OFF(&mhp->rhp->rdev, mhp->attr.pbl_addr)>>3)); 689 fr->tpte.dca_mwbcnt_pstag = cpu_to_be32(0); 690 fr->tpte.len_hi = cpu_to_be32(0); 691 fr->tpte.len_lo = cpu_to_be32(mhp->ibmr.length); 692 fr->tpte.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32); 693 fr->tpte.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 0xffffffff); 694 695 p[0] = cpu_to_be64((u64)mhp->mpl[0]); 696 p[1] = cpu_to_be64((u64)mhp->mpl[1]); 697 698 *len16 = DIV_ROUND_UP(sizeof(*fr), 16); 699 } 700 701 static int build_memreg(struct t4_sq *sq, union t4_wr *wqe, 702 struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16, 703 bool dsgl_supported) 704 { 705 struct fw_ri_immd *imdp; 706 __be64 *p; 707 int i; 708 int pbllen = roundup(mhp->mpl_len * sizeof(u64), 32); 709 int rem; 710 711 if (mhp->mpl_len > t4_max_fr_depth(use_dsgl && dsgl_supported)) 712 return -EINVAL; 713 714 wqe->fr.qpbinde_to_dcacpu = 0; 715 wqe->fr.pgsz_shift = ilog2(wr->mr->page_size) - 12; 716 wqe->fr.addr_type = FW_RI_VA_BASED_TO; 717 wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->access); 718 wqe->fr.len_hi = 0; 719 wqe->fr.len_lo = cpu_to_be32(mhp->ibmr.length); 720 wqe->fr.stag = cpu_to_be32(wr->key); 721 wqe->fr.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32); 722 wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 723 0xffffffff); 724 725 if (dsgl_supported && use_dsgl && (pbllen > max_fr_immd)) { 726 struct fw_ri_dsgl *sglp; 727 728 for (i = 0; i < mhp->mpl_len; i++) 729 mhp->mpl[i] = 730 (__force u64)cpu_to_be64((u64)mhp->mpl[i]); 731 732 sglp = (struct fw_ri_dsgl *)(&wqe->fr + 1); 733 sglp->op = FW_RI_DATA_DSGL; 734 sglp->r1 = 0; 735 sglp->nsge = cpu_to_be16(1); 736 sglp->addr0 = cpu_to_be64(mhp->mpl_addr); 737 sglp->len0 = cpu_to_be32(pbllen); 738 739 *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*sglp), 16); 740 } else { 741 imdp = (struct fw_ri_immd *)(&wqe->fr + 1); 742 imdp->op = FW_RI_DATA_IMMD; 743 imdp->r1 = 0; 744 imdp->r2 = 0; 745 imdp->immdlen = cpu_to_be32(pbllen); 746 p = (__be64 *)(imdp + 1); 747 rem = pbllen; 748 for (i = 0; i < mhp->mpl_len; i++) { 749 *p = cpu_to_be64((u64)mhp->mpl[i]); 750 rem -= sizeof(*p); 751 if (++p == (__be64 *)&sq->queue[sq->size]) 752 p = (__be64 *)sq->queue; 753 } 754 BUG_ON(rem < 0); 755 while (rem) { 756 *p = 0; 757 rem -= sizeof(*p); 758 if (++p == (__be64 *)&sq->queue[sq->size]) 759 p = (__be64 *)sq->queue; 760 } 761 *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*imdp) 762 + pbllen, 16); 763 } 764 765 return 0; 766 } 767 768 int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 769 struct ib_send_wr **bad_wr) 770 { 771 int err = 0; 772 u8 len16 = 0; 773 enum fw_wr_opcodes fw_opcode = 0; 774 enum fw_ri_wr_flags fw_flags; 775 struct c4iw_qp *qhp; 776 union t4_wr *wqe = NULL; 777 u32 num_wrs; 778 struct t4_swsqe *swsqe; 779 unsigned long flag; 780 u16 idx = 0; 781 struct c4iw_rdev *rdev; 782 783 qhp = to_c4iw_qp(ibqp); 784 rdev = &qhp->rhp->rdev; 785 spin_lock_irqsave(&qhp->lock, flag); 786 if (t4_wq_in_error(&qhp->wq)) { 787 spin_unlock_irqrestore(&qhp->lock, flag); 788 complete_sq_drain_wr(qhp, wr); 789 return err; 790 } 791 num_wrs = t4_sq_avail(&qhp->wq); 792 if (num_wrs == 0) { 793 spin_unlock_irqrestore(&qhp->lock, flag); 794 *bad_wr = wr; 795 return -ENOMEM; 796 } 797 while (wr) { 798 if (num_wrs == 0) { 799 err = -ENOMEM; 800 *bad_wr = wr; 801 break; 802 } 803 wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue + 804 qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE); 805 806 fw_flags = 0; 807 if (wr->send_flags & IB_SEND_SOLICITED) 808 fw_flags |= FW_RI_SOLICITED_EVENT_FLAG; 809 if (wr->send_flags & IB_SEND_SIGNALED || qhp->sq_sig_all) 810 fw_flags |= FW_RI_COMPLETION_FLAG; 811 swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx]; 812 switch (wr->opcode) { 813 case IB_WR_SEND_WITH_INV: 814 case IB_WR_SEND: 815 if (wr->send_flags & IB_SEND_FENCE) 816 fw_flags |= FW_RI_READ_FENCE_FLAG; 817 fw_opcode = FW_RI_SEND_WR; 818 if (wr->opcode == IB_WR_SEND) 819 swsqe->opcode = FW_RI_SEND; 820 else 821 swsqe->opcode = FW_RI_SEND_WITH_INV; 822 err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16); 823 break; 824 case IB_WR_RDMA_WRITE: 825 fw_opcode = FW_RI_RDMA_WRITE_WR; 826 swsqe->opcode = FW_RI_RDMA_WRITE; 827 err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16); 828 break; 829 case IB_WR_RDMA_READ: 830 case IB_WR_RDMA_READ_WITH_INV: 831 fw_opcode = FW_RI_RDMA_READ_WR; 832 swsqe->opcode = FW_RI_READ_REQ; 833 if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) { 834 c4iw_invalidate_mr(qhp->rhp, 835 wr->sg_list[0].lkey); 836 fw_flags = FW_RI_RDMA_READ_INVALIDATE; 837 } else { 838 fw_flags = 0; 839 } 840 err = build_rdma_read(wqe, wr, &len16); 841 if (err) 842 break; 843 swsqe->read_len = wr->sg_list[0].length; 844 if (!qhp->wq.sq.oldest_read) 845 qhp->wq.sq.oldest_read = swsqe; 846 break; 847 case IB_WR_REG_MR: { 848 struct c4iw_mr *mhp = to_c4iw_mr(reg_wr(wr)->mr); 849 850 swsqe->opcode = FW_RI_FAST_REGISTER; 851 if (rdev->adap->params.fr_nsmr_tpte_wr_support && 852 !mhp->attr.state && mhp->mpl_len <= 2) { 853 fw_opcode = FW_RI_FR_NSMR_TPTE_WR; 854 build_tpte_memreg(&wqe->fr_tpte, reg_wr(wr), 855 mhp, &len16); 856 } else { 857 fw_opcode = FW_RI_FR_NSMR_WR; 858 err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr), 859 mhp, &len16, 860 rdev->adap->params.ulptx_memwrite_dsgl); 861 if (err) 862 break; 863 } 864 mhp->attr.state = 1; 865 break; 866 } 867 case IB_WR_LOCAL_INV: 868 if (wr->send_flags & IB_SEND_FENCE) 869 fw_flags |= FW_RI_LOCAL_FENCE_FLAG; 870 fw_opcode = FW_RI_INV_LSTAG_WR; 871 swsqe->opcode = FW_RI_LOCAL_INV; 872 err = build_inv_stag(wqe, wr, &len16); 873 c4iw_invalidate_mr(qhp->rhp, wr->ex.invalidate_rkey); 874 break; 875 default: 876 CTR2(KTR_IW_CXGBE, "%s post of type =%d TBD!", __func__, 877 wr->opcode); 878 err = -EINVAL; 879 } 880 if (err) { 881 *bad_wr = wr; 882 break; 883 } 884 swsqe->idx = qhp->wq.sq.pidx; 885 swsqe->complete = 0; 886 swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED) || 887 qhp->sq_sig_all; 888 swsqe->flushed = 0; 889 swsqe->wr_id = wr->wr_id; 890 891 init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16); 892 893 CTR5(KTR_IW_CXGBE, 894 "%s cookie 0x%llx pidx 0x%x opcode 0x%x read_len %u", 895 __func__, (unsigned long long)wr->wr_id, qhp->wq.sq.pidx, 896 swsqe->opcode, swsqe->read_len); 897 wr = wr->next; 898 num_wrs--; 899 t4_sq_produce(&qhp->wq, len16); 900 idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); 901 } 902 903 t4_ring_sq_db(&qhp->wq, idx, wqe, rdev->adap->iwt.wc_en); 904 spin_unlock_irqrestore(&qhp->lock, flag); 905 return err; 906 } 907 908 int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 909 struct ib_recv_wr **bad_wr) 910 { 911 int err = 0; 912 struct c4iw_qp *qhp; 913 union t4_recv_wr *wqe = NULL; 914 u32 num_wrs; 915 u8 len16 = 0; 916 unsigned long flag; 917 u16 idx = 0; 918 919 qhp = to_c4iw_qp(ibqp); 920 spin_lock_irqsave(&qhp->lock, flag); 921 if (t4_wq_in_error(&qhp->wq)) { 922 spin_unlock_irqrestore(&qhp->lock, flag); 923 complete_rq_drain_wr(qhp, wr); 924 return err; 925 } 926 num_wrs = t4_rq_avail(&qhp->wq); 927 if (num_wrs == 0) { 928 spin_unlock_irqrestore(&qhp->lock, flag); 929 *bad_wr = wr; 930 return -ENOMEM; 931 } 932 while (wr) { 933 if (wr->num_sge > T4_MAX_RECV_SGE) { 934 err = -EINVAL; 935 *bad_wr = wr; 936 break; 937 } 938 wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue + 939 qhp->wq.rq.wq_pidx * 940 T4_EQ_ENTRY_SIZE); 941 if (num_wrs) 942 err = build_rdma_recv(qhp, wqe, wr, &len16); 943 else 944 err = -ENOMEM; 945 if (err) { 946 *bad_wr = wr; 947 break; 948 } 949 950 qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].wr_id = wr->wr_id; 951 952 wqe->recv.opcode = FW_RI_RECV_WR; 953 wqe->recv.r1 = 0; 954 wqe->recv.wrid = qhp->wq.rq.pidx; 955 wqe->recv.r2[0] = 0; 956 wqe->recv.r2[1] = 0; 957 wqe->recv.r2[2] = 0; 958 wqe->recv.len16 = len16; 959 CTR3(KTR_IW_CXGBE, "%s cookie 0x%llx pidx %u", __func__, 960 (unsigned long long) wr->wr_id, qhp->wq.rq.pidx); 961 t4_rq_produce(&qhp->wq, len16); 962 idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); 963 wr = wr->next; 964 num_wrs--; 965 } 966 967 t4_ring_rq_db(&qhp->wq, idx, wqe, qhp->rhp->rdev.adap->iwt.wc_en); 968 spin_unlock_irqrestore(&qhp->lock, flag); 969 return err; 970 } 971 972 static inline void build_term_codes(struct t4_cqe *err_cqe, u8 *layer_type, 973 u8 *ecode) 974 { 975 int status; 976 int tagged; 977 int opcode; 978 int rqtype; 979 int send_inv; 980 981 if (!err_cqe) { 982 *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 983 *ecode = 0; 984 return; 985 } 986 987 status = CQE_STATUS(err_cqe); 988 opcode = CQE_OPCODE(err_cqe); 989 rqtype = RQ_TYPE(err_cqe); 990 send_inv = (opcode == FW_RI_SEND_WITH_INV) || 991 (opcode == FW_RI_SEND_WITH_SE_INV); 992 tagged = (opcode == FW_RI_RDMA_WRITE) || 993 (rqtype && (opcode == FW_RI_READ_RESP)); 994 995 switch (status) { 996 case T4_ERR_STAG: 997 if (send_inv) { 998 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 999 *ecode = RDMAP_CANT_INV_STAG; 1000 } else { 1001 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 1002 *ecode = RDMAP_INV_STAG; 1003 } 1004 break; 1005 case T4_ERR_PDID: 1006 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 1007 if ((opcode == FW_RI_SEND_WITH_INV) || 1008 (opcode == FW_RI_SEND_WITH_SE_INV)) 1009 *ecode = RDMAP_CANT_INV_STAG; 1010 else 1011 *ecode = RDMAP_STAG_NOT_ASSOC; 1012 break; 1013 case T4_ERR_QPID: 1014 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 1015 *ecode = RDMAP_STAG_NOT_ASSOC; 1016 break; 1017 case T4_ERR_ACCESS: 1018 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 1019 *ecode = RDMAP_ACC_VIOL; 1020 break; 1021 case T4_ERR_WRAP: 1022 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 1023 *ecode = RDMAP_TO_WRAP; 1024 break; 1025 case T4_ERR_BOUND: 1026 if (tagged) { 1027 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 1028 *ecode = DDPT_BASE_BOUNDS; 1029 } else { 1030 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 1031 *ecode = RDMAP_BASE_BOUNDS; 1032 } 1033 break; 1034 case T4_ERR_INVALIDATE_SHARED_MR: 1035 case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND: 1036 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 1037 *ecode = RDMAP_CANT_INV_STAG; 1038 break; 1039 case T4_ERR_ECC: 1040 case T4_ERR_ECC_PSTAG: 1041 case T4_ERR_INTERNAL_ERR: 1042 *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; 1043 *ecode = 0; 1044 break; 1045 case T4_ERR_OUT_OF_RQE: 1046 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 1047 *ecode = DDPU_INV_MSN_NOBUF; 1048 break; 1049 case T4_ERR_PBL_ADDR_BOUND: 1050 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 1051 *ecode = DDPT_BASE_BOUNDS; 1052 break; 1053 case T4_ERR_CRC: 1054 *layer_type = LAYER_MPA|DDP_LLP; 1055 *ecode = MPA_CRC_ERR; 1056 break; 1057 case T4_ERR_MARKER: 1058 *layer_type = LAYER_MPA|DDP_LLP; 1059 *ecode = MPA_MARKER_ERR; 1060 break; 1061 case T4_ERR_PDU_LEN_ERR: 1062 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 1063 *ecode = DDPU_MSG_TOOBIG; 1064 break; 1065 case T4_ERR_DDP_VERSION: 1066 if (tagged) { 1067 *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 1068 *ecode = DDPT_INV_VERS; 1069 } else { 1070 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 1071 *ecode = DDPU_INV_VERS; 1072 } 1073 break; 1074 case T4_ERR_RDMA_VERSION: 1075 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 1076 *ecode = RDMAP_INV_VERS; 1077 break; 1078 case T4_ERR_OPCODE: 1079 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 1080 *ecode = RDMAP_INV_OPCODE; 1081 break; 1082 case T4_ERR_DDP_QUEUE_NUM: 1083 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 1084 *ecode = DDPU_INV_QN; 1085 break; 1086 case T4_ERR_MSN: 1087 case T4_ERR_MSN_GAP: 1088 case T4_ERR_MSN_RANGE: 1089 case T4_ERR_IRD_OVERFLOW: 1090 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 1091 *ecode = DDPU_INV_MSN_RANGE; 1092 break; 1093 case T4_ERR_TBIT: 1094 *layer_type = LAYER_DDP|DDP_LOCAL_CATA; 1095 *ecode = 0; 1096 break; 1097 case T4_ERR_MO: 1098 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 1099 *ecode = DDPU_INV_MO; 1100 break; 1101 default: 1102 *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 1103 *ecode = 0; 1104 break; 1105 } 1106 } 1107 1108 static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe, 1109 gfp_t gfp) 1110 { 1111 int ret; 1112 struct fw_ri_wr *wqe; 1113 struct terminate_message *term; 1114 struct wrqe *wr; 1115 struct socket *so = qhp->ep->com.so; 1116 struct inpcb *inp = sotoinpcb(so); 1117 struct tcpcb *tp = intotcpcb(inp); 1118 struct toepcb *toep = tp->t_toe; 1119 1120 CTR4(KTR_IW_CXGBE, "%s qhp %p qid 0x%x tid %u", __func__, qhp, 1121 qhp->wq.sq.qid, qhp->ep->hwtid); 1122 1123 wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); 1124 if (wr == NULL) 1125 return; 1126 wqe = wrtod(wr); 1127 1128 memset(wqe, 0, sizeof *wqe); 1129 wqe->op_compl = cpu_to_be32(V_FW_WR_OP(FW_RI_WR)); 1130 wqe->flowid_len16 = cpu_to_be32( 1131 V_FW_WR_FLOWID(qhp->ep->hwtid) | 1132 V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); 1133 1134 wqe->u.terminate.type = FW_RI_TYPE_TERMINATE; 1135 wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term); 1136 term = (struct terminate_message *)wqe->u.terminate.termmsg; 1137 if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) { 1138 term->layer_etype = qhp->attr.layer_etype; 1139 term->ecode = qhp->attr.ecode; 1140 } else 1141 build_term_codes(err_cqe, &term->layer_etype, &term->ecode); 1142 ret = creds(toep, inp, sizeof(*wqe)); 1143 if (ret) { 1144 free_wrqe(wr); 1145 return; 1146 } 1147 t4_wrq_tx(qhp->rhp->rdev.adap, wr); 1148 } 1149 1150 /* Assumes qhp lock is held. */ 1151 static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp, 1152 struct c4iw_cq *schp) 1153 { 1154 int count; 1155 int rq_flushed, sq_flushed; 1156 unsigned long flag; 1157 1158 CTR4(KTR_IW_CXGBE, "%s qhp %p rchp %p schp %p", __func__, qhp, rchp, 1159 schp); 1160 1161 /* locking hierarchy: cq lock first, then qp lock. */ 1162 spin_lock_irqsave(&rchp->lock, flag); 1163 spin_lock(&qhp->lock); 1164 1165 if (qhp->wq.flushed) { 1166 spin_unlock(&qhp->lock); 1167 spin_unlock_irqrestore(&rchp->lock, flag); 1168 return; 1169 } 1170 qhp->wq.flushed = 1; 1171 1172 c4iw_flush_hw_cq(rchp); 1173 c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count); 1174 rq_flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count); 1175 spin_unlock(&qhp->lock); 1176 spin_unlock_irqrestore(&rchp->lock, flag); 1177 1178 /* locking hierarchy: cq lock first, then qp lock. */ 1179 spin_lock_irqsave(&schp->lock, flag); 1180 spin_lock(&qhp->lock); 1181 if (schp != rchp) 1182 c4iw_flush_hw_cq(schp); 1183 sq_flushed = c4iw_flush_sq(qhp); 1184 spin_unlock(&qhp->lock); 1185 spin_unlock_irqrestore(&schp->lock, flag); 1186 1187 if (schp == rchp) { 1188 if (t4_clear_cq_armed(&rchp->cq) && 1189 (rq_flushed || sq_flushed)) { 1190 spin_lock_irqsave(&rchp->comp_handler_lock, flag); 1191 (*rchp->ibcq.comp_handler)(&rchp->ibcq, 1192 rchp->ibcq.cq_context); 1193 spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); 1194 } 1195 } else { 1196 if (t4_clear_cq_armed(&rchp->cq) && rq_flushed) { 1197 spin_lock_irqsave(&rchp->comp_handler_lock, flag); 1198 (*rchp->ibcq.comp_handler)(&rchp->ibcq, 1199 rchp->ibcq.cq_context); 1200 spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); 1201 } 1202 if (t4_clear_cq_armed(&schp->cq) && sq_flushed) { 1203 spin_lock_irqsave(&schp->comp_handler_lock, flag); 1204 (*schp->ibcq.comp_handler)(&schp->ibcq, 1205 schp->ibcq.cq_context); 1206 spin_unlock_irqrestore(&schp->comp_handler_lock, flag); 1207 } 1208 } 1209 } 1210 1211 static void flush_qp(struct c4iw_qp *qhp) 1212 { 1213 struct c4iw_cq *rchp, *schp; 1214 unsigned long flag; 1215 1216 rchp = to_c4iw_cq(qhp->ibqp.recv_cq); 1217 schp = to_c4iw_cq(qhp->ibqp.send_cq); 1218 1219 t4_set_wq_in_error(&qhp->wq); 1220 if (qhp->ibqp.uobject) { 1221 t4_set_cq_in_error(&rchp->cq); 1222 spin_lock_irqsave(&rchp->comp_handler_lock, flag); 1223 (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); 1224 spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); 1225 if (schp != rchp) { 1226 t4_set_cq_in_error(&schp->cq); 1227 spin_lock_irqsave(&schp->comp_handler_lock, flag); 1228 (*schp->ibcq.comp_handler)(&schp->ibcq, 1229 schp->ibcq.cq_context); 1230 spin_unlock_irqrestore(&schp->comp_handler_lock, flag); 1231 } 1232 return; 1233 } 1234 __flush_qp(qhp, rchp, schp); 1235 } 1236 1237 static int 1238 rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, struct c4iw_ep *ep) 1239 { 1240 struct c4iw_rdev *rdev = &rhp->rdev; 1241 struct adapter *sc = rdev->adap; 1242 struct fw_ri_wr *wqe; 1243 int ret; 1244 struct wrqe *wr; 1245 struct socket *so = ep->com.so; 1246 struct inpcb *inp = sotoinpcb(so); 1247 struct tcpcb *tp = intotcpcb(inp); 1248 struct toepcb *toep = tp->t_toe; 1249 1250 KASSERT(rhp == qhp->rhp && ep == qhp->ep, ("%s: EDOOFUS", __func__)); 1251 1252 CTR5(KTR_IW_CXGBE, "%s qhp %p qid 0x%x ep %p tid %u", __func__, qhp, 1253 qhp->wq.sq.qid, ep, ep->hwtid); 1254 1255 wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); 1256 if (wr == NULL) 1257 return (0); 1258 wqe = wrtod(wr); 1259 1260 memset(wqe, 0, sizeof *wqe); 1261 1262 wqe->op_compl = cpu_to_be32(V_FW_WR_OP(FW_RI_WR) | F_FW_WR_COMPL); 1263 wqe->flowid_len16 = cpu_to_be32(V_FW_WR_FLOWID(ep->hwtid) | 1264 V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); 1265 wqe->cookie = (unsigned long) &ep->com.wr_wait; 1266 wqe->u.fini.type = FW_RI_TYPE_FINI; 1267 1268 c4iw_init_wr_wait(&ep->com.wr_wait); 1269 1270 ret = creds(toep, inp, sizeof(*wqe)); 1271 if (ret) { 1272 free_wrqe(wr); 1273 return ret; 1274 } 1275 t4_wrq_tx(sc, wr); 1276 1277 ret = c4iw_wait_for_reply(rdev, &ep->com.wr_wait, ep->hwtid, 1278 qhp->wq.sq.qid, ep->com.so, __func__); 1279 return ret; 1280 } 1281 1282 static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init) 1283 { 1284 CTR2(KTR_IW_CXGBE, "%s p2p_type = %d", __func__, p2p_type); 1285 memset(&init->u, 0, sizeof init->u); 1286 switch (p2p_type) { 1287 case FW_RI_INIT_P2PTYPE_RDMA_WRITE: 1288 init->u.write.opcode = FW_RI_RDMA_WRITE_WR; 1289 init->u.write.stag_sink = cpu_to_be32(1); 1290 init->u.write.to_sink = cpu_to_be64(1); 1291 init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD; 1292 init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write + 1293 sizeof(struct fw_ri_immd), 1294 16); 1295 break; 1296 case FW_RI_INIT_P2PTYPE_READ_REQ: 1297 init->u.write.opcode = FW_RI_RDMA_READ_WR; 1298 init->u.read.stag_src = cpu_to_be32(1); 1299 init->u.read.to_src_lo = cpu_to_be32(1); 1300 init->u.read.stag_sink = cpu_to_be32(1); 1301 init->u.read.to_sink_lo = cpu_to_be32(1); 1302 init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16); 1303 break; 1304 } 1305 } 1306 1307 static int 1308 creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize) 1309 { 1310 struct ofld_tx_sdesc *txsd; 1311 1312 CTR3(KTR_IW_CXGBE, "%s:creB %p %u", __func__, toep , wrsize); 1313 INP_WLOCK(inp); 1314 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) { 1315 INP_WUNLOCK(inp); 1316 return (EINVAL); 1317 } 1318 txsd = &toep->txsd[toep->txsd_pidx]; 1319 txsd->tx_credits = howmany(wrsize, 16); 1320 txsd->plen = 0; 1321 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 1322 ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 1323 toep->tx_credits -= txsd->tx_credits; 1324 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 1325 toep->txsd_pidx = 0; 1326 toep->txsd_avail--; 1327 INP_WUNLOCK(inp); 1328 CTR5(KTR_IW_CXGBE, "%s:creE %p %u %u %u", __func__, toep , 1329 txsd->tx_credits, toep->tx_credits, toep->txsd_pidx); 1330 return (0); 1331 } 1332 1333 static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp) 1334 { 1335 struct fw_ri_wr *wqe; 1336 int ret; 1337 struct wrqe *wr; 1338 struct c4iw_ep *ep = qhp->ep; 1339 struct c4iw_rdev *rdev = &qhp->rhp->rdev; 1340 struct adapter *sc = rdev->adap; 1341 struct socket *so = ep->com.so; 1342 struct inpcb *inp = sotoinpcb(so); 1343 struct tcpcb *tp = intotcpcb(inp); 1344 struct toepcb *toep = tp->t_toe; 1345 1346 CTR5(KTR_IW_CXGBE, "%s qhp %p qid 0x%x ep %p tid %u", __func__, qhp, 1347 qhp->wq.sq.qid, ep, ep->hwtid); 1348 1349 wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); 1350 if (wr == NULL) 1351 return (0); 1352 wqe = wrtod(wr); 1353 ret = alloc_ird(rhp, qhp->attr.max_ird); 1354 if (ret) { 1355 qhp->attr.max_ird = 0; 1356 free_wrqe(wr); 1357 return ret; 1358 } 1359 1360 memset(wqe, 0, sizeof *wqe); 1361 1362 wqe->op_compl = cpu_to_be32( 1363 V_FW_WR_OP(FW_RI_WR) | 1364 F_FW_WR_COMPL); 1365 wqe->flowid_len16 = cpu_to_be32(V_FW_WR_FLOWID(ep->hwtid) | 1366 V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); 1367 1368 wqe->cookie = (unsigned long) &ep->com.wr_wait; 1369 1370 wqe->u.init.type = FW_RI_TYPE_INIT; 1371 wqe->u.init.mpareqbit_p2ptype = 1372 V_FW_RI_WR_MPAREQBIT(qhp->attr.mpa_attr.initiator) | 1373 V_FW_RI_WR_P2PTYPE(qhp->attr.mpa_attr.p2p_type); 1374 wqe->u.init.mpa_attrs = FW_RI_MPA_IETF_ENABLE; 1375 if (qhp->attr.mpa_attr.recv_marker_enabled) 1376 wqe->u.init.mpa_attrs |= FW_RI_MPA_RX_MARKER_ENABLE; 1377 if (qhp->attr.mpa_attr.xmit_marker_enabled) 1378 wqe->u.init.mpa_attrs |= FW_RI_MPA_TX_MARKER_ENABLE; 1379 if (qhp->attr.mpa_attr.crc_enabled) 1380 wqe->u.init.mpa_attrs |= FW_RI_MPA_CRC_ENABLE; 1381 1382 wqe->u.init.qp_caps = FW_RI_QP_RDMA_READ_ENABLE | 1383 FW_RI_QP_RDMA_WRITE_ENABLE | 1384 FW_RI_QP_BIND_ENABLE; 1385 if (!qhp->ibqp.uobject) 1386 wqe->u.init.qp_caps |= FW_RI_QP_FAST_REGISTER_ENABLE | 1387 FW_RI_QP_STAG0_ENABLE; 1388 wqe->u.init.nrqe = cpu_to_be16(t4_rqes_posted(&qhp->wq)); 1389 wqe->u.init.pdid = cpu_to_be32(qhp->attr.pd); 1390 wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid); 1391 wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid); 1392 wqe->u.init.rq_eqid = cpu_to_be32(qhp->wq.rq.qid); 1393 wqe->u.init.scqid = cpu_to_be32(qhp->attr.scq); 1394 wqe->u.init.rcqid = cpu_to_be32(qhp->attr.rcq); 1395 wqe->u.init.ord_max = cpu_to_be32(qhp->attr.max_ord); 1396 wqe->u.init.ird_max = cpu_to_be32(qhp->attr.max_ird); 1397 wqe->u.init.iss = cpu_to_be32(ep->snd_seq); 1398 wqe->u.init.irs = cpu_to_be32(ep->rcv_seq); 1399 wqe->u.init.hwrqsize = cpu_to_be32(qhp->wq.rq.rqt_size); 1400 wqe->u.init.hwrqaddr = cpu_to_be32(qhp->wq.rq.rqt_hwaddr - 1401 sc->vres.rq.start); 1402 if (qhp->attr.mpa_attr.initiator) 1403 build_rtr_msg(qhp->attr.mpa_attr.p2p_type, &wqe->u.init); 1404 1405 c4iw_init_wr_wait(&ep->com.wr_wait); 1406 1407 ret = creds(toep, inp, sizeof(*wqe)); 1408 if (ret) { 1409 free_wrqe(wr); 1410 free_ird(rhp, qhp->attr.max_ird); 1411 return ret; 1412 } 1413 t4_wrq_tx(sc, wr); 1414 1415 ret = c4iw_wait_for_reply(rdev, &ep->com.wr_wait, ep->hwtid, 1416 qhp->wq.sq.qid, ep->com.so, __func__); 1417 1418 toep->ulp_mode = ULP_MODE_RDMA; 1419 free_ird(rhp, qhp->attr.max_ird); 1420 1421 return ret; 1422 } 1423 1424 int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp, 1425 enum c4iw_qp_attr_mask mask, 1426 struct c4iw_qp_attributes *attrs, 1427 int internal) 1428 { 1429 int ret = 0; 1430 struct c4iw_qp_attributes newattr = qhp->attr; 1431 int disconnect = 0; 1432 int terminate = 0; 1433 int abort = 0; 1434 int free = 0; 1435 struct c4iw_ep *ep = NULL; 1436 1437 CTR5(KTR_IW_CXGBE, "%s qhp %p sqid 0x%x rqid 0x%x ep %p", __func__, qhp, 1438 qhp->wq.sq.qid, qhp->wq.rq.qid, qhp->ep); 1439 CTR3(KTR_IW_CXGBE, "%s state %d -> %d", __func__, qhp->attr.state, 1440 (mask & C4IW_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); 1441 1442 mutex_lock(&qhp->mutex); 1443 1444 /* Process attr changes if in IDLE */ 1445 if (mask & C4IW_QP_ATTR_VALID_MODIFY) { 1446 if (qhp->attr.state != C4IW_QP_STATE_IDLE) { 1447 ret = -EIO; 1448 goto out; 1449 } 1450 if (mask & C4IW_QP_ATTR_ENABLE_RDMA_READ) 1451 newattr.enable_rdma_read = attrs->enable_rdma_read; 1452 if (mask & C4IW_QP_ATTR_ENABLE_RDMA_WRITE) 1453 newattr.enable_rdma_write = attrs->enable_rdma_write; 1454 if (mask & C4IW_QP_ATTR_ENABLE_RDMA_BIND) 1455 newattr.enable_bind = attrs->enable_bind; 1456 if (mask & C4IW_QP_ATTR_MAX_ORD) { 1457 if (attrs->max_ord > c4iw_max_read_depth) { 1458 ret = -EINVAL; 1459 goto out; 1460 } 1461 newattr.max_ord = attrs->max_ord; 1462 } 1463 if (mask & C4IW_QP_ATTR_MAX_IRD) { 1464 if (attrs->max_ird > cur_max_read_depth(rhp)) { 1465 ret = -EINVAL; 1466 goto out; 1467 } 1468 newattr.max_ird = attrs->max_ird; 1469 } 1470 qhp->attr = newattr; 1471 } 1472 1473 if (!(mask & C4IW_QP_ATTR_NEXT_STATE)) 1474 goto out; 1475 if (qhp->attr.state == attrs->next_state) 1476 goto out; 1477 1478 switch (qhp->attr.state) { 1479 case C4IW_QP_STATE_IDLE: 1480 switch (attrs->next_state) { 1481 case C4IW_QP_STATE_RTS: 1482 if (!(mask & C4IW_QP_ATTR_LLP_STREAM_HANDLE)) { 1483 ret = -EINVAL; 1484 goto out; 1485 } 1486 if (!(mask & C4IW_QP_ATTR_MPA_ATTR)) { 1487 ret = -EINVAL; 1488 goto out; 1489 } 1490 qhp->attr.mpa_attr = attrs->mpa_attr; 1491 qhp->attr.llp_stream_handle = attrs->llp_stream_handle; 1492 qhp->ep = qhp->attr.llp_stream_handle; 1493 set_state(qhp, C4IW_QP_STATE_RTS); 1494 1495 /* 1496 * Ref the endpoint here and deref when we 1497 * disassociate the endpoint from the QP. This 1498 * happens in CLOSING->IDLE transition or *->ERROR 1499 * transition. 1500 */ 1501 c4iw_get_ep(&qhp->ep->com); 1502 ret = rdma_init(rhp, qhp); 1503 if (ret) 1504 goto err; 1505 break; 1506 case C4IW_QP_STATE_ERROR: 1507 set_state(qhp, C4IW_QP_STATE_ERROR); 1508 flush_qp(qhp); 1509 break; 1510 default: 1511 ret = -EINVAL; 1512 goto out; 1513 } 1514 break; 1515 case C4IW_QP_STATE_RTS: 1516 switch (attrs->next_state) { 1517 case C4IW_QP_STATE_CLOSING: 1518 BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); 1519 t4_set_wq_in_error(&qhp->wq); 1520 set_state(qhp, C4IW_QP_STATE_CLOSING); 1521 ep = qhp->ep; 1522 if (!internal) { 1523 abort = 0; 1524 disconnect = 1; 1525 c4iw_get_ep(&qhp->ep->com); 1526 } 1527 ret = rdma_fini(rhp, qhp, ep); 1528 if (ret) 1529 goto err; 1530 break; 1531 case C4IW_QP_STATE_TERMINATE: 1532 t4_set_wq_in_error(&qhp->wq); 1533 set_state(qhp, C4IW_QP_STATE_TERMINATE); 1534 qhp->attr.layer_etype = attrs->layer_etype; 1535 qhp->attr.ecode = attrs->ecode; 1536 ep = qhp->ep; 1537 if (!internal) { 1538 c4iw_get_ep(&qhp->ep->com); 1539 terminate = 1; 1540 disconnect = 1; 1541 } else { 1542 terminate = qhp->attr.send_term; 1543 ret = rdma_fini(rhp, qhp, ep); 1544 if (ret) 1545 goto err; 1546 } 1547 break; 1548 case C4IW_QP_STATE_ERROR: 1549 t4_set_wq_in_error(&qhp->wq); 1550 set_state(qhp, C4IW_QP_STATE_ERROR); 1551 if (!internal) { 1552 abort = 1; 1553 disconnect = 1; 1554 ep = qhp->ep; 1555 c4iw_get_ep(&qhp->ep->com); 1556 } 1557 goto err; 1558 break; 1559 default: 1560 ret = -EINVAL; 1561 goto out; 1562 } 1563 break; 1564 case C4IW_QP_STATE_CLOSING: 1565 1566 /* 1567 * Allow kernel users to move to ERROR for qp draining. 1568 */ 1569 if (!internal && (qhp->ibqp.uobject || attrs->next_state != 1570 C4IW_QP_STATE_ERROR)) { 1571 ret = -EINVAL; 1572 goto out; 1573 } 1574 switch (attrs->next_state) { 1575 case C4IW_QP_STATE_IDLE: 1576 flush_qp(qhp); 1577 set_state(qhp, C4IW_QP_STATE_IDLE); 1578 qhp->attr.llp_stream_handle = NULL; 1579 c4iw_put_ep(&qhp->ep->com); 1580 qhp->ep = NULL; 1581 wake_up(&qhp->wait); 1582 break; 1583 case C4IW_QP_STATE_ERROR: 1584 goto err; 1585 default: 1586 ret = -EINVAL; 1587 goto err; 1588 } 1589 break; 1590 case C4IW_QP_STATE_ERROR: 1591 if (attrs->next_state != C4IW_QP_STATE_IDLE) { 1592 ret = -EINVAL; 1593 goto out; 1594 } 1595 if (!t4_sq_empty(&qhp->wq) || !t4_rq_empty(&qhp->wq)) { 1596 ret = -EINVAL; 1597 goto out; 1598 } 1599 set_state(qhp, C4IW_QP_STATE_IDLE); 1600 break; 1601 case C4IW_QP_STATE_TERMINATE: 1602 if (!internal) { 1603 ret = -EINVAL; 1604 goto out; 1605 } 1606 goto err; 1607 break; 1608 default: 1609 printf("%s in a bad state %d\n", 1610 __func__, qhp->attr.state); 1611 ret = -EINVAL; 1612 goto err; 1613 break; 1614 } 1615 goto out; 1616 err: 1617 CTR3(KTR_IW_CXGBE, "%s disassociating ep %p qpid 0x%x", __func__, 1618 qhp->ep, qhp->wq.sq.qid); 1619 1620 /* disassociate the LLP connection */ 1621 qhp->attr.llp_stream_handle = NULL; 1622 if (!ep) 1623 ep = qhp->ep; 1624 qhp->ep = NULL; 1625 set_state(qhp, C4IW_QP_STATE_ERROR); 1626 free = 1; 1627 abort = 1; 1628 BUG_ON(!ep); 1629 flush_qp(qhp); 1630 wake_up(&qhp->wait); 1631 out: 1632 mutex_unlock(&qhp->mutex); 1633 1634 if (terminate) 1635 post_terminate(qhp, NULL, internal ? GFP_ATOMIC : GFP_KERNEL); 1636 1637 /* 1638 * If disconnect is 1, then we need to initiate a disconnect 1639 * on the EP. This can be a normal close (RTS->CLOSING) or 1640 * an abnormal close (RTS/CLOSING->ERROR). 1641 */ 1642 if (disconnect) { 1643 __c4iw_ep_disconnect(ep, abort, internal ? GFP_ATOMIC : 1644 GFP_KERNEL); 1645 c4iw_put_ep(&ep->com); 1646 } 1647 1648 /* 1649 * If free is 1, then we've disassociated the EP from the QP 1650 * and we need to dereference the EP. 1651 */ 1652 if (free) 1653 c4iw_put_ep(&ep->com); 1654 CTR2(KTR_IW_CXGBE, "%s exit state %d", __func__, qhp->attr.state); 1655 return ret; 1656 } 1657 1658 int c4iw_destroy_qp(struct ib_qp *ib_qp) 1659 { 1660 struct c4iw_dev *rhp; 1661 struct c4iw_qp *qhp; 1662 struct c4iw_qp_attributes attrs; 1663 1664 CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, ib_qp); 1665 qhp = to_c4iw_qp(ib_qp); 1666 rhp = qhp->rhp; 1667 1668 attrs.next_state = C4IW_QP_STATE_ERROR; 1669 if (qhp->attr.state == C4IW_QP_STATE_TERMINATE) 1670 c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1671 else 1672 c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1673 wait_event(qhp->wait, !qhp->ep); 1674 1675 remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid); 1676 1677 free_ird(rhp, qhp->attr.max_ird); 1678 c4iw_qp_rem_ref(ib_qp); 1679 1680 CTR3(KTR_IW_CXGBE, "%s ib_qp %p qpid 0x%0x", __func__, ib_qp, 1681 qhp->wq.sq.qid); 1682 return 0; 1683 } 1684 1685 struct ib_qp * 1686 c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, 1687 struct ib_udata *udata) 1688 { 1689 struct c4iw_dev *rhp; 1690 struct c4iw_qp *qhp; 1691 struct c4iw_pd *php; 1692 struct c4iw_cq *schp; 1693 struct c4iw_cq *rchp; 1694 struct c4iw_create_qp_resp uresp; 1695 unsigned int sqsize, rqsize; 1696 struct c4iw_ucontext *ucontext; 1697 int ret; 1698 struct c4iw_mm_entry *sq_key_mm = NULL, *rq_key_mm = NULL; 1699 struct c4iw_mm_entry *sq_db_key_mm = NULL, *rq_db_key_mm = NULL; 1700 1701 CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); 1702 1703 if (attrs->qp_type != IB_QPT_RC) 1704 return ERR_PTR(-EINVAL); 1705 1706 php = to_c4iw_pd(pd); 1707 rhp = php->rhp; 1708 schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid); 1709 rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid); 1710 if (!schp || !rchp) 1711 return ERR_PTR(-EINVAL); 1712 1713 if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE) 1714 return ERR_PTR(-EINVAL); 1715 1716 if (attrs->cap.max_recv_wr > rhp->rdev.hw_queue.t4_max_rq_size) 1717 return ERR_PTR(-E2BIG); 1718 rqsize = attrs->cap.max_recv_wr + 1; 1719 if (rqsize < 8) 1720 rqsize = 8; 1721 1722 if (attrs->cap.max_send_wr > rhp->rdev.hw_queue.t4_max_sq_size) 1723 return ERR_PTR(-E2BIG); 1724 sqsize = attrs->cap.max_send_wr + 1; 1725 if (sqsize < 8) 1726 sqsize = 8; 1727 1728 ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL; 1729 1730 qhp = kzalloc(sizeof(*qhp), GFP_KERNEL); 1731 if (!qhp) 1732 return ERR_PTR(-ENOMEM); 1733 qhp->wq.sq.size = sqsize; 1734 qhp->wq.sq.memsize = 1735 (sqsize + rhp->rdev.hw_queue.t4_eq_status_entries) * 1736 sizeof(*qhp->wq.sq.queue) + 16 * sizeof(__be64); 1737 qhp->wq.sq.flush_cidx = -1; 1738 qhp->wq.rq.size = rqsize; 1739 qhp->wq.rq.memsize = 1740 (rqsize + rhp->rdev.hw_queue.t4_eq_status_entries) * 1741 sizeof(*qhp->wq.rq.queue); 1742 1743 if (ucontext) { 1744 qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE); 1745 qhp->wq.rq.memsize = roundup(qhp->wq.rq.memsize, PAGE_SIZE); 1746 } 1747 1748 CTR5(KTR_IW_CXGBE, "%s sqsize %u sqmemsize %zu rqsize %u rqmemsize %zu", 1749 __func__, sqsize, qhp->wq.sq.memsize, rqsize, qhp->wq.rq.memsize); 1750 1751 ret = create_qp(&rhp->rdev, &qhp->wq, &schp->cq, &rchp->cq, 1752 ucontext ? &ucontext->uctx : &rhp->rdev.uctx); 1753 if (ret) 1754 goto err1; 1755 1756 attrs->cap.max_recv_wr = rqsize - 1; 1757 attrs->cap.max_send_wr = sqsize - 1; 1758 attrs->cap.max_inline_data = T4_MAX_SEND_INLINE; 1759 1760 qhp->rhp = rhp; 1761 qhp->attr.pd = php->pdid; 1762 qhp->attr.scq = ((struct c4iw_cq *) attrs->send_cq)->cq.cqid; 1763 qhp->attr.rcq = ((struct c4iw_cq *) attrs->recv_cq)->cq.cqid; 1764 qhp->attr.sq_num_entries = attrs->cap.max_send_wr; 1765 qhp->attr.rq_num_entries = attrs->cap.max_recv_wr; 1766 qhp->attr.sq_max_sges = attrs->cap.max_send_sge; 1767 qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge; 1768 qhp->attr.rq_max_sges = attrs->cap.max_recv_sge; 1769 qhp->attr.state = C4IW_QP_STATE_IDLE; 1770 qhp->attr.next_state = C4IW_QP_STATE_IDLE; 1771 qhp->attr.enable_rdma_read = 1; 1772 qhp->attr.enable_rdma_write = 1; 1773 qhp->attr.enable_bind = 1; 1774 qhp->attr.max_ord = 0; 1775 qhp->attr.max_ird = 0; 1776 qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; 1777 spin_lock_init(&qhp->lock); 1778 mutex_init(&qhp->mutex); 1779 init_waitqueue_head(&qhp->wait); 1780 kref_init(&qhp->kref); 1781 INIT_WORK(&qhp->free_work, free_qp_work); 1782 1783 ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid); 1784 if (ret) 1785 goto err2; 1786 1787 if (udata) { 1788 sq_key_mm = kmalloc(sizeof(*sq_key_mm), GFP_KERNEL); 1789 if (!sq_key_mm) { 1790 ret = -ENOMEM; 1791 goto err3; 1792 } 1793 rq_key_mm = kmalloc(sizeof(*rq_key_mm), GFP_KERNEL); 1794 if (!rq_key_mm) { 1795 ret = -ENOMEM; 1796 goto err4; 1797 } 1798 sq_db_key_mm = kmalloc(sizeof(*sq_db_key_mm), GFP_KERNEL); 1799 if (!sq_db_key_mm) { 1800 ret = -ENOMEM; 1801 goto err5; 1802 } 1803 rq_db_key_mm = kmalloc(sizeof(*rq_db_key_mm), GFP_KERNEL); 1804 if (!rq_db_key_mm) { 1805 ret = -ENOMEM; 1806 goto err6; 1807 } 1808 uresp.flags = 0; 1809 uresp.qid_mask = rhp->rdev.qpmask; 1810 uresp.sqid = qhp->wq.sq.qid; 1811 uresp.sq_size = qhp->wq.sq.size; 1812 uresp.sq_memsize = qhp->wq.sq.memsize; 1813 uresp.rqid = qhp->wq.rq.qid; 1814 uresp.rq_size = qhp->wq.rq.size; 1815 uresp.rq_memsize = qhp->wq.rq.memsize; 1816 spin_lock(&ucontext->mmap_lock); 1817 uresp.ma_sync_key = 0; 1818 uresp.sq_key = ucontext->key; 1819 ucontext->key += PAGE_SIZE; 1820 uresp.rq_key = ucontext->key; 1821 ucontext->key += PAGE_SIZE; 1822 uresp.sq_db_gts_key = ucontext->key; 1823 ucontext->key += PAGE_SIZE; 1824 uresp.rq_db_gts_key = ucontext->key; 1825 ucontext->key += PAGE_SIZE; 1826 spin_unlock(&ucontext->mmap_lock); 1827 ret = ib_copy_to_udata(udata, &uresp, sizeof uresp); 1828 if (ret) 1829 goto err7; 1830 sq_key_mm->key = uresp.sq_key; 1831 sq_key_mm->addr = qhp->wq.sq.phys_addr; 1832 sq_key_mm->len = PAGE_ALIGN(qhp->wq.sq.memsize); 1833 CTR4(KTR_IW_CXGBE, "%s sq_key_mm %x, %x, %d", __func__, 1834 sq_key_mm->key, sq_key_mm->addr, 1835 sq_key_mm->len); 1836 insert_mmap(ucontext, sq_key_mm); 1837 rq_key_mm->key = uresp.rq_key; 1838 rq_key_mm->addr = qhp->wq.rq.phys_addr; 1839 rq_key_mm->len = PAGE_ALIGN(qhp->wq.rq.memsize); 1840 CTR4(KTR_IW_CXGBE, "%s rq_key_mm %x, %x, %d", __func__, 1841 rq_key_mm->key, rq_key_mm->addr, 1842 rq_key_mm->len); 1843 insert_mmap(ucontext, rq_key_mm); 1844 sq_db_key_mm->key = uresp.sq_db_gts_key; 1845 sq_db_key_mm->addr = (u64)qhp->wq.sq.bar2_pa; 1846 sq_db_key_mm->len = PAGE_SIZE; 1847 CTR4(KTR_IW_CXGBE, "%s sq_db_key_mm %x, %x, %d", __func__, 1848 sq_db_key_mm->key, sq_db_key_mm->addr, 1849 sq_db_key_mm->len); 1850 insert_mmap(ucontext, sq_db_key_mm); 1851 rq_db_key_mm->key = uresp.rq_db_gts_key; 1852 rq_db_key_mm->addr = (u64)qhp->wq.rq.bar2_pa; 1853 rq_db_key_mm->len = PAGE_SIZE; 1854 CTR4(KTR_IW_CXGBE, "%s rq_db_key_mm %x, %x, %d", __func__, 1855 rq_db_key_mm->key, rq_db_key_mm->addr, 1856 rq_db_key_mm->len); 1857 insert_mmap(ucontext, rq_db_key_mm); 1858 1859 c4iw_get_ucontext(ucontext); 1860 qhp->ucontext = ucontext; 1861 } 1862 qhp->ibqp.qp_num = qhp->wq.sq.qid; 1863 init_timer(&(qhp->timer)); 1864 1865 CTR5(KTR_IW_CXGBE, "%s sq id %u size %u memsize %zu num_entries %u\n", 1866 __func__, qhp->wq.sq.qid, 1867 qhp->wq.sq.size, qhp->wq.sq.memsize, attrs->cap.max_send_wr); 1868 CTR5(KTR_IW_CXGBE, "%s rq id %u size %u memsize %zu num_entries %u\n", 1869 __func__, qhp->wq.rq.qid, 1870 qhp->wq.rq.size, qhp->wq.rq.memsize, attrs->cap.max_recv_wr); 1871 return &qhp->ibqp; 1872 err7: 1873 kfree(rq_db_key_mm); 1874 err6: 1875 kfree(sq_db_key_mm); 1876 err5: 1877 kfree(rq_key_mm); 1878 err4: 1879 kfree(sq_key_mm); 1880 err3: 1881 remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid); 1882 err2: 1883 destroy_qp(&rhp->rdev, &qhp->wq, 1884 ucontext ? &ucontext->uctx : &rhp->rdev.uctx); 1885 err1: 1886 kfree(qhp); 1887 return ERR_PTR(ret); 1888 } 1889 1890 int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1891 int attr_mask, struct ib_udata *udata) 1892 { 1893 struct c4iw_dev *rhp; 1894 struct c4iw_qp *qhp; 1895 enum c4iw_qp_attr_mask mask = 0; 1896 struct c4iw_qp_attributes attrs; 1897 1898 CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, ibqp); 1899 1900 /* iwarp does not support the RTR state */ 1901 if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR)) 1902 attr_mask &= ~IB_QP_STATE; 1903 1904 /* Make sure we still have something left to do */ 1905 if (!attr_mask) 1906 return 0; 1907 1908 memset(&attrs, 0, sizeof attrs); 1909 qhp = to_c4iw_qp(ibqp); 1910 rhp = qhp->rhp; 1911 1912 attrs.next_state = c4iw_convert_state(attr->qp_state); 1913 attrs.enable_rdma_read = (attr->qp_access_flags & 1914 IB_ACCESS_REMOTE_READ) ? 1 : 0; 1915 attrs.enable_rdma_write = (attr->qp_access_flags & 1916 IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1917 attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0; 1918 1919 1920 mask |= (attr_mask & IB_QP_STATE) ? C4IW_QP_ATTR_NEXT_STATE : 0; 1921 mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ? 1922 (C4IW_QP_ATTR_ENABLE_RDMA_READ | 1923 C4IW_QP_ATTR_ENABLE_RDMA_WRITE | 1924 C4IW_QP_ATTR_ENABLE_RDMA_BIND) : 0; 1925 1926 return c4iw_modify_qp(rhp, qhp, mask, &attrs, 0); 1927 } 1928 1929 struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn) 1930 { 1931 CTR3(KTR_IW_CXGBE, "%s ib_dev %p qpn 0x%x", __func__, dev, qpn); 1932 return (struct ib_qp *)get_qhp(to_c4iw_dev(dev), qpn); 1933 } 1934 1935 int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1936 int attr_mask, struct ib_qp_init_attr *init_attr) 1937 { 1938 struct c4iw_qp *qhp = to_c4iw_qp(ibqp); 1939 1940 memset(attr, 0, sizeof *attr); 1941 memset(init_attr, 0, sizeof *init_attr); 1942 attr->qp_state = to_ib_qp_state(qhp->attr.state); 1943 init_attr->cap.max_send_wr = qhp->attr.sq_num_entries; 1944 init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries; 1945 init_attr->cap.max_send_sge = qhp->attr.sq_max_sges; 1946 init_attr->cap.max_recv_sge = qhp->attr.sq_max_sges; 1947 init_attr->cap.max_inline_data = T4_MAX_SEND_INLINE; 1948 init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0; 1949 return 0; 1950 } 1951 #endif 1952