1 /* 2 * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_inet.h" 36 37 #ifdef TCP_OFFLOAD 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/bus.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/rwlock.h> 46 #include <sys/socket.h> 47 #include <sys/sbuf.h> 48 49 #include "iw_cxgbe.h" 50 #include "user.h" 51 52 static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, 53 struct c4iw_dev_ucontext *uctx) 54 { 55 struct adapter *sc = rdev->adap; 56 struct fw_ri_res_wr *res_wr; 57 struct fw_ri_res *res; 58 int wr_len; 59 struct c4iw_wr_wait wr_wait; 60 struct wrqe *wr; 61 62 wr_len = sizeof *res_wr + sizeof *res; 63 wr = alloc_wrqe(wr_len, &sc->sge.mgmtq); 64 if (wr == NULL) 65 return (0); 66 res_wr = wrtod(wr); 67 memset(res_wr, 0, wr_len); 68 res_wr->op_nres = cpu_to_be32( 69 V_FW_WR_OP(FW_RI_RES_WR) | 70 V_FW_RI_RES_WR_NRES(1) | 71 F_FW_WR_COMPL); 72 res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); 73 res_wr->cookie = (unsigned long) &wr_wait; 74 res = res_wr->res; 75 res->u.cq.restype = FW_RI_RES_TYPE_CQ; 76 res->u.cq.op = FW_RI_RES_OP_RESET; 77 res->u.cq.iqid = cpu_to_be32(cq->cqid); 78 79 c4iw_init_wr_wait(&wr_wait); 80 81 t4_wrq_tx(sc, wr); 82 83 c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); 84 85 kfree(cq->sw_queue); 86 contigfree(cq->queue, cq->memsize, M_DEVBUF); 87 c4iw_put_cqid(rdev, cq->cqid, uctx); 88 return 0; 89 } 90 91 static int 92 create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, 93 struct c4iw_dev_ucontext *uctx) 94 { 95 struct adapter *sc = rdev->adap; 96 struct fw_ri_res_wr *res_wr; 97 struct fw_ri_res *res; 98 int wr_len; 99 int user = (uctx != &rdev->uctx); 100 struct c4iw_wr_wait wr_wait; 101 int ret; 102 struct wrqe *wr; 103 104 cq->cqid = c4iw_get_cqid(rdev, uctx); 105 if (!cq->cqid) { 106 ret = -ENOMEM; 107 goto err1; 108 } 109 110 if (!user) { 111 cq->sw_queue = kzalloc(cq->memsize, GFP_KERNEL); 112 if (!cq->sw_queue) { 113 ret = -ENOMEM; 114 goto err2; 115 } 116 } 117 118 cq->queue = contigmalloc(cq->memsize, M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 119 PAGE_SIZE, 0); 120 if (cq->queue) 121 cq->dma_addr = vtophys(cq->queue); 122 else { 123 ret = -ENOMEM; 124 goto err3; 125 } 126 127 pci_unmap_addr_set(cq, mapping, cq->dma_addr); 128 memset(cq->queue, 0, cq->memsize); 129 130 /* build fw_ri_res_wr */ 131 wr_len = sizeof *res_wr + sizeof *res; 132 133 wr = alloc_wrqe(wr_len, &sc->sge.mgmtq); 134 if (wr == NULL) 135 return (0); 136 res_wr = wrtod(wr); 137 138 memset(res_wr, 0, wr_len); 139 res_wr->op_nres = cpu_to_be32( 140 V_FW_WR_OP(FW_RI_RES_WR) | 141 V_FW_RI_RES_WR_NRES(1) | 142 F_FW_WR_COMPL); 143 res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); 144 res_wr->cookie = (unsigned long) &wr_wait; 145 res = res_wr->res; 146 res->u.cq.restype = FW_RI_RES_TYPE_CQ; 147 res->u.cq.op = FW_RI_RES_OP_WRITE; 148 res->u.cq.iqid = cpu_to_be32(cq->cqid); 149 //Fixme: Always use first queue id for IQANDSTINDEX. Linux does the same. 150 res->u.cq.iqandst_to_iqandstindex = cpu_to_be32( 151 V_FW_RI_RES_WR_IQANUS(0) | 152 V_FW_RI_RES_WR_IQANUD(1) | 153 F_FW_RI_RES_WR_IQANDST | 154 V_FW_RI_RES_WR_IQANDSTINDEX(sc->sge.ofld_rxq[0].iq.abs_id)); 155 res->u.cq.iqdroprss_to_iqesize = cpu_to_be16( 156 F_FW_RI_RES_WR_IQDROPRSS | 157 V_FW_RI_RES_WR_IQPCIECH(2) | 158 V_FW_RI_RES_WR_IQINTCNTTHRESH(0) | 159 F_FW_RI_RES_WR_IQO | 160 V_FW_RI_RES_WR_IQESIZE(1)); 161 res->u.cq.iqsize = cpu_to_be16(cq->size); 162 res->u.cq.iqaddr = cpu_to_be64(cq->dma_addr); 163 164 c4iw_init_wr_wait(&wr_wait); 165 166 t4_wrq_tx(sc, wr); 167 168 CTR2(KTR_IW_CXGBE, "%s wait_event wr_wait %p", __func__, &wr_wait); 169 ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); 170 if (ret) 171 goto err4; 172 173 cq->gen = 1; 174 cq->gts = (void *)((unsigned long)rman_get_virtual(sc->regs_res) + 175 sc->sge_gts_reg); 176 cq->rdev = rdev; 177 178 if (user) { 179 cq->ugts = (u64)((char*)rman_get_virtual(sc->udbs_res) + 180 (cq->cqid << rdev->cqshift)); 181 cq->ugts &= PAGE_MASK; 182 CTR5(KTR_IW_CXGBE, 183 "%s: UGTS %p cqid %x cqshift %d page_mask %x", __func__, 184 cq->ugts, cq->cqid, rdev->cqshift, PAGE_MASK); 185 } 186 return 0; 187 err4: 188 contigfree(cq->queue, cq->memsize, M_DEVBUF); 189 err3: 190 kfree(cq->sw_queue); 191 err2: 192 c4iw_put_cqid(rdev, cq->cqid, uctx); 193 err1: 194 return ret; 195 } 196 197 static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq) 198 { 199 struct t4_cqe cqe; 200 201 CTR5(KTR_IW_CXGBE, "%s wq %p cq %p sw_cidx %u sw_pidx %u", __func__, wq, 202 cq, cq->sw_cidx, cq->sw_pidx); 203 memset(&cqe, 0, sizeof(cqe)); 204 cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | 205 V_CQE_OPCODE(FW_RI_SEND) | 206 V_CQE_TYPE(0) | 207 V_CQE_SWCQE(1) | 208 V_CQE_QPID(wq->sq.qid)); 209 cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); 210 cq->sw_queue[cq->sw_pidx] = cqe; 211 t4_swcq_produce(cq); 212 } 213 214 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count) 215 { 216 int flushed = 0; 217 int in_use = wq->rq.in_use - count; 218 219 BUG_ON(in_use < 0); 220 CTR5(KTR_IW_CXGBE, "%s wq %p cq %p rq.in_use %u skip count %u", 221 __func__, wq, cq, wq->rq.in_use, count); 222 while (in_use--) { 223 insert_recv_cqe(wq, cq); 224 flushed++; 225 } 226 return flushed; 227 } 228 229 static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq, 230 struct t4_swsqe *swcqe) 231 { 232 struct t4_cqe cqe; 233 234 CTR5(KTR_IW_CXGBE, "%s wq %p cq %p sw_cidx %u sw_pidx %u", __func__, wq, 235 cq, cq->sw_cidx, cq->sw_pidx); 236 memset(&cqe, 0, sizeof(cqe)); 237 cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) | 238 V_CQE_OPCODE(swcqe->opcode) | 239 V_CQE_TYPE(1) | 240 V_CQE_SWCQE(1) | 241 V_CQE_QPID(wq->sq.qid)); 242 CQE_WRID_SQ_IDX(&cqe) = swcqe->idx; 243 cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen)); 244 cq->sw_queue[cq->sw_pidx] = cqe; 245 t4_swcq_produce(cq); 246 } 247 248 int c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count) 249 { 250 int flushed = 0; 251 struct t4_swsqe *swsqe = &wq->sq.sw_sq[wq->sq.cidx + count]; 252 int in_use = wq->sq.in_use - count; 253 254 BUG_ON(in_use < 0); 255 while (in_use--) { 256 swsqe->signaled = 0; 257 insert_sq_cqe(wq, cq, swsqe); 258 swsqe++; 259 if (swsqe == (wq->sq.sw_sq + wq->sq.size)) 260 swsqe = wq->sq.sw_sq; 261 flushed++; 262 } 263 return flushed; 264 } 265 266 /* 267 * Move all CQEs from the HWCQ into the SWCQ. 268 */ 269 void c4iw_flush_hw_cq(struct t4_cq *cq) 270 { 271 struct t4_cqe *cqe = NULL, *swcqe; 272 int ret; 273 274 CTR3(KTR_IW_CXGBE, "%s cq %p cqid 0x%x", __func__, cq, cq->cqid); 275 ret = t4_next_hw_cqe(cq, &cqe); 276 while (!ret) { 277 CTR3(KTR_IW_CXGBE, "%s flushing hwcq cidx 0x%x swcq pidx 0x%x", 278 __func__, cq->cidx, cq->sw_pidx); 279 swcqe = &cq->sw_queue[cq->sw_pidx]; 280 *swcqe = *cqe; 281 swcqe->header |= cpu_to_be32(V_CQE_SWCQE(1)); 282 t4_swcq_produce(cq); 283 t4_hwcq_consume(cq); 284 ret = t4_next_hw_cqe(cq, &cqe); 285 } 286 } 287 288 static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq) 289 { 290 if (CQE_OPCODE(cqe) == FW_RI_TERMINATE) 291 return 0; 292 293 if ((CQE_OPCODE(cqe) == FW_RI_RDMA_WRITE) && RQ_TYPE(cqe)) 294 return 0; 295 296 if ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) && SQ_TYPE(cqe)) 297 return 0; 298 299 if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq)) 300 return 0; 301 return 1; 302 } 303 304 void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count) 305 { 306 struct t4_cqe *cqe; 307 u32 ptr; 308 309 *count = 0; 310 ptr = cq->sw_cidx; 311 while (ptr != cq->sw_pidx) { 312 cqe = &cq->sw_queue[ptr]; 313 if ((SQ_TYPE(cqe) || ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) && 314 wq->sq.oldest_read)) && 315 (CQE_QPID(cqe) == wq->sq.qid)) 316 (*count)++; 317 if (++ptr == cq->size) 318 ptr = 0; 319 } 320 CTR3(KTR_IW_CXGBE, "%s cq %p count %d", __func__, cq, *count); 321 } 322 323 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count) 324 { 325 struct t4_cqe *cqe; 326 u32 ptr; 327 328 *count = 0; 329 CTR2(KTR_IW_CXGBE, "%s count zero %d", __func__, *count); 330 ptr = cq->sw_cidx; 331 while (ptr != cq->sw_pidx) { 332 cqe = &cq->sw_queue[ptr]; 333 if (RQ_TYPE(cqe) && (CQE_OPCODE(cqe) != FW_RI_READ_RESP) && 334 (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq)) 335 (*count)++; 336 if (++ptr == cq->size) 337 ptr = 0; 338 } 339 CTR3(KTR_IW_CXGBE, "%s cq %p count %d", __func__, cq, *count); 340 } 341 342 static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq) 343 { 344 struct t4_swsqe *swsqe; 345 u16 ptr = wq->sq.cidx; 346 int count = wq->sq.in_use; 347 int unsignaled = 0; 348 349 swsqe = &wq->sq.sw_sq[ptr]; 350 while (count--) 351 if (!swsqe->signaled) { 352 if (++ptr == wq->sq.size) 353 ptr = 0; 354 swsqe = &wq->sq.sw_sq[ptr]; 355 unsignaled++; 356 } else if (swsqe->complete) { 357 358 /* 359 * Insert this completed cqe into the swcq. 360 */ 361 CTR3(KTR_IW_CXGBE, 362 "%s moving cqe into swcq sq idx %u cq idx %u", 363 __func__, ptr, cq->sw_pidx); 364 swsqe->cqe.header |= htonl(V_CQE_SWCQE(1)); 365 cq->sw_queue[cq->sw_pidx] = swsqe->cqe; 366 t4_swcq_produce(cq); 367 swsqe->signaled = 0; 368 wq->sq.in_use -= unsignaled; 369 break; 370 } else 371 break; 372 } 373 374 static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe, 375 struct t4_cqe *read_cqe) 376 { 377 read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx; 378 read_cqe->len = cpu_to_be32(wq->sq.oldest_read->read_len); 379 read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(hw_cqe)) | 380 V_CQE_SWCQE(SW_CQE(hw_cqe)) | 381 V_CQE_OPCODE(FW_RI_READ_REQ) | 382 V_CQE_TYPE(1)); 383 read_cqe->bits_type_ts = hw_cqe->bits_type_ts; 384 } 385 386 /* 387 * Return a ptr to the next read wr in the SWSQ or NULL. 388 */ 389 static void advance_oldest_read(struct t4_wq *wq) 390 { 391 392 u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1; 393 394 if (rptr == wq->sq.size) 395 rptr = 0; 396 while (rptr != wq->sq.pidx) { 397 wq->sq.oldest_read = &wq->sq.sw_sq[rptr]; 398 399 if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ) 400 return; 401 if (++rptr == wq->sq.size) 402 rptr = 0; 403 } 404 wq->sq.oldest_read = NULL; 405 } 406 407 /* 408 * poll_cq 409 * 410 * Caller must: 411 * check the validity of the first CQE, 412 * supply the wq assicated with the qpid. 413 * 414 * credit: cq credit to return to sge. 415 * cqe_flushed: 1 iff the CQE is flushed. 416 * cqe: copy of the polled CQE. 417 * 418 * return value: 419 * 0 CQE returned ok. 420 * -EAGAIN CQE skipped, try again. 421 * -EOVERFLOW CQ overflow detected. 422 */ 423 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe, 424 u8 *cqe_flushed, u64 *cookie, u32 *credit) 425 { 426 int ret = 0; 427 struct t4_cqe *hw_cqe, read_cqe; 428 429 *cqe_flushed = 0; 430 *credit = 0; 431 ret = t4_next_cqe(cq, &hw_cqe); 432 if (ret) 433 return ret; 434 435 CTR6(KTR_IW_CXGBE, 436 "%s CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x", __func__, 437 CQE_OVFBIT(hw_cqe), CQE_QPID(hw_cqe), CQE_GENBIT(hw_cqe), 438 CQE_TYPE(hw_cqe), CQE_STATUS(hw_cqe)); 439 CTR5(KTR_IW_CXGBE, 440 "%s opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x", 441 __func__, CQE_OPCODE(hw_cqe), CQE_LEN(hw_cqe), CQE_WRID_HI(hw_cqe), 442 CQE_WRID_LOW(hw_cqe)); 443 444 /* 445 * skip cqe's not affiliated with a QP. 446 */ 447 if (wq == NULL) { 448 ret = -EAGAIN; 449 goto skip_cqe; 450 } 451 452 /* 453 * Special cqe for drain WR completions... 454 */ 455 if (CQE_OPCODE(hw_cqe) == C4IW_DRAIN_OPCODE) { 456 *cookie = CQE_DRAIN_COOKIE(hw_cqe); 457 *cqe = *hw_cqe; 458 goto skip_cqe; 459 } 460 461 /* 462 * Gotta tweak READ completions: 463 * 1) the cqe doesn't contain the sq_wptr from the wr. 464 * 2) opcode not reflected from the wr. 465 * 3) read_len not reflected from the wr. 466 * 4) cq_type is RQ_TYPE not SQ_TYPE. 467 */ 468 if (RQ_TYPE(hw_cqe) && (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP)) { 469 470 /* 471 * If this is an unsolicited read response, then the read 472 * was generated by the kernel driver as part of peer-2-peer 473 * connection setup. So ignore the completion. 474 */ 475 if (!wq->sq.oldest_read) { 476 if (CQE_STATUS(hw_cqe)) 477 t4_set_wq_in_error(wq); 478 ret = -EAGAIN; 479 goto skip_cqe; 480 } 481 482 /* 483 * Don't write to the HWCQ, so create a new read req CQE 484 * in local memory. 485 */ 486 create_read_req_cqe(wq, hw_cqe, &read_cqe); 487 hw_cqe = &read_cqe; 488 advance_oldest_read(wq); 489 } 490 491 if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) { 492 *cqe_flushed = t4_wq_in_error(wq); 493 t4_set_wq_in_error(wq); 494 goto proc_cqe; 495 } 496 497 if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) { 498 ret = -EAGAIN; 499 goto skip_cqe; 500 } 501 502 /* 503 * RECV completion. 504 */ 505 if (RQ_TYPE(hw_cqe)) { 506 507 /* 508 * HW only validates 4 bits of MSN. So we must validate that 509 * the MSN in the SEND is the next expected MSN. If its not, 510 * then we complete this with T4_ERR_MSN and mark the wq in 511 * error. 512 */ 513 514 if (t4_rq_empty(wq)) { 515 t4_set_wq_in_error(wq); 516 ret = -EAGAIN; 517 goto skip_cqe; 518 } 519 if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) { 520 t4_set_wq_in_error(wq); 521 hw_cqe->header |= htonl(V_CQE_STATUS(T4_ERR_MSN)); 522 goto proc_cqe; 523 } 524 goto proc_cqe; 525 } 526 527 /* 528 * If we get here its a send completion. 529 * 530 * Handle out of order completion. These get stuffed 531 * in the SW SQ. Then the SW SQ is walked to move any 532 * now in-order completions into the SW CQ. This handles 533 * 2 cases: 534 * 1) reaping unsignaled WRs when the first subsequent 535 * signaled WR is completed. 536 * 2) out of order read completions. 537 */ 538 if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) { 539 struct t4_swsqe *swsqe; 540 541 CTR2(KTR_IW_CXGBE, 542 "%s out of order completion going in sw_sq at idx %u", 543 __func__, CQE_WRID_SQ_IDX(hw_cqe)); 544 swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)]; 545 swsqe->cqe = *hw_cqe; 546 swsqe->complete = 1; 547 ret = -EAGAIN; 548 goto flush_wq; 549 } 550 551 proc_cqe: 552 *cqe = *hw_cqe; 553 554 /* 555 * Reap the associated WR(s) that are freed up with this 556 * completion. 557 */ 558 if (SQ_TYPE(hw_cqe)) { 559 wq->sq.cidx = CQE_WRID_SQ_IDX(hw_cqe); 560 CTR2(KTR_IW_CXGBE, "%s completing sq idx %u", 561 __func__, wq->sq.cidx); 562 *cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id; 563 t4_sq_consume(wq); 564 } else { 565 CTR2(KTR_IW_CXGBE, "%s completing rq idx %u", 566 __func__, wq->rq.cidx); 567 *cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id; 568 BUG_ON(t4_rq_empty(wq)); 569 t4_rq_consume(wq); 570 } 571 572 flush_wq: 573 /* 574 * Flush any completed cqes that are now in-order. 575 */ 576 flush_completed_wrs(wq, cq); 577 578 skip_cqe: 579 if (SW_CQE(hw_cqe)) { 580 CTR4(KTR_IW_CXGBE, "%s cq %p cqid 0x%x skip sw cqe cidx %u", 581 __func__, cq, cq->cqid, cq->sw_cidx); 582 t4_swcq_consume(cq); 583 } else { 584 CTR4(KTR_IW_CXGBE, "%s cq %p cqid 0x%x skip hw cqe cidx %u", 585 __func__, cq, cq->cqid, cq->cidx); 586 t4_hwcq_consume(cq); 587 } 588 return ret; 589 } 590 591 /* 592 * Get one cq entry from c4iw and map it to openib. 593 * 594 * Returns: 595 * 0 cqe returned 596 * -ENODATA EMPTY; 597 * -EAGAIN caller must try again 598 * any other -errno fatal error 599 */ 600 static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) 601 { 602 struct c4iw_qp *qhp = NULL; 603 struct t4_cqe cqe = {0, 0}, *rd_cqe; 604 struct t4_wq *wq; 605 u32 credit = 0; 606 u8 cqe_flushed; 607 u64 cookie = 0; 608 int ret; 609 610 ret = t4_next_cqe(&chp->cq, &rd_cqe); 611 612 if (ret) 613 return ret; 614 615 qhp = get_qhp(chp->rhp, CQE_QPID(rd_cqe)); 616 if (!qhp) 617 wq = NULL; 618 else { 619 spin_lock(&qhp->lock); 620 wq = &(qhp->wq); 621 } 622 ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit); 623 if (ret) 624 goto out; 625 626 wc->wr_id = cookie; 627 wc->qp = &qhp->ibqp; 628 wc->vendor_err = CQE_STATUS(&cqe); 629 wc->wc_flags = 0; 630 631 CTR5(KTR_IW_CXGBE, "%s qpid 0x%x type %d opcode %d status 0x%x", 632 __func__, CQE_QPID(&cqe), CQE_TYPE(&cqe), CQE_OPCODE(&cqe), 633 CQE_STATUS(&cqe)); 634 CTR5(KTR_IW_CXGBE, "%s len %u wrid hi 0x%x lo 0x%x cookie 0x%llx", 635 __func__, CQE_LEN(&cqe), CQE_WRID_HI(&cqe), CQE_WRID_LOW(&cqe), 636 (unsigned long long)cookie); 637 638 if (CQE_TYPE(&cqe) == 0) { 639 if (!CQE_STATUS(&cqe)) 640 wc->byte_len = CQE_LEN(&cqe); 641 else 642 wc->byte_len = 0; 643 wc->opcode = IB_WC_RECV; 644 if (CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_INV || 645 CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) { 646 wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe); 647 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 648 } 649 } else { 650 switch (CQE_OPCODE(&cqe)) { 651 case FW_RI_RDMA_WRITE: 652 wc->opcode = IB_WC_RDMA_WRITE; 653 break; 654 case FW_RI_READ_REQ: 655 wc->opcode = IB_WC_RDMA_READ; 656 wc->byte_len = CQE_LEN(&cqe); 657 break; 658 case FW_RI_SEND_WITH_INV: 659 case FW_RI_SEND_WITH_SE_INV: 660 wc->opcode = IB_WC_SEND; 661 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 662 break; 663 case FW_RI_SEND: 664 case FW_RI_SEND_WITH_SE: 665 wc->opcode = IB_WC_SEND; 666 break; 667 case FW_RI_BIND_MW: 668 wc->opcode = IB_WC_BIND_MW; 669 break; 670 671 case FW_RI_LOCAL_INV: 672 wc->opcode = IB_WC_LOCAL_INV; 673 break; 674 case FW_RI_FAST_REGISTER: 675 wc->opcode = IB_WC_FAST_REG_MR; 676 break; 677 case C4IW_DRAIN_OPCODE: 678 wc->opcode = IB_WC_SEND; 679 break; 680 default: 681 printf("Unexpected opcode %d " 682 "in the CQE received for QPID = 0x%0x\n", 683 CQE_OPCODE(&cqe), CQE_QPID(&cqe)); 684 ret = -EINVAL; 685 goto out; 686 } 687 } 688 689 if (cqe_flushed) 690 wc->status = IB_WC_WR_FLUSH_ERR; 691 else { 692 693 switch (CQE_STATUS(&cqe)) { 694 case T4_ERR_SUCCESS: 695 wc->status = IB_WC_SUCCESS; 696 break; 697 case T4_ERR_STAG: 698 wc->status = IB_WC_LOC_ACCESS_ERR; 699 break; 700 case T4_ERR_PDID: 701 wc->status = IB_WC_LOC_PROT_ERR; 702 break; 703 case T4_ERR_QPID: 704 case T4_ERR_ACCESS: 705 wc->status = IB_WC_LOC_ACCESS_ERR; 706 break; 707 case T4_ERR_WRAP: 708 wc->status = IB_WC_GENERAL_ERR; 709 break; 710 case T4_ERR_BOUND: 711 wc->status = IB_WC_LOC_LEN_ERR; 712 break; 713 case T4_ERR_INVALIDATE_SHARED_MR: 714 case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND: 715 wc->status = IB_WC_MW_BIND_ERR; 716 break; 717 case T4_ERR_CRC: 718 case T4_ERR_MARKER: 719 case T4_ERR_PDU_LEN_ERR: 720 case T4_ERR_OUT_OF_RQE: 721 case T4_ERR_DDP_VERSION: 722 case T4_ERR_RDMA_VERSION: 723 case T4_ERR_DDP_QUEUE_NUM: 724 case T4_ERR_MSN: 725 case T4_ERR_TBIT: 726 case T4_ERR_MO: 727 case T4_ERR_MSN_RANGE: 728 case T4_ERR_IRD_OVERFLOW: 729 case T4_ERR_OPCODE: 730 case T4_ERR_INTERNAL_ERR: 731 wc->status = IB_WC_FATAL_ERR; 732 break; 733 case T4_ERR_SWFLUSH: 734 wc->status = IB_WC_WR_FLUSH_ERR; 735 break; 736 default: 737 printf("Unexpected cqe_status 0x%x for QPID = 0x%0x\n", 738 CQE_STATUS(&cqe), CQE_QPID(&cqe)); 739 wc->status = IB_WC_FATAL_ERR; 740 } 741 } 742 out: 743 if (wq) 744 spin_unlock(&qhp->lock); 745 return ret; 746 } 747 748 int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 749 { 750 struct c4iw_cq *chp; 751 unsigned long flags; 752 int npolled; 753 int err = 0; 754 755 chp = to_c4iw_cq(ibcq); 756 757 spin_lock_irqsave(&chp->lock, flags); 758 for (npolled = 0; npolled < num_entries; ++npolled) { 759 do { 760 err = c4iw_poll_cq_one(chp, wc + npolled); 761 } while (err == -EAGAIN); 762 if (err) 763 break; 764 } 765 spin_unlock_irqrestore(&chp->lock, flags); 766 return !err || err == -ENODATA ? npolled : err; 767 } 768 769 int c4iw_destroy_cq(struct ib_cq *ib_cq) 770 { 771 struct c4iw_cq *chp; 772 struct c4iw_ucontext *ucontext; 773 774 CTR2(KTR_IW_CXGBE, "%s ib_cq %p", __func__, ib_cq); 775 chp = to_c4iw_cq(ib_cq); 776 777 remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid); 778 atomic_dec(&chp->refcnt); 779 wait_event(chp->wait, !atomic_read(&chp->refcnt)); 780 781 ucontext = ib_cq->uobject ? to_c4iw_ucontext(ib_cq->uobject->context) 782 : NULL; 783 destroy_cq(&chp->rhp->rdev, &chp->cq, 784 ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx); 785 kfree(chp); 786 return 0; 787 } 788 789 struct ib_cq * 790 c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr, 791 struct ib_ucontext *ib_context, struct ib_udata *udata) 792 { 793 struct c4iw_dev *rhp; 794 struct c4iw_cq *chp; 795 struct c4iw_create_cq_resp uresp; 796 struct c4iw_ucontext *ucontext = NULL; 797 int ret; 798 size_t memsize, hwentries; 799 struct c4iw_mm_entry *mm, *mm2; 800 int entries = attr->cqe; 801 802 CTR3(KTR_IW_CXGBE, "%s ib_dev %p entries %d", __func__, ibdev, entries); 803 804 rhp = to_c4iw_dev(ibdev); 805 806 chp = kzalloc(sizeof(*chp), GFP_KERNEL); 807 if (!chp) 808 return ERR_PTR(-ENOMEM); 809 810 if (ib_context) 811 ucontext = to_c4iw_ucontext(ib_context); 812 813 /* account for the status page. */ 814 entries++; 815 816 /* IQ needs one extra entry to differentiate full vs empty. */ 817 entries++; 818 819 /* 820 * entries must be multiple of 16 for HW. 821 */ 822 entries = roundup(entries, 16); 823 824 /* 825 * Make actual HW queue 2x to avoid cidx_inc overflows. 826 */ 827 hwentries = entries * 2; 828 829 /* 830 * Make HW queue at least 64 entries so GTS updates aren't too 831 * frequent. 832 */ 833 if (hwentries < 64) 834 hwentries = 64; 835 836 memsize = hwentries * sizeof *chp->cq.queue; 837 838 /* 839 * memsize must be a multiple of the page size if its a user cq. 840 */ 841 if (ucontext) { 842 memsize = roundup(memsize, PAGE_SIZE); 843 hwentries = memsize / sizeof *chp->cq.queue; 844 while (hwentries > T4_MAX_IQ_SIZE) { 845 memsize -= PAGE_SIZE; 846 hwentries = memsize / sizeof *chp->cq.queue; 847 } 848 } 849 chp->cq.size = hwentries; 850 chp->cq.memsize = memsize; 851 852 ret = create_cq(&rhp->rdev, &chp->cq, 853 ucontext ? &ucontext->uctx : &rhp->rdev.uctx); 854 if (ret) 855 goto err1; 856 857 chp->rhp = rhp; 858 chp->cq.size--; /* status page */ 859 chp->ibcq.cqe = entries - 2; 860 spin_lock_init(&chp->lock); 861 spin_lock_init(&chp->comp_handler_lock); 862 atomic_set(&chp->refcnt, 1); 863 init_waitqueue_head(&chp->wait); 864 ret = insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid); 865 if (ret) 866 goto err2; 867 868 if (ucontext) { 869 mm = kmalloc(sizeof *mm, GFP_KERNEL); 870 if (!mm) 871 goto err3; 872 mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); 873 if (!mm2) 874 goto err4; 875 876 memset(&uresp, 0, sizeof(uresp)); 877 uresp.qid_mask = rhp->rdev.cqmask; 878 uresp.cqid = chp->cq.cqid; 879 uresp.size = chp->cq.size; 880 uresp.memsize = chp->cq.memsize; 881 spin_lock(&ucontext->mmap_lock); 882 uresp.key = ucontext->key; 883 ucontext->key += PAGE_SIZE; 884 uresp.gts_key = ucontext->key; 885 ucontext->key += PAGE_SIZE; 886 spin_unlock(&ucontext->mmap_lock); 887 ret = ib_copy_to_udata(udata, &uresp, 888 sizeof(uresp) - sizeof(uresp.reserved)); 889 if (ret) 890 goto err5; 891 892 mm->key = uresp.key; 893 mm->addr = vtophys(chp->cq.queue); 894 mm->len = chp->cq.memsize; 895 insert_mmap(ucontext, mm); 896 897 mm2->key = uresp.gts_key; 898 mm2->addr = chp->cq.ugts; 899 mm2->len = PAGE_SIZE; 900 insert_mmap(ucontext, mm2); 901 } 902 CTR6(KTR_IW_CXGBE, 903 "%s cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx", 904 __func__, chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize, 905 (unsigned long long) chp->cq.dma_addr); 906 return &chp->ibcq; 907 err5: 908 kfree(mm2); 909 err4: 910 kfree(mm); 911 err3: 912 remove_handle(rhp, &rhp->cqidr, chp->cq.cqid); 913 err2: 914 destroy_cq(&chp->rhp->rdev, &chp->cq, 915 ucontext ? &ucontext->uctx : &rhp->rdev.uctx); 916 err1: 917 kfree(chp); 918 return ERR_PTR(ret); 919 } 920 921 int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata) 922 { 923 return -ENOSYS; 924 } 925 926 int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 927 { 928 struct c4iw_cq *chp; 929 int ret; 930 unsigned long flag; 931 932 chp = to_c4iw_cq(ibcq); 933 spin_lock_irqsave(&chp->lock, flag); 934 ret = t4_arm_cq(&chp->cq, 935 (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED); 936 spin_unlock_irqrestore(&chp->lock, flag); 937 if (ret && !(flags & IB_CQ_REPORT_MISSED_EVENTS)) 938 ret = 0; 939 return ret; 940 } 941 #endif 942