1 /* 2 * Copyright (c) 2006-2016 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <config.h> 33 34 #include <stdio.h> 35 #include <syslog.h> 36 #include <pthread.h> 37 #include <sys/errno.h> 38 #include <infiniband/opcode.h> 39 #include "libcxgb4.h" 40 #include "cxgb4-abi.h" 41 42 static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq) 43 { 44 struct t4_cqe cqe; 45 46 PDBG("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__, 47 wq, cq, cq->sw_cidx, cq->sw_pidx); 48 memset(&cqe, 0, sizeof(cqe)); 49 cqe.header = htobe32(V_CQE_STATUS(T4_ERR_SWFLUSH) | 50 V_CQE_OPCODE(FW_RI_SEND) | 51 V_CQE_TYPE(0) | 52 V_CQE_SWCQE(1) | 53 V_CQE_QPID(wq->sq.qid)); 54 cqe.bits_type_ts = htobe64(V_CQE_GENBIT((u64)cq->gen)); 55 cq->sw_queue[cq->sw_pidx] = cqe; 56 t4_swcq_produce(cq); 57 } 58 59 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count) 60 { 61 int flushed = 0; 62 int in_use = wq->rq.in_use - count; 63 64 BUG_ON(in_use < 0); 65 PDBG("%s wq %p cq %p rq.in_use %u skip count %u\n", __func__, 66 wq, cq, wq->rq.in_use, count); 67 while (in_use--) { 68 insert_recv_cqe(wq, cq); 69 flushed++; 70 } 71 return flushed; 72 } 73 74 static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq, 75 struct t4_swsqe *swcqe) 76 { 77 struct t4_cqe cqe; 78 79 PDBG("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__, 80 wq, cq, cq->sw_cidx, cq->sw_pidx); 81 memset(&cqe, 0, sizeof(cqe)); 82 cqe.header = htobe32(V_CQE_STATUS(T4_ERR_SWFLUSH) | 83 V_CQE_OPCODE(swcqe->opcode) | 84 V_CQE_TYPE(1) | 85 V_CQE_SWCQE(1) | 86 V_CQE_QPID(wq->sq.qid)); 87 CQE_WRID_SQ_IDX(&cqe) = swcqe->idx; 88 cqe.bits_type_ts = htobe64(V_CQE_GENBIT((u64)cq->gen)); 89 cq->sw_queue[cq->sw_pidx] = cqe; 90 t4_swcq_produce(cq); 91 } 92 93 static void advance_oldest_read(struct t4_wq *wq); 94 95 void c4iw_flush_sq(struct c4iw_qp *qhp) 96 { 97 unsigned short flushed = 0; 98 struct t4_wq *wq = &qhp->wq; 99 struct c4iw_cq *chp = to_c4iw_cq(qhp->ibv_qp.send_cq); 100 struct t4_cq *cq = &chp->cq; 101 int idx; 102 struct t4_swsqe *swsqe; 103 104 if (wq->sq.flush_cidx == -1) 105 wq->sq.flush_cidx = wq->sq.cidx; 106 idx = wq->sq.flush_cidx; 107 BUG_ON(idx >= wq->sq.size); 108 while (idx != wq->sq.pidx) { 109 swsqe = &wq->sq.sw_sq[idx]; 110 BUG_ON(swsqe->flushed); 111 swsqe->flushed = 1; 112 insert_sq_cqe(wq, cq, swsqe); 113 if (wq->sq.oldest_read == swsqe) { 114 BUG_ON(swsqe->opcode != FW_RI_READ_REQ); 115 advance_oldest_read(wq); 116 } 117 flushed++; 118 if (++idx == wq->sq.size) 119 idx = 0; 120 } 121 wq->sq.flush_cidx += flushed; 122 if (wq->sq.flush_cidx >= wq->sq.size) 123 wq->sq.flush_cidx -= wq->sq.size; 124 } 125 126 static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq) 127 { 128 struct t4_swsqe *swsqe; 129 unsigned short cidx; 130 131 if (wq->sq.flush_cidx == -1) 132 wq->sq.flush_cidx = wq->sq.cidx; 133 cidx = wq->sq.flush_cidx; 134 BUG_ON(cidx >= wq->sq.size); 135 136 while (cidx != wq->sq.pidx) { 137 swsqe = &wq->sq.sw_sq[cidx]; 138 if (!swsqe->signaled) { 139 if (++cidx == wq->sq.size) 140 cidx = 0; 141 } else if (swsqe->complete) { 142 143 BUG_ON(swsqe->flushed); 144 145 /* 146 * Insert this completed cqe into the swcq. 147 */ 148 PDBG("%s moving cqe into swcq sq idx %u cq idx %u\n", 149 __func__, cidx, cq->sw_pidx); 150 151 swsqe->cqe.header |= htobe32(V_CQE_SWCQE(1)); 152 cq->sw_queue[cq->sw_pidx] = swsqe->cqe; 153 t4_swcq_produce(cq); 154 swsqe->flushed = 1; 155 if (++cidx == wq->sq.size) 156 cidx = 0; 157 wq->sq.flush_cidx = cidx; 158 } else 159 break; 160 } 161 } 162 163 static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe, 164 struct t4_cqe *read_cqe) 165 { 166 read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx; 167 read_cqe->len = be32toh(wq->sq.oldest_read->read_len); 168 read_cqe->header = htobe32(V_CQE_QPID(CQE_QPID(hw_cqe)) | 169 V_CQE_SWCQE(SW_CQE(hw_cqe)) | 170 V_CQE_OPCODE(FW_RI_READ_REQ) | 171 V_CQE_TYPE(1)); 172 read_cqe->bits_type_ts = hw_cqe->bits_type_ts; 173 } 174 175 static void advance_oldest_read(struct t4_wq *wq) 176 { 177 178 u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1; 179 180 if (rptr == wq->sq.size) 181 rptr = 0; 182 while (rptr != wq->sq.pidx) { 183 wq->sq.oldest_read = &wq->sq.sw_sq[rptr]; 184 185 if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ) 186 return; 187 if (++rptr == wq->sq.size) 188 rptr = 0; 189 } 190 wq->sq.oldest_read = NULL; 191 } 192 193 /* 194 * Move all CQEs from the HWCQ into the SWCQ. 195 * Deal with out-of-order and/or completions that complete 196 * prior unsignalled WRs. 197 */ 198 void c4iw_flush_hw_cq(struct c4iw_cq *chp) 199 { 200 struct t4_cqe *hw_cqe, *swcqe, read_cqe; 201 struct c4iw_qp *qhp; 202 struct t4_swsqe *swsqe; 203 int ret; 204 205 PDBG("%s cqid 0x%x\n", __func__, chp->cq.cqid); 206 ret = t4_next_hw_cqe(&chp->cq, &hw_cqe); 207 208 /* 209 * This logic is similar to poll_cq(), but not quite the same 210 * unfortunately. Need to move pertinent HW CQEs to the SW CQ but 211 * also do any translation magic that poll_cq() normally does. 212 */ 213 while (!ret) { 214 qhp = get_qhp(chp->rhp, CQE_QPID(hw_cqe)); 215 216 /* 217 * drop CQEs with no associated QP 218 */ 219 if (qhp == NULL) 220 goto next_cqe; 221 222 if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) 223 goto next_cqe; 224 225 if (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP) { 226 227 /* 228 * If we have reached here because of async 229 * event or other error, and have egress error 230 * then drop 231 */ 232 if (CQE_TYPE(hw_cqe) == 1) { 233 syslog(LOG_CRIT, "%s: got egress error in \ 234 read-response, dropping!\n", __func__); 235 goto next_cqe; 236 } 237 238 /* 239 * drop peer2peer RTR reads. 240 */ 241 if (CQE_WRID_STAG(hw_cqe) == 1) 242 goto next_cqe; 243 244 /* 245 * Eat completions for unsignaled read WRs. 246 */ 247 if (!qhp->wq.sq.oldest_read->signaled) { 248 advance_oldest_read(&qhp->wq); 249 goto next_cqe; 250 } 251 252 /* 253 * Don't write to the HWCQ, create a new read req CQE 254 * in local memory and move it into the swcq. 255 */ 256 create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe); 257 hw_cqe = &read_cqe; 258 advance_oldest_read(&qhp->wq); 259 } 260 261 /* if its a SQ completion, then do the magic to move all the 262 * unsignaled and now in-order completions into the swcq. 263 */ 264 if (SQ_TYPE(hw_cqe)) { 265 int idx = CQE_WRID_SQ_IDX(hw_cqe); 266 267 BUG_ON(idx >= qhp->wq.sq.size); 268 swsqe = &qhp->wq.sq.sw_sq[idx]; 269 swsqe->cqe = *hw_cqe; 270 swsqe->complete = 1; 271 flush_completed_wrs(&qhp->wq, &chp->cq); 272 } else { 273 swcqe = &chp->cq.sw_queue[chp->cq.sw_pidx]; 274 *swcqe = *hw_cqe; 275 swcqe->header |= htobe32(V_CQE_SWCQE(1)); 276 t4_swcq_produce(&chp->cq); 277 } 278 next_cqe: 279 t4_hwcq_consume(&chp->cq); 280 ret = t4_next_hw_cqe(&chp->cq, &hw_cqe); 281 } 282 } 283 284 static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq) 285 { 286 if (CQE_OPCODE(cqe) == FW_RI_TERMINATE) 287 return 0; 288 289 if ((CQE_OPCODE(cqe) == FW_RI_RDMA_WRITE) && RQ_TYPE(cqe)) 290 return 0; 291 292 if ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) && SQ_TYPE(cqe)) 293 return 0; 294 295 if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq)) 296 return 0; 297 return 1; 298 } 299 300 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count) 301 { 302 struct t4_cqe *cqe; 303 u32 ptr; 304 305 *count = 0; 306 ptr = cq->sw_cidx; 307 BUG_ON(ptr >= cq->size); 308 while (ptr != cq->sw_pidx) { 309 cqe = &cq->sw_queue[ptr]; 310 if (RQ_TYPE(cqe) && (CQE_OPCODE(cqe) != FW_RI_READ_RESP) && 311 (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq)) 312 (*count)++; 313 if (++ptr == cq->size) 314 ptr = 0; 315 } 316 PDBG("%s cq %p count %d\n", __func__, cq, *count); 317 } 318 319 static void dump_cqe(void *arg) 320 { 321 u64 *p = arg; 322 syslog(LOG_NOTICE, "cxgb4 err cqe %016llx %016llx %016llx %016llx\n", 323 (long long)be64toh(p[0]), 324 (long long)be64toh(p[1]), 325 (long long)be64toh(p[2]), 326 (long long)be64toh(p[3])); 327 } 328 329 /* 330 * poll_cq 331 * 332 * Caller must: 333 * check the validity of the first CQE, 334 * supply the wq assicated with the qpid. 335 * 336 * credit: cq credit to return to sge. 337 * cqe_flushed: 1 iff the CQE is flushed. 338 * cqe: copy of the polled CQE. 339 * 340 * return value: 341 * 0 CQE returned ok. 342 * -EAGAIN CQE skipped, try again. 343 * -EOVERFLOW CQ overflow detected. 344 */ 345 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe, 346 u8 *cqe_flushed, u64 *cookie, u32 *credit) 347 { 348 int ret = 0; 349 struct t4_cqe *hw_cqe, read_cqe; 350 351 *cqe_flushed = 0; 352 *credit = 0; 353 354 ret = t4_next_cqe(cq, &hw_cqe); 355 if (ret) 356 return ret; 357 358 PDBG("%s CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x" 359 " opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n", 360 __func__, CQE_OVFBIT(hw_cqe), CQE_QPID(hw_cqe), 361 CQE_GENBIT(hw_cqe), CQE_TYPE(hw_cqe), CQE_STATUS(hw_cqe), 362 CQE_OPCODE(hw_cqe), CQE_LEN(hw_cqe), CQE_WRID_HI(hw_cqe), 363 CQE_WRID_LOW(hw_cqe)); 364 365 /* 366 * skip cqe's not affiliated with a QP. 367 */ 368 if (wq == NULL) { 369 ret = -EAGAIN; 370 goto skip_cqe; 371 } 372 373 /* 374 * Gotta tweak READ completions: 375 * 1) the cqe doesn't contain the sq_wptr from the wr. 376 * 2) opcode not reflected from the wr. 377 * 3) read_len not reflected from the wr. 378 * 4) T4 HW (for now) inserts target read response failures which 379 * need to be skipped. 380 */ 381 if (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP) { 382 383 /* 384 * If we have reached here because of async 385 * event or other error, and have egress error 386 * then drop 387 */ 388 if (CQE_TYPE(hw_cqe) == 1) { 389 syslog(LOG_CRIT, "%s: got egress error in \ 390 read-response, dropping!\n", __func__); 391 if (CQE_STATUS(hw_cqe)) 392 t4_set_wq_in_error(wq); 393 ret = -EAGAIN; 394 goto skip_cqe; 395 } 396 397 /* 398 * If this is an unsolicited read response, then the read 399 * was generated by the kernel driver as part of peer-2-peer 400 * connection setup, or a target read response failure. 401 * So skip the completion. 402 */ 403 if (CQE_WRID_STAG(hw_cqe) == 1) { 404 if (CQE_STATUS(hw_cqe)) 405 t4_set_wq_in_error(wq); 406 ret = -EAGAIN; 407 goto skip_cqe; 408 } 409 410 /* 411 * Eat completions for unsignaled read WRs. 412 */ 413 if (!wq->sq.oldest_read->signaled) { 414 advance_oldest_read(wq); 415 ret = -EAGAIN; 416 goto skip_cqe; 417 } 418 419 /* 420 * Don't write to the HWCQ, so create a new read req CQE 421 * in local memory. 422 */ 423 create_read_req_cqe(wq, hw_cqe, &read_cqe); 424 hw_cqe = &read_cqe; 425 advance_oldest_read(wq); 426 } 427 428 if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) { 429 ret = -EAGAIN; 430 goto skip_cqe; 431 } 432 433 if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) { 434 *cqe_flushed = (CQE_STATUS(hw_cqe) == T4_ERR_SWFLUSH); 435 wq->error = 1; 436 437 if (!*cqe_flushed && CQE_STATUS(hw_cqe)) 438 dump_cqe(hw_cqe); 439 440 BUG_ON((cqe_flushed == 0) && !SW_CQE(hw_cqe)); 441 goto proc_cqe; 442 } 443 444 /* 445 * RECV completion. 446 */ 447 if (RQ_TYPE(hw_cqe)) { 448 449 /* 450 * HW only validates 4 bits of MSN. So we must validate that 451 * the MSN in the SEND is the next expected MSN. If its not, 452 * then we complete this with T4_ERR_MSN and mark the wq in 453 * error. 454 */ 455 456 if (t4_rq_empty(wq)) { 457 t4_set_wq_in_error(wq); 458 ret = -EAGAIN; 459 goto skip_cqe; 460 } 461 if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) { 462 t4_set_wq_in_error(wq); 463 hw_cqe->header |= htobe32(V_CQE_STATUS(T4_ERR_MSN)); 464 goto proc_cqe; 465 } 466 goto proc_cqe; 467 } 468 469 /* 470 * If we get here its a send completion. 471 * 472 * Handle out of order completion. These get stuffed 473 * in the SW SQ. Then the SW SQ is walked to move any 474 * now in-order completions into the SW CQ. This handles 475 * 2 cases: 476 * 1) reaping unsignaled WRs when the first subsequent 477 * signaled WR is completed. 478 * 2) out of order read completions. 479 */ 480 if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) { 481 struct t4_swsqe *swsqe; 482 int idx = CQE_WRID_SQ_IDX(hw_cqe); 483 484 PDBG("%s out of order completion going in sw_sq at idx %u\n", 485 __func__, idx); 486 BUG_ON(idx >= wq->sq.size); 487 swsqe = &wq->sq.sw_sq[idx]; 488 swsqe->cqe = *hw_cqe; 489 swsqe->complete = 1; 490 ret = -EAGAIN; 491 goto flush_wq; 492 } 493 494 proc_cqe: 495 *cqe = *hw_cqe; 496 497 /* 498 * Reap the associated WR(s) that are freed up with this 499 * completion. 500 */ 501 if (SQ_TYPE(hw_cqe)) { 502 int idx = CQE_WRID_SQ_IDX(hw_cqe); 503 BUG_ON(idx >= wq->sq.size); 504 505 /* 506 * Account for any unsignaled completions completed by 507 * this signaled completion. In this case, cidx points 508 * to the first unsignaled one, and idx points to the 509 * signaled one. So adjust in_use based on this delta. 510 * if this is not completing any unsigned wrs, then the 511 * delta will be 0. Handle wrapping also! 512 */ 513 if (idx < wq->sq.cidx) 514 wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx; 515 else 516 wq->sq.in_use -= idx - wq->sq.cidx; 517 BUG_ON(wq->sq.in_use <= 0 || wq->sq.in_use >= wq->sq.size); 518 519 wq->sq.cidx = (u16)idx; 520 PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx); 521 *cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id; 522 t4_sq_consume(wq); 523 } else { 524 PDBG("%s completing rq idx %u\n", __func__, wq->rq.cidx); 525 BUG_ON(wq->rq.cidx >= wq->rq.size); 526 *cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id; 527 BUG_ON(t4_rq_empty(wq)); 528 t4_rq_consume(wq); 529 goto skip_cqe; 530 } 531 532 flush_wq: 533 /* 534 * Flush any completed cqes that are now in-order. 535 */ 536 flush_completed_wrs(wq, cq); 537 538 skip_cqe: 539 if (SW_CQE(hw_cqe)) { 540 PDBG("%s cq %p cqid 0x%x skip sw cqe cidx %u\n", 541 __func__, cq, cq->cqid, cq->sw_cidx); 542 t4_swcq_consume(cq); 543 } else { 544 PDBG("%s cq %p cqid 0x%x skip hw cqe cidx %u\n", 545 __func__, cq, cq->cqid, cq->cidx); 546 t4_hwcq_consume(cq); 547 } 548 return ret; 549 } 550 551 /* 552 * Get one cq entry from c4iw and map it to openib. 553 * 554 * Returns: 555 * 0 cqe returned 556 * -ENODATA EMPTY; 557 * -EAGAIN caller must try again 558 * any other -errno fatal error 559 */ 560 static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ibv_wc *wc) 561 { 562 struct c4iw_qp *qhp = NULL; 563 struct t4_cqe cqe, *rd_cqe; 564 struct t4_wq *wq; 565 u32 credit = 0; 566 u8 cqe_flushed; 567 u64 cookie = 0; 568 int ret; 569 570 ret = t4_next_cqe(&chp->cq, &rd_cqe); 571 572 if (ret) { 573 #ifdef STALL_DETECTION 574 if (ret == -ENODATA && stall_to && !chp->dumped) { 575 struct timeval t; 576 577 gettimeofday(&t, NULL); 578 if ((t.tv_sec - chp->time.tv_sec) > stall_to) { 579 dump_state(); 580 chp->dumped = 1; 581 } 582 } 583 #endif 584 return ret; 585 } 586 587 #ifdef STALL_DETECTION 588 gettimeofday(&chp->time, NULL); 589 #endif 590 591 qhp = get_qhp(chp->rhp, CQE_QPID(rd_cqe)); 592 if (!qhp) 593 wq = NULL; 594 else { 595 pthread_spin_lock(&qhp->lock); 596 wq = &(qhp->wq); 597 } 598 ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit); 599 if (ret) 600 goto out; 601 602 INC_STAT(cqe); 603 wc->wr_id = cookie; 604 wc->qp_num = qhp->wq.sq.qid; 605 wc->vendor_err = CQE_STATUS(&cqe); 606 wc->wc_flags = 0; 607 608 PDBG("%s qpid 0x%x type %d opcode %d status 0x%x wrid hi 0x%x " 609 "lo 0x%x cookie 0x%llx\n", __func__, 610 CQE_QPID(&cqe), CQE_TYPE(&cqe), 611 CQE_OPCODE(&cqe), CQE_STATUS(&cqe), CQE_WRID_HI(&cqe), 612 CQE_WRID_LOW(&cqe), (unsigned long long)cookie); 613 614 if (CQE_TYPE(&cqe) == 0) { 615 if (!CQE_STATUS(&cqe)) 616 wc->byte_len = CQE_LEN(&cqe); 617 else 618 wc->byte_len = 0; 619 wc->opcode = IBV_WC_RECV; 620 } else { 621 switch (CQE_OPCODE(&cqe)) { 622 case FW_RI_RDMA_WRITE: 623 wc->opcode = IBV_WC_RDMA_WRITE; 624 break; 625 case FW_RI_READ_REQ: 626 wc->opcode = IBV_WC_RDMA_READ; 627 wc->byte_len = CQE_LEN(&cqe); 628 break; 629 case FW_RI_SEND: 630 case FW_RI_SEND_WITH_SE: 631 case FW_RI_SEND_WITH_INV: 632 case FW_RI_SEND_WITH_SE_INV: 633 wc->opcode = IBV_WC_SEND; 634 break; 635 case FW_RI_BIND_MW: 636 wc->opcode = IBV_WC_BIND_MW; 637 break; 638 default: 639 PDBG("Unexpected opcode %d " 640 "in the CQE received for QPID=0x%0x\n", 641 CQE_OPCODE(&cqe), CQE_QPID(&cqe)); 642 ret = -EINVAL; 643 goto out; 644 } 645 } 646 647 if (cqe_flushed) 648 wc->status = IBV_WC_WR_FLUSH_ERR; 649 else { 650 651 switch (CQE_STATUS(&cqe)) { 652 case T4_ERR_SUCCESS: 653 wc->status = IBV_WC_SUCCESS; 654 break; 655 case T4_ERR_STAG: 656 wc->status = IBV_WC_LOC_ACCESS_ERR; 657 break; 658 case T4_ERR_PDID: 659 wc->status = IBV_WC_LOC_PROT_ERR; 660 break; 661 case T4_ERR_QPID: 662 case T4_ERR_ACCESS: 663 wc->status = IBV_WC_LOC_ACCESS_ERR; 664 break; 665 case T4_ERR_WRAP: 666 wc->status = IBV_WC_GENERAL_ERR; 667 break; 668 case T4_ERR_BOUND: 669 wc->status = IBV_WC_LOC_LEN_ERR; 670 break; 671 case T4_ERR_INVALIDATE_SHARED_MR: 672 case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND: 673 wc->status = IBV_WC_MW_BIND_ERR; 674 break; 675 case T4_ERR_CRC: 676 case T4_ERR_MARKER: 677 case T4_ERR_PDU_LEN_ERR: 678 case T4_ERR_OUT_OF_RQE: 679 case T4_ERR_DDP_VERSION: 680 case T4_ERR_RDMA_VERSION: 681 case T4_ERR_DDP_QUEUE_NUM: 682 case T4_ERR_MSN: 683 case T4_ERR_TBIT: 684 case T4_ERR_MO: 685 case T4_ERR_MSN_RANGE: 686 case T4_ERR_IRD_OVERFLOW: 687 case T4_ERR_OPCODE: 688 case T4_ERR_INTERNAL_ERR: 689 wc->status = IBV_WC_FATAL_ERR; 690 break; 691 case T4_ERR_SWFLUSH: 692 wc->status = IBV_WC_WR_FLUSH_ERR; 693 break; 694 default: 695 PDBG("Unexpected cqe_status 0x%x for QPID=0x%0x\n", 696 CQE_STATUS(&cqe), CQE_QPID(&cqe)); 697 wc->status = IBV_WC_FATAL_ERR; 698 } 699 } 700 if (wc->status && wc->status != IBV_WC_WR_FLUSH_ERR) 701 syslog(LOG_NOTICE, "cxgb4 app err cqid %u qpid %u " 702 "type %u opcode %u status 0x%x\n", 703 chp->cq.cqid, CQE_QPID(&cqe), CQE_TYPE(&cqe), 704 CQE_OPCODE(&cqe), CQE_STATUS(&cqe)); 705 out: 706 if (wq) 707 pthread_spin_unlock(&qhp->lock); 708 return ret; 709 } 710 711 int c4iw_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc *wc) 712 { 713 struct c4iw_cq *chp; 714 int npolled; 715 int err = 0; 716 717 chp = to_c4iw_cq(ibcq); 718 719 if (t4_cq_in_error(&chp->cq)) { 720 t4_reset_cq_in_error(&chp->cq); 721 c4iw_flush_qps(chp->rhp); 722 } 723 724 if (!num_entries) 725 return t4_cq_notempty(&chp->cq); 726 727 pthread_spin_lock(&chp->lock); 728 for (npolled = 0; npolled < num_entries; ++npolled) { 729 do { 730 err = c4iw_poll_cq_one(chp, wc + npolled); 731 } while (err == -EAGAIN); 732 if (err) 733 break; 734 } 735 pthread_spin_unlock(&chp->lock); 736 return !err || err == -ENODATA ? npolled : err; 737 } 738 739 int c4iw_arm_cq(struct ibv_cq *ibcq, int solicited) 740 { 741 struct c4iw_cq *chp; 742 int ret; 743 744 INC_STAT(arm); 745 chp = to_c4iw_cq(ibcq); 746 pthread_spin_lock(&chp->lock); 747 ret = t4_arm_cq(&chp->cq, solicited); 748 pthread_spin_unlock(&chp->lock); 749 return ret; 750 } 751