1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 4 /* Copyright (c) 2008-2019, IBM Corporation */ 5 6 #include <linux/errno.h> 7 #include <linux/types.h> 8 #include <linux/net.h> 9 #include <linux/scatterlist.h> 10 #include <linux/highmem.h> 11 #include <net/tcp.h> 12 13 #include <rdma/iw_cm.h> 14 #include <rdma/ib_verbs.h> 15 #include <rdma/ib_user_verbs.h> 16 17 #include "siw.h" 18 #include "siw_verbs.h" 19 #include "siw_mem.h" 20 21 #define MAX_HDR_INLINE \ 22 (((uint32_t)(sizeof(struct siw_rreq_pkt) - \ 23 sizeof(struct iwarp_send))) & 0xF8) 24 25 static struct page *siw_get_pblpage(struct siw_mem *mem, u64 addr, int *idx) 26 { 27 struct siw_pbl *pbl = mem->pbl; 28 u64 offset = addr - mem->va; 29 dma_addr_t paddr = siw_pbl_get_buffer(pbl, offset, NULL, idx); 30 31 if (paddr) 32 return ib_virt_dma_to_page(paddr); 33 34 return NULL; 35 } 36 37 static struct page *siw_get_page(struct siw_mem *mem, struct siw_sge *sge, 38 unsigned long offset, int *pbl_idx) 39 { 40 if (!mem->is_pbl) 41 return siw_get_upage(mem->umem, sge->laddr + offset); 42 else 43 return siw_get_pblpage(mem, sge->laddr + offset, pbl_idx); 44 } 45 46 /* 47 * Copy short payload at provided destination payload address 48 */ 49 static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr) 50 { 51 struct siw_wqe *wqe = &c_tx->wqe_active; 52 struct siw_sge *sge = &wqe->sqe.sge[0]; 53 u32 bytes = sge->length; 54 55 if (bytes > MAX_HDR_INLINE || wqe->sqe.num_sge != 1) 56 return MAX_HDR_INLINE + 1; 57 58 if (!bytes) 59 return 0; 60 61 if (tx_flags(wqe) & SIW_WQE_INLINE) { 62 memcpy(paddr, &wqe->sqe.sge[1], bytes); 63 } else { 64 struct siw_mem *mem = wqe->mem[0]; 65 66 if (!mem->mem_obj) { 67 /* Kernel client using kva */ 68 memcpy(paddr, ib_virt_dma_to_ptr(sge->laddr), bytes); 69 } else if (c_tx->in_syscall) { 70 if (copy_from_user(paddr, u64_to_user_ptr(sge->laddr), 71 bytes)) 72 return -EFAULT; 73 } else { 74 unsigned int off = sge->laddr & ~PAGE_MASK; 75 struct page *p; 76 char *buffer; 77 int pbl_idx = 0; 78 79 p = siw_get_page(mem, sge, 0, &pbl_idx); 80 if (unlikely(!p)) 81 return -EFAULT; 82 83 buffer = kmap_local_page(p); 84 85 if (likely(PAGE_SIZE - off >= bytes)) { 86 memcpy(paddr, buffer + off, bytes); 87 } else { 88 unsigned long part = bytes - (PAGE_SIZE - off); 89 90 memcpy(paddr, buffer + off, part); 91 kunmap_local(buffer); 92 93 p = siw_get_page(mem, sge, part, &pbl_idx); 94 if (unlikely(!p)) 95 return -EFAULT; 96 97 buffer = kmap_local_page(p); 98 memcpy(paddr + part, buffer, bytes - part); 99 } 100 kunmap_local(buffer); 101 } 102 } 103 return (int)bytes; 104 } 105 106 #define PKT_FRAGMENTED 1 107 #define PKT_COMPLETE 0 108 109 /* 110 * siw_qp_prepare_tx() 111 * 112 * Prepare tx state for sending out one fpdu. Builds complete pkt 113 * if no user data or only immediate data are present. 114 * 115 * returns PKT_COMPLETE if complete pkt built, PKT_FRAGMENTED otherwise. 116 */ 117 static int siw_qp_prepare_tx(struct siw_iwarp_tx *c_tx) 118 { 119 struct siw_wqe *wqe = &c_tx->wqe_active; 120 char *crc = NULL; 121 int data = 0; 122 123 switch (tx_type(wqe)) { 124 case SIW_OP_READ: 125 case SIW_OP_READ_LOCAL_INV: 126 memcpy(&c_tx->pkt.ctrl, 127 &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl, 128 sizeof(struct iwarp_ctrl)); 129 130 c_tx->pkt.rreq.rsvd = 0; 131 c_tx->pkt.rreq.ddp_qn = htonl(RDMAP_UNTAGGED_QN_RDMA_READ); 132 c_tx->pkt.rreq.ddp_msn = 133 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ]); 134 c_tx->pkt.rreq.ddp_mo = 0; 135 c_tx->pkt.rreq.sink_stag = htonl(wqe->sqe.sge[0].lkey); 136 c_tx->pkt.rreq.sink_to = 137 cpu_to_be64(wqe->sqe.sge[0].laddr); 138 c_tx->pkt.rreq.source_stag = htonl(wqe->sqe.rkey); 139 c_tx->pkt.rreq.source_to = cpu_to_be64(wqe->sqe.raddr); 140 c_tx->pkt.rreq.read_size = htonl(wqe->sqe.sge[0].length); 141 142 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rreq); 143 crc = (char *)&c_tx->pkt.rreq_pkt.crc; 144 break; 145 146 case SIW_OP_SEND: 147 if (tx_flags(wqe) & SIW_WQE_SOLICITED) 148 memcpy(&c_tx->pkt.ctrl, 149 &iwarp_pktinfo[RDMAP_SEND_SE].ctrl, 150 sizeof(struct iwarp_ctrl)); 151 else 152 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_SEND].ctrl, 153 sizeof(struct iwarp_ctrl)); 154 155 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND; 156 c_tx->pkt.send.ddp_msn = 157 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); 158 c_tx->pkt.send.ddp_mo = 0; 159 160 c_tx->pkt.send_inv.inval_stag = 0; 161 162 c_tx->ctrl_len = sizeof(struct iwarp_send); 163 164 crc = (char *)&c_tx->pkt.send_pkt.crc; 165 data = siw_try_1seg(c_tx, crc); 166 break; 167 168 case SIW_OP_SEND_REMOTE_INV: 169 if (tx_flags(wqe) & SIW_WQE_SOLICITED) 170 memcpy(&c_tx->pkt.ctrl, 171 &iwarp_pktinfo[RDMAP_SEND_SE_INVAL].ctrl, 172 sizeof(struct iwarp_ctrl)); 173 else 174 memcpy(&c_tx->pkt.ctrl, 175 &iwarp_pktinfo[RDMAP_SEND_INVAL].ctrl, 176 sizeof(struct iwarp_ctrl)); 177 178 c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND; 179 c_tx->pkt.send.ddp_msn = 180 htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]); 181 c_tx->pkt.send.ddp_mo = 0; 182 183 c_tx->pkt.send_inv.inval_stag = cpu_to_be32(wqe->sqe.rkey); 184 185 c_tx->ctrl_len = sizeof(struct iwarp_send_inv); 186 187 crc = (char *)&c_tx->pkt.send_pkt.crc; 188 data = siw_try_1seg(c_tx, crc); 189 break; 190 191 case SIW_OP_WRITE: 192 memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_RDMA_WRITE].ctrl, 193 sizeof(struct iwarp_ctrl)); 194 195 c_tx->pkt.rwrite.sink_stag = htonl(wqe->sqe.rkey); 196 c_tx->pkt.rwrite.sink_to = cpu_to_be64(wqe->sqe.raddr); 197 c_tx->ctrl_len = sizeof(struct iwarp_rdma_write); 198 199 crc = (char *)&c_tx->pkt.write_pkt.crc; 200 data = siw_try_1seg(c_tx, crc); 201 break; 202 203 case SIW_OP_READ_RESPONSE: 204 memcpy(&c_tx->pkt.ctrl, 205 &iwarp_pktinfo[RDMAP_RDMA_READ_RESP].ctrl, 206 sizeof(struct iwarp_ctrl)); 207 208 /* NBO */ 209 c_tx->pkt.rresp.sink_stag = cpu_to_be32(wqe->sqe.rkey); 210 c_tx->pkt.rresp.sink_to = cpu_to_be64(wqe->sqe.raddr); 211 212 c_tx->ctrl_len = sizeof(struct iwarp_rdma_rresp); 213 214 crc = (char *)&c_tx->pkt.write_pkt.crc; 215 data = siw_try_1seg(c_tx, crc); 216 break; 217 218 default: 219 siw_dbg_qp(tx_qp(c_tx), "stale wqe type %d\n", tx_type(wqe)); 220 return -EOPNOTSUPP; 221 } 222 if (unlikely(data < 0)) 223 return data; 224 225 c_tx->ctrl_sent = 0; 226 227 if (data <= MAX_HDR_INLINE) { 228 if (data) { 229 wqe->processed = data; 230 231 c_tx->pkt.ctrl.mpa_len = 232 htons(c_tx->ctrl_len + data - MPA_HDR_SIZE); 233 234 /* Add pad, if needed */ 235 data += -(int)data & 0x3; 236 /* advance CRC location after payload */ 237 crc += data; 238 c_tx->ctrl_len += data; 239 240 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)) 241 c_tx->pkt.c_untagged.ddp_mo = 0; 242 else 243 c_tx->pkt.c_tagged.ddp_to = 244 cpu_to_be64(wqe->sqe.raddr); 245 } 246 247 *(u32 *)crc = 0; 248 /* 249 * Do complete CRC if enabled and short packet 250 */ 251 if (c_tx->mpa_crc_enabled) 252 siw_crc_oneshot(&c_tx->pkt, c_tx->ctrl_len, (u8 *)crc); 253 c_tx->ctrl_len += MPA_CRC_SIZE; 254 255 return PKT_COMPLETE; 256 } 257 c_tx->ctrl_len += MPA_CRC_SIZE; 258 c_tx->sge_idx = 0; 259 c_tx->sge_off = 0; 260 c_tx->pbl_idx = 0; 261 262 /* 263 * Allow direct sending out of user buffer if WR is non signalled 264 * and payload is over threshold. 265 * Per RDMA verbs, the application should not change the send buffer 266 * until the work completed. In iWarp, work completion is only 267 * local delivery to TCP. TCP may reuse the buffer for 268 * retransmission. Changing unsent data also breaks the CRC, 269 * if applied. 270 */ 271 if (c_tx->zcopy_tx && wqe->bytes >= SENDPAGE_THRESH && 272 !(tx_flags(wqe) & SIW_WQE_SIGNALLED)) 273 c_tx->use_sendpage = 1; 274 else 275 c_tx->use_sendpage = 0; 276 277 return PKT_FRAGMENTED; 278 } 279 280 static noinline_for_stack int 281 siw_sendmsg(struct socket *sock, unsigned int msg_flags, 282 struct kvec *vec, size_t num, size_t len) 283 { 284 struct msghdr msg = { .msg_flags = msg_flags }; 285 286 return kernel_sendmsg(sock, &msg, vec, num, len); 287 } 288 289 /* 290 * Send out one complete control type FPDU, or header of FPDU carrying 291 * data. Used for fixed sized packets like Read.Requests or zero length 292 * SENDs, WRITEs, READ.Responses, or header only. 293 */ 294 static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s, 295 int flags) 296 { 297 struct kvec iov = { .iov_base = 298 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent, 299 .iov_len = c_tx->ctrl_len - c_tx->ctrl_sent }; 300 301 int rv = siw_sendmsg(s, flags, &iov, 1, iov.iov_len); 302 303 if (rv >= 0) { 304 c_tx->ctrl_sent += rv; 305 306 if (c_tx->ctrl_sent == c_tx->ctrl_len) 307 rv = 0; 308 else 309 rv = -EAGAIN; 310 } 311 return rv; 312 } 313 314 /* 315 * 0copy TCP transmit interface: Use MSG_SPLICE_PAGES. 316 * 317 * Using sendpage to push page by page appears to be less efficient 318 * than using sendmsg, even if data are copied. 319 * 320 * A general performance limitation might be the extra four bytes 321 * trailer checksum segment to be pushed after user data. 322 */ 323 static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset, 324 size_t size) 325 { 326 struct bio_vec bvec; 327 struct msghdr msg = { 328 .msg_flags = (MSG_MORE | MSG_DONTWAIT | MSG_SPLICE_PAGES), 329 }; 330 struct sock *sk = s->sk; 331 int i = 0, rv = 0, sent = 0; 332 333 while (size) { 334 size_t bytes = min_t(size_t, PAGE_SIZE - offset, size); 335 336 if (size + offset <= PAGE_SIZE) 337 msg.msg_flags &= ~MSG_MORE; 338 339 tcp_rate_check_app_limited(sk); 340 if (!sendpage_ok(page[i])) 341 msg.msg_flags &= ~MSG_SPLICE_PAGES; 342 bvec_set_page(&bvec, page[i], bytes, offset); 343 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, bytes); 344 345 try_page_again: 346 lock_sock(sk); 347 rv = tcp_sendmsg_locked(sk, &msg, bytes); 348 release_sock(sk); 349 350 if (rv > 0) { 351 size -= rv; 352 sent += rv; 353 if (rv != bytes) { 354 bytes -= rv; 355 goto try_page_again; 356 } 357 offset = 0; 358 } else { 359 if (rv == -EAGAIN || rv == 0) 360 break; 361 return rv; 362 } 363 i++; 364 } 365 return sent; 366 } 367 368 /* 369 * siw_0copy_tx() 370 * 371 * Pushes list of pages to TCP socket. If pages from multiple 372 * SGE's, all referenced pages of each SGE are pushed in one 373 * shot. 374 */ 375 static int siw_0copy_tx(struct socket *s, struct page **page, 376 struct siw_sge *sge, unsigned int offset, 377 unsigned int size) 378 { 379 int i = 0, sent = 0, rv; 380 int sge_bytes = min(sge->length - offset, size); 381 382 offset = (sge->laddr + offset) & ~PAGE_MASK; 383 384 while (sent != size) { 385 rv = siw_tcp_sendpages(s, &page[i], offset, sge_bytes); 386 if (rv >= 0) { 387 sent += rv; 388 if (size == sent || sge_bytes > rv) 389 break; 390 391 i += PAGE_ALIGN(sge_bytes + offset) >> PAGE_SHIFT; 392 sge++; 393 sge_bytes = min(sge->length, size - sent); 394 offset = sge->laddr & ~PAGE_MASK; 395 } else { 396 sent = rv; 397 break; 398 } 399 } 400 return sent; 401 } 402 403 #define MAX_TRAILER (MPA_CRC_SIZE + 4) 404 405 static void siw_unmap_pages(struct kvec *iov, unsigned long kmap_mask, int len) 406 { 407 int i; 408 409 /* 410 * Work backwards through the array to honor the kmap_local_page() 411 * ordering requirements. 412 */ 413 for (i = (len-1); i >= 0; i--) { 414 if (kmap_mask & BIT(i)) { 415 unsigned long addr = (unsigned long)iov[i].iov_base; 416 417 kunmap_local((void *)(addr & PAGE_MASK)); 418 } 419 } 420 } 421 422 /* 423 * siw_tx_hdt() tries to push a complete packet to TCP where all 424 * packet fragments are referenced by the elements of one iovec. 425 * For the data portion, each involved page must be referenced by 426 * one extra element. All sge's data can be non-aligned to page 427 * boundaries. Two more elements are referencing iWARP header 428 * and trailer: 429 * MAX_ARRAY = 64KB/PAGE_SIZE + 1 + (2 * (SIW_MAX_SGE - 1) + HDR + TRL 430 */ 431 #define MAX_ARRAY ((0xffff / PAGE_SIZE) + 1 + (2 * (SIW_MAX_SGE - 1) + 2)) 432 433 /* 434 * Write out iov referencing hdr, data and trailer of current FPDU. 435 * Update transmit state dependent on write return status 436 */ 437 static noinline_for_stack int siw_tx_hdt(struct siw_iwarp_tx *c_tx, 438 struct socket *s) 439 { 440 struct siw_wqe *wqe = &c_tx->wqe_active; 441 struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx]; 442 struct kvec iov[MAX_ARRAY]; 443 struct page *page_array[MAX_ARRAY]; 444 445 int seg = 0, do_crc = c_tx->do_crc, is_kva = 0, rv; 446 unsigned int data_len = c_tx->bytes_unsent, hdr_len = 0, trl_len = 0, 447 sge_off = c_tx->sge_off, sge_idx = c_tx->sge_idx, 448 pbl_idx = c_tx->pbl_idx; 449 unsigned long kmap_mask = 0L; 450 451 if (c_tx->state == SIW_SEND_HDR) { 452 if (c_tx->use_sendpage) { 453 rv = siw_tx_ctrl(c_tx, s, MSG_DONTWAIT | MSG_MORE); 454 if (rv) 455 goto done; 456 457 c_tx->state = SIW_SEND_DATA; 458 } else { 459 iov[0].iov_base = 460 (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent; 461 iov[0].iov_len = hdr_len = 462 c_tx->ctrl_len - c_tx->ctrl_sent; 463 seg = 1; 464 } 465 } 466 467 wqe->processed += data_len; 468 469 while (data_len) { /* walk the list of SGE's */ 470 unsigned int sge_len = min(sge->length - sge_off, data_len); 471 unsigned int fp_off = (sge->laddr + sge_off) & ~PAGE_MASK; 472 struct siw_mem *mem; 473 474 if (!(tx_flags(wqe) & SIW_WQE_INLINE)) { 475 mem = wqe->mem[sge_idx]; 476 is_kva = mem->mem_obj == NULL ? 1 : 0; 477 } else { 478 is_kva = 1; 479 } 480 if (is_kva && !c_tx->use_sendpage) { 481 /* 482 * tx from kernel virtual address: either inline data 483 * or memory region with assigned kernel buffer 484 */ 485 iov[seg].iov_base = 486 ib_virt_dma_to_ptr(sge->laddr + sge_off); 487 iov[seg].iov_len = sge_len; 488 489 if (do_crc) 490 siw_crc_update(&c_tx->mpa_crc, 491 iov[seg].iov_base, sge_len); 492 sge_off += sge_len; 493 data_len -= sge_len; 494 seg++; 495 goto sge_done; 496 } 497 498 while (sge_len) { 499 size_t plen = min((int)PAGE_SIZE - fp_off, sge_len); 500 void *kaddr; 501 502 if (!is_kva) { 503 struct page *p; 504 505 p = siw_get_page(mem, sge, sge_off, &pbl_idx); 506 if (unlikely(!p)) { 507 siw_unmap_pages(iov, kmap_mask, seg); 508 wqe->processed -= c_tx->bytes_unsent; 509 rv = -EFAULT; 510 goto done_crc; 511 } 512 page_array[seg] = p; 513 514 if (!c_tx->use_sendpage) { 515 void *kaddr = kmap_local_page(p); 516 517 /* Remember for later kunmap() */ 518 kmap_mask |= BIT(seg); 519 iov[seg].iov_base = kaddr + fp_off; 520 iov[seg].iov_len = plen; 521 522 if (do_crc) 523 siw_crc_update( 524 &c_tx->mpa_crc, 525 iov[seg].iov_base, 526 plen); 527 } else if (do_crc) { 528 kaddr = kmap_local_page(p); 529 siw_crc_update(&c_tx->mpa_crc, 530 kaddr + fp_off, plen); 531 kunmap_local(kaddr); 532 } 533 } else { 534 /* 535 * Cast to an uintptr_t to preserve all 64 bits 536 * in sge->laddr. 537 */ 538 u64 va = sge->laddr + sge_off; 539 540 page_array[seg] = ib_virt_dma_to_page(va); 541 if (do_crc) 542 siw_crc_update(&c_tx->mpa_crc, 543 ib_virt_dma_to_ptr(va), 544 plen); 545 } 546 547 sge_len -= plen; 548 sge_off += plen; 549 data_len -= plen; 550 fp_off = 0; 551 552 if (++seg >= (int)MAX_ARRAY) { 553 siw_dbg_qp(tx_qp(c_tx), "to many fragments\n"); 554 siw_unmap_pages(iov, kmap_mask, seg-1); 555 wqe->processed -= c_tx->bytes_unsent; 556 rv = -EMSGSIZE; 557 goto done_crc; 558 } 559 } 560 sge_done: 561 /* Update SGE variables at end of SGE */ 562 if (sge_off == sge->length && 563 (data_len != 0 || wqe->processed < wqe->bytes)) { 564 sge_idx++; 565 sge++; 566 sge_off = 0; 567 } 568 } 569 /* trailer */ 570 if (likely(c_tx->state != SIW_SEND_TRAILER)) { 571 iov[seg].iov_base = &c_tx->trailer.pad[4 - c_tx->pad]; 572 iov[seg].iov_len = trl_len = MAX_TRAILER - (4 - c_tx->pad); 573 } else { 574 iov[seg].iov_base = &c_tx->trailer.pad[c_tx->ctrl_sent]; 575 iov[seg].iov_len = trl_len = MAX_TRAILER - c_tx->ctrl_sent; 576 } 577 578 if (c_tx->pad) { 579 *(u32 *)c_tx->trailer.pad = 0; 580 if (do_crc) 581 siw_crc_update(&c_tx->mpa_crc, 582 (u8 *)&c_tx->trailer.crc - c_tx->pad, 583 c_tx->pad); 584 } 585 if (!c_tx->mpa_crc_enabled) 586 c_tx->trailer.crc = 0; 587 else if (do_crc) 588 siw_crc_final(&c_tx->mpa_crc, (u8 *)&c_tx->trailer.crc); 589 590 data_len = c_tx->bytes_unsent; 591 592 if (c_tx->use_sendpage) { 593 rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx], 594 c_tx->sge_off, data_len); 595 if (rv == data_len) { 596 597 rv = siw_sendmsg(s, MSG_DONTWAIT | MSG_EOR, &iov[seg], 598 1, trl_len); 599 if (rv > 0) 600 rv += data_len; 601 else 602 rv = data_len; 603 } 604 } else { 605 rv = siw_sendmsg(s, MSG_DONTWAIT | MSG_EOR, iov, seg + 1, 606 hdr_len + data_len + trl_len); 607 siw_unmap_pages(iov, kmap_mask, seg); 608 } 609 if (rv < (int)hdr_len) { 610 /* Not even complete hdr pushed or negative rv */ 611 wqe->processed -= data_len; 612 if (rv >= 0) { 613 c_tx->ctrl_sent += rv; 614 rv = -EAGAIN; 615 } 616 goto done_crc; 617 } 618 rv -= hdr_len; 619 620 if (rv >= (int)data_len) { 621 /* all user data pushed to TCP or no data to push */ 622 if (data_len > 0 && wqe->processed < wqe->bytes) { 623 /* Save the current state for next tx */ 624 c_tx->sge_idx = sge_idx; 625 c_tx->sge_off = sge_off; 626 c_tx->pbl_idx = pbl_idx; 627 } 628 rv -= data_len; 629 630 if (rv == trl_len) /* all pushed */ 631 rv = 0; 632 else { 633 c_tx->state = SIW_SEND_TRAILER; 634 c_tx->ctrl_len = MAX_TRAILER; 635 c_tx->ctrl_sent = rv + 4 - c_tx->pad; 636 c_tx->bytes_unsent = 0; 637 rv = -EAGAIN; 638 } 639 640 } else if (data_len > 0) { 641 /* Maybe some user data pushed to TCP */ 642 c_tx->state = SIW_SEND_DATA; 643 wqe->processed -= data_len - rv; 644 645 if (rv) { 646 /* 647 * Some bytes out. Recompute tx state based 648 * on old state and bytes pushed 649 */ 650 unsigned int sge_unsent; 651 652 c_tx->bytes_unsent -= rv; 653 sge = &wqe->sqe.sge[c_tx->sge_idx]; 654 sge_unsent = sge->length - c_tx->sge_off; 655 656 while (sge_unsent <= rv) { 657 rv -= sge_unsent; 658 c_tx->sge_idx++; 659 c_tx->sge_off = 0; 660 sge++; 661 sge_unsent = sge->length; 662 } 663 c_tx->sge_off += rv; 664 } 665 rv = -EAGAIN; 666 } 667 done_crc: 668 c_tx->do_crc = 0; 669 done: 670 return rv; 671 } 672 673 static void siw_update_tcpseg(struct siw_iwarp_tx *c_tx, 674 struct socket *s) 675 { 676 struct tcp_sock *tp = tcp_sk(s->sk); 677 678 if (tp->gso_segs) { 679 if (c_tx->gso_seg_limit == 0) 680 c_tx->tcp_seglen = tp->mss_cache * tp->gso_segs; 681 else 682 c_tx->tcp_seglen = 683 tp->mss_cache * 684 min_t(u16, c_tx->gso_seg_limit, tp->gso_segs); 685 } else { 686 c_tx->tcp_seglen = tp->mss_cache; 687 } 688 /* Loopback may give odd numbers */ 689 c_tx->tcp_seglen &= 0xfffffff8; 690 } 691 692 /* 693 * siw_prepare_fpdu() 694 * 695 * Prepares transmit context to send out one FPDU if FPDU will contain 696 * user data and user data are not immediate data. 697 * Computes maximum FPDU length to fill up TCP MSS if possible. 698 * 699 * @qp: QP from which to transmit 700 * @wqe: Current WQE causing transmission 701 * 702 * TODO: Take into account real available sendspace on socket 703 * to avoid header misalignment due to send pausing within 704 * fpdu transmission 705 */ 706 static void siw_prepare_fpdu(struct siw_qp *qp, struct siw_wqe *wqe) 707 { 708 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; 709 int data_len; 710 711 c_tx->ctrl_len = 712 iwarp_pktinfo[__rdmap_get_opcode(&c_tx->pkt.ctrl)].hdr_len; 713 c_tx->ctrl_sent = 0; 714 715 /* 716 * Update target buffer offset if any 717 */ 718 if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)) 719 /* Untagged message */ 720 c_tx->pkt.c_untagged.ddp_mo = cpu_to_be32(wqe->processed); 721 else /* Tagged message */ 722 c_tx->pkt.c_tagged.ddp_to = 723 cpu_to_be64(wqe->sqe.raddr + wqe->processed); 724 725 data_len = wqe->bytes - wqe->processed; 726 if (data_len + c_tx->ctrl_len + MPA_CRC_SIZE > c_tx->tcp_seglen) { 727 /* Trim DDP payload to fit into current TCP segment */ 728 data_len = c_tx->tcp_seglen - (c_tx->ctrl_len + MPA_CRC_SIZE); 729 c_tx->pkt.ctrl.ddp_rdmap_ctrl &= ~DDP_FLAG_LAST; 730 c_tx->pad = 0; 731 } else { 732 c_tx->pkt.ctrl.ddp_rdmap_ctrl |= DDP_FLAG_LAST; 733 c_tx->pad = -data_len & 0x3; 734 } 735 c_tx->bytes_unsent = data_len; 736 737 c_tx->pkt.ctrl.mpa_len = 738 htons(c_tx->ctrl_len + data_len - MPA_HDR_SIZE); 739 740 /* 741 * Init MPA CRC computation 742 */ 743 if (c_tx->mpa_crc_enabled) { 744 siw_crc_init(&c_tx->mpa_crc); 745 siw_crc_update(&c_tx->mpa_crc, &c_tx->pkt, c_tx->ctrl_len); 746 c_tx->do_crc = 1; 747 } 748 } 749 750 /* 751 * siw_check_sgl_tx() 752 * 753 * Check permissions for a list of SGE's (SGL). 754 * A successful check will have all memory referenced 755 * for transmission resolved and assigned to the WQE. 756 * 757 * @pd: Protection Domain SGL should belong to 758 * @wqe: WQE to be checked 759 * @perms: requested access permissions 760 * 761 */ 762 763 static int siw_check_sgl_tx(struct ib_pd *pd, struct siw_wqe *wqe, 764 enum ib_access_flags perms) 765 { 766 struct siw_sge *sge = &wqe->sqe.sge[0]; 767 int i, len, num_sge = wqe->sqe.num_sge; 768 769 if (unlikely(num_sge > SIW_MAX_SGE)) 770 return -EINVAL; 771 772 for (i = 0, len = 0; num_sge; num_sge--, i++, sge++) { 773 /* 774 * rdma verbs: do not check stag for a zero length sge 775 */ 776 if (sge->length) { 777 int rv = siw_check_sge(pd, sge, &wqe->mem[i], perms, 0, 778 sge->length); 779 780 if (unlikely(rv != E_ACCESS_OK)) 781 return rv; 782 } 783 len += sge->length; 784 } 785 return len; 786 } 787 788 /* 789 * siw_qp_sq_proc_tx() 790 * 791 * Process one WQE which needs transmission on the wire. 792 */ 793 static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe) 794 { 795 struct siw_iwarp_tx *c_tx = &qp->tx_ctx; 796 struct socket *s = qp->attrs.sk; 797 int rv = 0, burst_len = qp->tx_ctx.burst; 798 enum rdmap_ecode ecode = RDMAP_ECODE_CATASTROPHIC_STREAM; 799 800 if (unlikely(wqe->wr_status == SIW_WR_IDLE)) 801 return 0; 802 803 if (!burst_len) 804 burst_len = SQ_USER_MAXBURST; 805 806 if (wqe->wr_status == SIW_WR_QUEUED) { 807 if (!(wqe->sqe.flags & SIW_WQE_INLINE)) { 808 if (tx_type(wqe) == SIW_OP_READ_RESPONSE) 809 wqe->sqe.num_sge = 1; 810 811 if (tx_type(wqe) != SIW_OP_READ && 812 tx_type(wqe) != SIW_OP_READ_LOCAL_INV) { 813 /* 814 * Reference memory to be tx'd w/o checking 815 * access for LOCAL_READ permission, since 816 * not defined in RDMA core. 817 */ 818 rv = siw_check_sgl_tx(qp->pd, wqe, 0); 819 if (rv < 0) { 820 if (tx_type(wqe) == 821 SIW_OP_READ_RESPONSE) 822 ecode = siw_rdmap_error(-rv); 823 rv = -EINVAL; 824 goto tx_error; 825 } 826 wqe->bytes = rv; 827 } else { 828 wqe->bytes = 0; 829 } 830 } else { 831 wqe->bytes = wqe->sqe.sge[0].length; 832 if (!rdma_is_kernel_res(&qp->base_qp.res)) { 833 if (wqe->bytes > SIW_MAX_INLINE) { 834 rv = -EINVAL; 835 goto tx_error; 836 } 837 wqe->sqe.sge[0].laddr = 838 (u64)(uintptr_t)&wqe->sqe.sge[1]; 839 } 840 } 841 wqe->wr_status = SIW_WR_INPROGRESS; 842 wqe->processed = 0; 843 844 siw_update_tcpseg(c_tx, s); 845 846 rv = siw_qp_prepare_tx(c_tx); 847 if (rv == PKT_FRAGMENTED) { 848 c_tx->state = SIW_SEND_HDR; 849 siw_prepare_fpdu(qp, wqe); 850 } else if (rv == PKT_COMPLETE) { 851 c_tx->state = SIW_SEND_SHORT_FPDU; 852 } else { 853 goto tx_error; 854 } 855 } 856 857 next_segment: 858 siw_dbg_qp(qp, "wr type %d, state %d, data %u, sent %u, id %llx\n", 859 tx_type(wqe), wqe->wr_status, wqe->bytes, wqe->processed, 860 wqe->sqe.id); 861 862 if (--burst_len == 0) { 863 rv = -EINPROGRESS; 864 goto tx_done; 865 } 866 if (c_tx->state == SIW_SEND_SHORT_FPDU) { 867 enum siw_opcode tx_type = tx_type(wqe); 868 unsigned int msg_flags; 869 870 if (siw_sq_empty(qp) || !siw_tcp_nagle || burst_len == 1) 871 /* 872 * End current TCP segment, if SQ runs empty, 873 * or siw_tcp_nagle is not set, or we bail out 874 * soon due to no burst credit left. 875 */ 876 msg_flags = MSG_DONTWAIT; 877 else 878 msg_flags = MSG_DONTWAIT | MSG_MORE; 879 880 rv = siw_tx_ctrl(c_tx, s, msg_flags); 881 882 if (!rv && tx_type != SIW_OP_READ && 883 tx_type != SIW_OP_READ_LOCAL_INV) 884 wqe->processed = wqe->bytes; 885 886 goto tx_done; 887 888 } else { 889 rv = siw_tx_hdt(c_tx, s); 890 } 891 if (!rv) { 892 /* 893 * One segment sent. Processing completed if last 894 * segment, Do next segment otherwise. 895 */ 896 if (unlikely(c_tx->tx_suspend)) { 897 /* 898 * Verbs, 6.4.: Try stopping sending after a full 899 * DDP segment if the connection goes down 900 * (== peer halfclose) 901 */ 902 rv = -ECONNABORTED; 903 goto tx_done; 904 } 905 if (c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_LAST) { 906 siw_dbg_qp(qp, "WQE completed\n"); 907 goto tx_done; 908 } 909 c_tx->state = SIW_SEND_HDR; 910 911 siw_update_tcpseg(c_tx, s); 912 913 siw_prepare_fpdu(qp, wqe); 914 goto next_segment; 915 } 916 tx_done: 917 qp->tx_ctx.burst = burst_len; 918 return rv; 919 920 tx_error: 921 if (ecode != RDMAP_ECODE_CATASTROPHIC_STREAM) 922 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, 923 RDMAP_ETYPE_REMOTE_PROTECTION, ecode, 1); 924 else 925 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, 926 RDMAP_ETYPE_CATASTROPHIC, 927 RDMAP_ECODE_UNSPECIFIED, 1); 928 return rv; 929 } 930 931 static int siw_fastreg_mr(struct ib_pd *pd, struct siw_sqe *sqe) 932 { 933 struct ib_mr *base_mr = (struct ib_mr *)(uintptr_t)sqe->base_mr; 934 struct siw_device *sdev = to_siw_dev(pd->device); 935 struct siw_mem *mem; 936 int rv = 0; 937 938 siw_dbg_pd(pd, "STag 0x%08x\n", sqe->rkey); 939 940 if (unlikely(!base_mr)) { 941 pr_warn("siw: fastreg: STag 0x%08x unknown\n", sqe->rkey); 942 return -EINVAL; 943 } 944 945 if (unlikely(base_mr->rkey >> 8 != sqe->rkey >> 8)) { 946 pr_warn("siw: fastreg: STag 0x%08x: bad MR\n", sqe->rkey); 947 return -EINVAL; 948 } 949 950 mem = siw_mem_id2obj(sdev, sqe->rkey >> 8); 951 if (unlikely(!mem)) { 952 pr_warn("siw: fastreg: STag 0x%08x unknown\n", sqe->rkey); 953 return -EINVAL; 954 } 955 956 if (unlikely(mem->pd != pd)) { 957 pr_warn("siw: fastreg: PD mismatch\n"); 958 rv = -EINVAL; 959 goto out; 960 } 961 if (unlikely(mem->stag_valid)) { 962 pr_warn("siw: fastreg: STag 0x%08x already valid\n", sqe->rkey); 963 rv = -EINVAL; 964 goto out; 965 } 966 /* Refresh STag since user may have changed key part */ 967 mem->stag = sqe->rkey; 968 mem->perms = sqe->access; 969 970 siw_dbg_mem(mem, "STag 0x%08x now valid\n", sqe->rkey); 971 mem->va = base_mr->iova; 972 mem->stag_valid = 1; 973 out: 974 siw_mem_put(mem); 975 return rv; 976 } 977 978 static int siw_qp_sq_proc_local(struct siw_qp *qp, struct siw_wqe *wqe) 979 { 980 int rv; 981 982 switch (tx_type(wqe)) { 983 case SIW_OP_REG_MR: 984 rv = siw_fastreg_mr(qp->pd, &wqe->sqe); 985 break; 986 987 case SIW_OP_INVAL_STAG: 988 rv = siw_invalidate_stag(qp->pd, wqe->sqe.rkey); 989 break; 990 991 default: 992 rv = -EINVAL; 993 } 994 return rv; 995 } 996 997 /* 998 * siw_qp_sq_process() 999 * 1000 * Core TX path routine for RDMAP/DDP/MPA using a TCP kernel socket. 1001 * Sends RDMAP payload for the current SQ WR @wqe of @qp in one or more 1002 * MPA FPDUs, each containing a DDP segment. 1003 * 1004 * SQ processing may occur in user context as a result of posting 1005 * new WQE's or from siw_tx_thread context. Processing in 1006 * user context is limited to non-kernel verbs users. 1007 * 1008 * SQ processing may get paused anytime, possibly in the middle of a WR 1009 * or FPDU, if insufficient send space is available. SQ processing 1010 * gets resumed from siw_tx_thread, if send space becomes available again. 1011 * 1012 * Must be called with the QP state read-locked. 1013 * 1014 * Note: 1015 * An outbound RREQ can be satisfied by the corresponding RRESP 1016 * _before_ it gets assigned to the ORQ. This happens regularly 1017 * in RDMA READ via loopback case. Since both outbound RREQ and 1018 * inbound RRESP can be handled by the same CPU, locking the ORQ 1019 * is dead-lock prone and thus not an option. With that, the 1020 * RREQ gets assigned to the ORQ _before_ being sent - see 1021 * siw_activate_tx() - and pulled back in case of send failure. 1022 */ 1023 int siw_qp_sq_process(struct siw_qp *qp) 1024 { 1025 struct siw_wqe *wqe = tx_wqe(qp); 1026 enum siw_opcode tx_type; 1027 unsigned long flags; 1028 int rv = 0; 1029 1030 siw_dbg_qp(qp, "enter for type %d\n", tx_type(wqe)); 1031 1032 next_wqe: 1033 /* 1034 * Stop QP processing if SQ state changed 1035 */ 1036 if (unlikely(qp->tx_ctx.tx_suspend)) { 1037 siw_dbg_qp(qp, "tx suspended\n"); 1038 goto done; 1039 } 1040 tx_type = tx_type(wqe); 1041 1042 if (tx_type <= SIW_OP_READ_RESPONSE) 1043 rv = siw_qp_sq_proc_tx(qp, wqe); 1044 else 1045 rv = siw_qp_sq_proc_local(qp, wqe); 1046 1047 if (!rv) { 1048 /* 1049 * WQE processing done 1050 */ 1051 switch (tx_type) { 1052 case SIW_OP_SEND: 1053 case SIW_OP_SEND_REMOTE_INV: 1054 case SIW_OP_WRITE: 1055 siw_wqe_put_mem(wqe, tx_type); 1056 fallthrough; 1057 1058 case SIW_OP_INVAL_STAG: 1059 case SIW_OP_REG_MR: 1060 if (tx_flags(wqe) & SIW_WQE_SIGNALLED) 1061 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, 1062 SIW_WC_SUCCESS); 1063 break; 1064 1065 case SIW_OP_READ: 1066 case SIW_OP_READ_LOCAL_INV: 1067 /* 1068 * already enqueued to ORQ queue 1069 */ 1070 break; 1071 1072 case SIW_OP_READ_RESPONSE: 1073 siw_wqe_put_mem(wqe, tx_type); 1074 break; 1075 1076 default: 1077 WARN(1, "undefined WQE type %d\n", tx_type); 1078 rv = -EINVAL; 1079 goto done; 1080 } 1081 1082 spin_lock_irqsave(&qp->sq_lock, flags); 1083 wqe->wr_status = SIW_WR_IDLE; 1084 rv = siw_activate_tx(qp); 1085 spin_unlock_irqrestore(&qp->sq_lock, flags); 1086 1087 if (rv <= 0) 1088 goto done; 1089 1090 goto next_wqe; 1091 1092 } else if (rv == -EAGAIN) { 1093 siw_dbg_qp(qp, "sq paused: hd/tr %d of %d, data %d\n", 1094 qp->tx_ctx.ctrl_sent, qp->tx_ctx.ctrl_len, 1095 qp->tx_ctx.bytes_unsent); 1096 rv = 0; 1097 goto done; 1098 } else if (rv == -EINPROGRESS) { 1099 rv = siw_sq_start(qp); 1100 goto done; 1101 } else { 1102 /* 1103 * WQE processing failed. 1104 * Verbs 8.3.2: 1105 * o It turns any WQE into a signalled WQE. 1106 * o Local catastrophic error must be surfaced 1107 * o QP must be moved into Terminate state: done by code 1108 * doing socket state change processing 1109 * 1110 * o TODO: Termination message must be sent. 1111 * o TODO: Implement more precise work completion errors, 1112 * see enum ib_wc_status in ib_verbs.h 1113 */ 1114 siw_dbg_qp(qp, "wqe type %d processing failed: %d\n", 1115 tx_type(wqe), rv); 1116 1117 spin_lock_irqsave(&qp->sq_lock, flags); 1118 /* 1119 * RREQ may have already been completed by inbound RRESP! 1120 */ 1121 if ((tx_type == SIW_OP_READ || 1122 tx_type == SIW_OP_READ_LOCAL_INV) && qp->attrs.orq_size) { 1123 /* Cleanup pending entry in ORQ */ 1124 qp->orq_put--; 1125 qp->orq[qp->orq_put % qp->attrs.orq_size].flags = 0; 1126 } 1127 spin_unlock_irqrestore(&qp->sq_lock, flags); 1128 /* 1129 * immediately suspends further TX processing 1130 */ 1131 if (!qp->tx_ctx.tx_suspend) 1132 siw_qp_cm_drop(qp, 0); 1133 1134 switch (tx_type) { 1135 case SIW_OP_SEND: 1136 case SIW_OP_SEND_REMOTE_INV: 1137 case SIW_OP_SEND_WITH_IMM: 1138 case SIW_OP_WRITE: 1139 case SIW_OP_READ: 1140 case SIW_OP_READ_LOCAL_INV: 1141 siw_wqe_put_mem(wqe, tx_type); 1142 fallthrough; 1143 1144 case SIW_OP_INVAL_STAG: 1145 case SIW_OP_REG_MR: 1146 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, 1147 SIW_WC_LOC_QP_OP_ERR); 1148 1149 siw_qp_event(qp, IB_EVENT_QP_FATAL); 1150 1151 break; 1152 1153 case SIW_OP_READ_RESPONSE: 1154 siw_dbg_qp(qp, "proc. read.response failed: %d\n", rv); 1155 1156 siw_qp_event(qp, IB_EVENT_QP_REQ_ERR); 1157 1158 siw_wqe_put_mem(wqe, SIW_OP_READ_RESPONSE); 1159 1160 break; 1161 1162 default: 1163 WARN(1, "undefined WQE type %d\n", tx_type); 1164 rv = -EINVAL; 1165 } 1166 wqe->wr_status = SIW_WR_IDLE; 1167 } 1168 done: 1169 return rv; 1170 } 1171 1172 static void siw_sq_resume(struct siw_qp *qp) 1173 { 1174 if (down_read_trylock(&qp->state_lock)) { 1175 if (likely(qp->attrs.state == SIW_QP_STATE_RTS && 1176 !qp->tx_ctx.tx_suspend)) { 1177 int rv = siw_qp_sq_process(qp); 1178 1179 up_read(&qp->state_lock); 1180 1181 if (unlikely(rv < 0)) { 1182 siw_dbg_qp(qp, "SQ task failed: err %d\n", rv); 1183 1184 if (!qp->tx_ctx.tx_suspend) 1185 siw_qp_cm_drop(qp, 0); 1186 } 1187 } else { 1188 up_read(&qp->state_lock); 1189 } 1190 } else { 1191 siw_dbg_qp(qp, "Resume SQ while QP locked\n"); 1192 } 1193 siw_qp_put(qp); 1194 } 1195 1196 struct tx_task_t { 1197 struct llist_head active; 1198 wait_queue_head_t waiting; 1199 }; 1200 1201 static DEFINE_PER_CPU(struct tx_task_t, siw_tx_task_g); 1202 1203 int siw_create_tx_threads(void) 1204 { 1205 int cpu, assigned = 0; 1206 1207 for_each_online_cpu(cpu) { 1208 struct tx_task_t *tx_task; 1209 1210 /* Skip HT cores */ 1211 if (cpu % cpumask_weight(topology_sibling_cpumask(cpu))) 1212 continue; 1213 1214 tx_task = &per_cpu(siw_tx_task_g, cpu); 1215 init_llist_head(&tx_task->active); 1216 init_waitqueue_head(&tx_task->waiting); 1217 1218 siw_tx_thread[cpu] = 1219 kthread_run_on_cpu(siw_run_sq, 1220 (unsigned long *)(long)cpu, 1221 cpu, "siw_tx/%u"); 1222 if (IS_ERR(siw_tx_thread[cpu])) { 1223 siw_tx_thread[cpu] = NULL; 1224 continue; 1225 } 1226 assigned++; 1227 } 1228 return assigned; 1229 } 1230 1231 void siw_stop_tx_threads(void) 1232 { 1233 int cpu; 1234 1235 for_each_possible_cpu(cpu) { 1236 if (siw_tx_thread[cpu]) { 1237 kthread_stop(siw_tx_thread[cpu]); 1238 wake_up(&per_cpu(siw_tx_task_g, cpu).waiting); 1239 siw_tx_thread[cpu] = NULL; 1240 } 1241 } 1242 } 1243 1244 int siw_run_sq(void *data) 1245 { 1246 const int nr_cpu = (unsigned int)(long)data; 1247 struct llist_node *active; 1248 struct siw_qp *qp; 1249 struct tx_task_t *tx_task = &per_cpu(siw_tx_task_g, nr_cpu); 1250 1251 while (1) { 1252 struct llist_node *fifo_list = NULL; 1253 1254 wait_event_interruptible(tx_task->waiting, 1255 !llist_empty(&tx_task->active) || 1256 kthread_should_stop()); 1257 1258 if (kthread_should_stop()) 1259 break; 1260 1261 active = llist_del_all(&tx_task->active); 1262 /* 1263 * llist_del_all returns a list with newest entry first. 1264 * Re-order list for fairness among QP's. 1265 */ 1266 fifo_list = llist_reverse_order(active); 1267 while (fifo_list) { 1268 qp = container_of(fifo_list, struct siw_qp, tx_list); 1269 fifo_list = llist_next(fifo_list); 1270 qp->tx_list.next = NULL; 1271 1272 siw_sq_resume(qp); 1273 } 1274 } 1275 active = llist_del_all(&tx_task->active); 1276 if (active) { 1277 llist_for_each_entry(qp, active, tx_list) { 1278 qp->tx_list.next = NULL; 1279 siw_sq_resume(qp); 1280 } 1281 } 1282 return 0; 1283 } 1284 1285 int siw_sq_start(struct siw_qp *qp) 1286 { 1287 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) 1288 return 0; 1289 1290 if (unlikely(!cpu_online(qp->tx_cpu))) { 1291 siw_put_tx_cpu(qp->tx_cpu); 1292 qp->tx_cpu = siw_get_tx_cpu(qp->sdev); 1293 if (qp->tx_cpu < 0) { 1294 pr_warn("siw: no tx cpu available\n"); 1295 1296 return -EIO; 1297 } 1298 } 1299 siw_qp_get(qp); 1300 1301 llist_add(&qp->tx_list, &per_cpu(siw_tx_task_g, qp->tx_cpu).active); 1302 1303 wake_up(&per_cpu(siw_tx_task_g, qp->tx_cpu).waiting); 1304 1305 return 0; 1306 } 1307