1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip6_checksum.h> 8 9 #include "ionic.h" 10 #include "ionic_lif.h" 11 #include "ionic_txrx.h" 12 13 static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs); 14 15 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 16 void *data, size_t len); 17 18 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 19 const skb_frag_t *frag, 20 size_t offset, size_t len); 21 22 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 23 struct ionic_desc_info *desc_info); 24 25 static void ionic_tx_clean(struct ionic_queue *q, 26 struct ionic_desc_info *desc_info, 27 struct ionic_cq_info *cq_info, 28 void *cb_arg); 29 30 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 31 ionic_desc_cb cb_func, void *cb_arg) 32 { 33 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 34 } 35 36 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, 37 ionic_desc_cb cb_func, void *cb_arg) 38 { 39 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 40 } 41 42 bool ionic_txq_poke_doorbell(struct ionic_queue *q) 43 { 44 unsigned long now, then, dif; 45 struct netdev_queue *netdev_txq; 46 struct net_device *netdev; 47 48 netdev = q->lif->netdev; 49 netdev_txq = netdev_get_tx_queue(netdev, q->index); 50 51 HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id()); 52 53 if (q->tail_idx == q->head_idx) { 54 HARD_TX_UNLOCK(netdev, netdev_txq); 55 return false; 56 } 57 58 now = READ_ONCE(jiffies); 59 then = q->dbell_jiffies; 60 dif = now - then; 61 62 if (dif > q->dbell_deadline) { 63 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 64 q->dbval | q->head_idx); 65 66 q->dbell_jiffies = now; 67 } 68 69 HARD_TX_UNLOCK(netdev, netdev_txq); 70 71 return true; 72 } 73 74 bool ionic_rxq_poke_doorbell(struct ionic_queue *q) 75 { 76 unsigned long now, then, dif; 77 78 /* no lock, called from rx napi or txrx napi, nothing else can fill */ 79 80 if (q->tail_idx == q->head_idx) 81 return false; 82 83 now = READ_ONCE(jiffies); 84 then = q->dbell_jiffies; 85 dif = now - then; 86 87 if (dif > q->dbell_deadline) { 88 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 89 q->dbval | q->head_idx); 90 91 q->dbell_jiffies = now; 92 93 dif = 2 * q->dbell_deadline; 94 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE) 95 dif = IONIC_RX_MAX_DOORBELL_DEADLINE; 96 97 q->dbell_deadline = dif; 98 } 99 100 return true; 101 } 102 103 static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q) 104 { 105 return netdev_get_tx_queue(q->lif->netdev, q->index); 106 } 107 108 static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info) 109 { 110 return page_address(buf_info->page) + buf_info->page_offset; 111 } 112 113 static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info) 114 { 115 return buf_info->dma_addr + buf_info->page_offset; 116 } 117 118 static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info) 119 { 120 return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset); 121 } 122 123 static int ionic_rx_page_alloc(struct ionic_queue *q, 124 struct ionic_buf_info *buf_info) 125 { 126 struct net_device *netdev = q->lif->netdev; 127 struct ionic_rx_stats *stats; 128 struct device *dev; 129 struct page *page; 130 131 dev = q->dev; 132 stats = q_to_rx_stats(q); 133 134 if (unlikely(!buf_info)) { 135 net_err_ratelimited("%s: %s invalid buf_info in alloc\n", 136 netdev->name, q->name); 137 return -EINVAL; 138 } 139 140 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 141 if (unlikely(!page)) { 142 net_err_ratelimited("%s: %s page alloc failed\n", 143 netdev->name, q->name); 144 stats->alloc_err++; 145 return -ENOMEM; 146 } 147 148 buf_info->dma_addr = dma_map_page(dev, page, 0, 149 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 150 if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) { 151 __free_pages(page, 0); 152 net_err_ratelimited("%s: %s dma map failed\n", 153 netdev->name, q->name); 154 stats->dma_map_err++; 155 return -EIO; 156 } 157 158 buf_info->page = page; 159 buf_info->page_offset = 0; 160 161 return 0; 162 } 163 164 static void ionic_rx_page_free(struct ionic_queue *q, 165 struct ionic_buf_info *buf_info) 166 { 167 struct net_device *netdev = q->lif->netdev; 168 struct device *dev = q->dev; 169 170 if (unlikely(!buf_info)) { 171 net_err_ratelimited("%s: %s invalid buf_info in free\n", 172 netdev->name, q->name); 173 return; 174 } 175 176 if (!buf_info->page) 177 return; 178 179 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 180 __free_pages(buf_info->page, 0); 181 buf_info->page = NULL; 182 } 183 184 static bool ionic_rx_buf_recycle(struct ionic_queue *q, 185 struct ionic_buf_info *buf_info, u32 used) 186 { 187 u32 size; 188 189 /* don't re-use pages allocated in low-mem condition */ 190 if (page_is_pfmemalloc(buf_info->page)) 191 return false; 192 193 /* don't re-use buffers from non-local numa nodes */ 194 if (page_to_nid(buf_info->page) != numa_mem_id()) 195 return false; 196 197 size = ALIGN(used, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ); 198 buf_info->page_offset += size; 199 if (buf_info->page_offset >= IONIC_PAGE_SIZE) 200 return false; 201 202 get_page(buf_info->page); 203 204 return true; 205 } 206 207 static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, 208 struct ionic_desc_info *desc_info, 209 unsigned int headroom, 210 unsigned int len, 211 unsigned int num_sg_elems, 212 bool synced) 213 { 214 struct net_device *netdev = q->lif->netdev; 215 struct ionic_buf_info *buf_info; 216 struct ionic_rx_stats *stats; 217 struct device *dev = q->dev; 218 struct sk_buff *skb; 219 unsigned int i; 220 u16 frag_len; 221 222 stats = q_to_rx_stats(q); 223 224 buf_info = &desc_info->bufs[0]; 225 226 prefetchw(buf_info->page); 227 228 skb = napi_get_frags(&q_to_qcq(q)->napi); 229 if (unlikely(!skb)) { 230 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 231 netdev->name, q->name); 232 stats->alloc_err++; 233 return NULL; 234 } 235 236 i = num_sg_elems + 1; 237 do { 238 if (unlikely(!buf_info->page)) { 239 dev_kfree_skb(skb); 240 return NULL; 241 } 242 243 if (headroom) 244 frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 245 else 246 frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 247 len -= frag_len; 248 249 if (!synced) 250 dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 251 headroom, frag_len, DMA_FROM_DEVICE); 252 253 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 254 buf_info->page, buf_info->page_offset + headroom, 255 frag_len, IONIC_PAGE_SIZE); 256 257 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { 258 dma_unmap_page(dev, buf_info->dma_addr, 259 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 260 buf_info->page = NULL; 261 } 262 263 /* only needed on the first buffer */ 264 if (headroom) 265 headroom = 0; 266 267 buf_info++; 268 269 i--; 270 } while (i > 0); 271 272 return skb; 273 } 274 275 static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, 276 struct ionic_desc_info *desc_info, 277 unsigned int headroom, 278 unsigned int len, 279 bool synced) 280 { 281 struct net_device *netdev = q->lif->netdev; 282 struct ionic_buf_info *buf_info; 283 struct ionic_rx_stats *stats; 284 struct device *dev = q->dev; 285 struct sk_buff *skb; 286 287 stats = q_to_rx_stats(q); 288 289 buf_info = &desc_info->bufs[0]; 290 291 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 292 if (unlikely(!skb)) { 293 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 294 netdev->name, q->name); 295 stats->alloc_err++; 296 return NULL; 297 } 298 299 if (unlikely(!buf_info->page)) { 300 dev_kfree_skb(skb); 301 return NULL; 302 } 303 304 if (!synced) 305 dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 306 headroom, len, DMA_FROM_DEVICE); 307 skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); 308 dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info), 309 headroom, len, DMA_FROM_DEVICE); 310 311 skb_put(skb, len); 312 skb->protocol = eth_type_trans(skb, q->lif->netdev); 313 314 return skb; 315 } 316 317 static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, 318 struct ionic_desc_info *desc_info) 319 { 320 unsigned int nbufs = desc_info->nbufs; 321 struct ionic_buf_info *buf_info; 322 struct device *dev = q->dev; 323 int i; 324 325 if (!nbufs) 326 return; 327 328 buf_info = desc_info->bufs; 329 dma_unmap_single(dev, buf_info->dma_addr, 330 buf_info->len, DMA_TO_DEVICE); 331 if (desc_info->act == XDP_TX) 332 __free_pages(buf_info->page, 0); 333 buf_info->page = NULL; 334 335 buf_info++; 336 for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) { 337 dma_unmap_page(dev, buf_info->dma_addr, 338 buf_info->len, DMA_TO_DEVICE); 339 if (desc_info->act == XDP_TX) 340 __free_pages(buf_info->page, 0); 341 buf_info->page = NULL; 342 } 343 344 if (desc_info->act == XDP_REDIRECT) 345 xdp_return_frame(desc_info->xdpf); 346 347 desc_info->nbufs = 0; 348 desc_info->xdpf = NULL; 349 desc_info->act = 0; 350 } 351 352 static int ionic_xdp_post_frame(struct net_device *netdev, 353 struct ionic_queue *q, struct xdp_frame *frame, 354 enum xdp_action act, struct page *page, int off, 355 bool ring_doorbell) 356 { 357 struct ionic_desc_info *desc_info; 358 struct ionic_buf_info *buf_info; 359 struct ionic_tx_stats *stats; 360 struct ionic_txq_desc *desc; 361 size_t len = frame->len; 362 dma_addr_t dma_addr; 363 u64 cmd; 364 365 desc_info = &q->info[q->head_idx]; 366 desc = desc_info->txq_desc; 367 buf_info = desc_info->bufs; 368 stats = q_to_tx_stats(q); 369 370 dma_addr = ionic_tx_map_single(q, frame->data, len); 371 if (dma_mapping_error(q->dev, dma_addr)) { 372 stats->dma_map_err++; 373 return -EIO; 374 } 375 buf_info->dma_addr = dma_addr; 376 buf_info->len = len; 377 buf_info->page = page; 378 buf_info->page_offset = off; 379 380 desc_info->nbufs = 1; 381 desc_info->xdpf = frame; 382 desc_info->act = act; 383 384 if (xdp_frame_has_frags(frame)) { 385 struct ionic_txq_sg_elem *elem; 386 struct skb_shared_info *sinfo; 387 struct ionic_buf_info *bi; 388 skb_frag_t *frag; 389 int i; 390 391 bi = &buf_info[1]; 392 sinfo = xdp_get_shared_info_from_frame(frame); 393 frag = sinfo->frags; 394 elem = desc_info->txq_sg_desc->elems; 395 for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) { 396 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 397 if (dma_mapping_error(q->dev, dma_addr)) { 398 stats->dma_map_err++; 399 ionic_tx_desc_unmap_bufs(q, desc_info); 400 return -EIO; 401 } 402 bi->dma_addr = dma_addr; 403 bi->len = skb_frag_size(frag); 404 bi->page = skb_frag_page(frag); 405 406 elem->addr = cpu_to_le64(bi->dma_addr); 407 elem->len = cpu_to_le16(bi->len); 408 elem++; 409 410 desc_info->nbufs++; 411 } 412 } 413 414 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 415 0, (desc_info->nbufs - 1), buf_info->dma_addr); 416 desc->cmd = cpu_to_le64(cmd); 417 desc->len = cpu_to_le16(len); 418 desc->csum_start = 0; 419 desc->csum_offset = 0; 420 421 stats->xdp_frames++; 422 stats->pkts++; 423 stats->bytes += len; 424 425 ionic_txq_post(q, ring_doorbell, ionic_tx_clean, NULL); 426 427 return 0; 428 } 429 430 int ionic_xdp_xmit(struct net_device *netdev, int n, 431 struct xdp_frame **xdp_frames, u32 flags) 432 { 433 struct ionic_lif *lif = netdev_priv(netdev); 434 struct ionic_queue *txq; 435 struct netdev_queue *nq; 436 int nxmit; 437 int space; 438 int cpu; 439 int qi; 440 441 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) 442 return -ENETDOWN; 443 444 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 445 return -EINVAL; 446 447 /* AdminQ is assumed on cpu 0, while we attempt to affinitize the 448 * TxRx queue pairs 0..n-1 on cpus 1..n. We try to keep with that 449 * affinitization here, but of course irqbalance and friends might 450 * have juggled things anyway, so we have to check for the 0 case. 451 */ 452 cpu = smp_processor_id(); 453 qi = cpu ? (cpu - 1) % lif->nxqs : cpu; 454 455 txq = &lif->txqcqs[qi]->q; 456 nq = netdev_get_tx_queue(netdev, txq->index); 457 __netif_tx_lock(nq, cpu); 458 txq_trans_cond_update(nq); 459 460 if (netif_tx_queue_stopped(nq) || 461 unlikely(ionic_maybe_stop_tx(txq, 1))) { 462 __netif_tx_unlock(nq); 463 return -EIO; 464 } 465 466 space = min_t(int, n, ionic_q_space_avail(txq)); 467 for (nxmit = 0; nxmit < space ; nxmit++) { 468 if (ionic_xdp_post_frame(netdev, txq, xdp_frames[nxmit], 469 XDP_REDIRECT, 470 virt_to_page(xdp_frames[nxmit]->data), 471 0, false)) { 472 nxmit--; 473 break; 474 } 475 } 476 477 if (flags & XDP_XMIT_FLUSH) 478 ionic_dbell_ring(lif->kern_dbpage, txq->hw_type, 479 txq->dbval | txq->head_idx); 480 481 ionic_maybe_stop_tx(txq, 4); 482 __netif_tx_unlock(nq); 483 484 return nxmit; 485 } 486 487 static bool ionic_run_xdp(struct ionic_rx_stats *stats, 488 struct net_device *netdev, 489 struct bpf_prog *xdp_prog, 490 struct ionic_queue *rxq, 491 struct ionic_buf_info *buf_info, 492 int len) 493 { 494 u32 xdp_action = XDP_ABORTED; 495 struct xdp_buff xdp_buf; 496 struct ionic_queue *txq; 497 struct netdev_queue *nq; 498 struct xdp_frame *xdpf; 499 int remain_len; 500 int frag_len; 501 int err = 0; 502 503 xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info); 504 frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 505 xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), 506 XDP_PACKET_HEADROOM, frag_len, false); 507 508 dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info), 509 XDP_PACKET_HEADROOM, len, 510 DMA_FROM_DEVICE); 511 512 prefetchw(&xdp_buf.data_hard_start); 513 514 /* We limit MTU size to one buffer if !xdp_has_frags, so 515 * if the recv len is bigger than one buffer 516 * then we know we have frag info to gather 517 */ 518 remain_len = len - frag_len; 519 if (remain_len) { 520 struct skb_shared_info *sinfo; 521 struct ionic_buf_info *bi; 522 skb_frag_t *frag; 523 524 bi = buf_info; 525 sinfo = xdp_get_shared_info_from_buff(&xdp_buf); 526 sinfo->nr_frags = 0; 527 sinfo->xdp_frags_size = 0; 528 xdp_buff_set_frags_flag(&xdp_buf); 529 530 do { 531 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) { 532 err = -ENOSPC; 533 goto out_xdp_abort; 534 } 535 536 frag = &sinfo->frags[sinfo->nr_frags]; 537 sinfo->nr_frags++; 538 bi++; 539 frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi)); 540 dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi), 541 0, frag_len, DMA_FROM_DEVICE); 542 skb_frag_fill_page_desc(frag, bi->page, 0, frag_len); 543 sinfo->xdp_frags_size += frag_len; 544 remain_len -= frag_len; 545 546 if (page_is_pfmemalloc(bi->page)) 547 xdp_buff_set_frag_pfmemalloc(&xdp_buf); 548 } while (remain_len > 0); 549 } 550 551 xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf); 552 553 switch (xdp_action) { 554 case XDP_PASS: 555 stats->xdp_pass++; 556 return false; /* false = we didn't consume the packet */ 557 558 case XDP_DROP: 559 ionic_rx_page_free(rxq, buf_info); 560 stats->xdp_drop++; 561 break; 562 563 case XDP_TX: 564 xdpf = xdp_convert_buff_to_frame(&xdp_buf); 565 if (!xdpf) 566 goto out_xdp_abort; 567 568 txq = rxq->partner; 569 nq = netdev_get_tx_queue(netdev, txq->index); 570 __netif_tx_lock(nq, smp_processor_id()); 571 txq_trans_cond_update(nq); 572 573 if (netif_tx_queue_stopped(nq) || 574 unlikely(ionic_maybe_stop_tx(txq, 1))) { 575 __netif_tx_unlock(nq); 576 goto out_xdp_abort; 577 } 578 579 dma_unmap_page(rxq->dev, buf_info->dma_addr, 580 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 581 582 err = ionic_xdp_post_frame(netdev, txq, xdpf, XDP_TX, 583 buf_info->page, 584 buf_info->page_offset, 585 true); 586 __netif_tx_unlock(nq); 587 if (err) { 588 netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); 589 goto out_xdp_abort; 590 } 591 stats->xdp_tx++; 592 593 /* the Tx completion will free the buffers */ 594 break; 595 596 case XDP_REDIRECT: 597 /* unmap the pages before handing them to a different device */ 598 dma_unmap_page(rxq->dev, buf_info->dma_addr, 599 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 600 601 err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog); 602 if (err) { 603 netdev_dbg(netdev, "xdp_do_redirect err %d\n", err); 604 goto out_xdp_abort; 605 } 606 buf_info->page = NULL; 607 rxq->xdp_flush = true; 608 stats->xdp_redirect++; 609 break; 610 611 case XDP_ABORTED: 612 default: 613 goto out_xdp_abort; 614 } 615 616 return true; 617 618 out_xdp_abort: 619 trace_xdp_exception(netdev, xdp_prog, xdp_action); 620 ionic_rx_page_free(rxq, buf_info); 621 stats->xdp_aborted++; 622 623 return true; 624 } 625 626 static void ionic_rx_clean(struct ionic_queue *q, 627 struct ionic_desc_info *desc_info, 628 struct ionic_cq_info *cq_info, 629 void *cb_arg) 630 { 631 struct net_device *netdev = q->lif->netdev; 632 struct ionic_qcq *qcq = q_to_qcq(q); 633 struct ionic_rx_stats *stats; 634 struct ionic_rxq_comp *comp; 635 struct bpf_prog *xdp_prog; 636 unsigned int headroom; 637 struct sk_buff *skb; 638 u16 len; 639 640 comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp); 641 642 stats = q_to_rx_stats(q); 643 644 if (comp->status) { 645 stats->dropped++; 646 return; 647 } 648 649 len = le16_to_cpu(comp->len); 650 stats->pkts++; 651 stats->bytes += len; 652 653 xdp_prog = READ_ONCE(q->lif->xdp_prog); 654 if (xdp_prog && 655 ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len)) 656 return; 657 658 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 659 if (len <= q->lif->rx_copybreak) 660 skb = ionic_rx_copybreak(q, desc_info, headroom, len, !!xdp_prog); 661 else 662 skb = ionic_rx_frags(q, desc_info, headroom, len, 663 comp->num_sg_elems, !!xdp_prog); 664 665 if (unlikely(!skb)) { 666 stats->dropped++; 667 return; 668 } 669 670 skb_record_rx_queue(skb, q->index); 671 672 if (likely(netdev->features & NETIF_F_RXHASH)) { 673 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 674 case IONIC_PKT_TYPE_IPV4: 675 case IONIC_PKT_TYPE_IPV6: 676 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 677 PKT_HASH_TYPE_L3); 678 break; 679 case IONIC_PKT_TYPE_IPV4_TCP: 680 case IONIC_PKT_TYPE_IPV6_TCP: 681 case IONIC_PKT_TYPE_IPV4_UDP: 682 case IONIC_PKT_TYPE_IPV6_UDP: 683 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 684 PKT_HASH_TYPE_L4); 685 break; 686 } 687 } 688 689 if (likely(netdev->features & NETIF_F_RXCSUM) && 690 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) { 691 skb->ip_summed = CHECKSUM_COMPLETE; 692 skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 693 stats->csum_complete++; 694 } else { 695 stats->csum_none++; 696 } 697 698 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 699 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 700 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 701 stats->csum_error++; 702 703 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 704 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 705 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 706 le16_to_cpu(comp->vlan_tci)); 707 stats->vlan_stripped++; 708 } 709 710 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) { 711 __le64 *cq_desc_hwstamp; 712 u64 hwstamp; 713 714 cq_desc_hwstamp = 715 cq_info->cq_desc + 716 qcq->cq.desc_size - 717 sizeof(struct ionic_rxq_comp) - 718 IONIC_HWSTAMP_CQ_NEGOFFSET; 719 720 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 721 722 if (hwstamp != IONIC_HWSTAMP_INVALID) { 723 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 724 stats->hwstamp_valid++; 725 } else { 726 stats->hwstamp_invalid++; 727 } 728 } 729 730 if (len <= q->lif->rx_copybreak) 731 napi_gro_receive(&qcq->napi, skb); 732 else 733 napi_gro_frags(&qcq->napi); 734 } 735 736 bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 737 { 738 struct ionic_queue *q = cq->bound_q; 739 struct ionic_desc_info *desc_info; 740 struct ionic_rxq_comp *comp; 741 742 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 743 744 if (!color_match(comp->pkt_type_color, cq->done_color)) 745 return false; 746 747 /* check for empty queue */ 748 if (q->tail_idx == q->head_idx) 749 return false; 750 751 if (q->tail_idx != le16_to_cpu(comp->comp_index)) 752 return false; 753 754 desc_info = &q->info[q->tail_idx]; 755 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 756 757 /* clean the related q entry, only one per qc completion */ 758 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); 759 760 desc_info->cb = NULL; 761 desc_info->cb_arg = NULL; 762 763 return true; 764 } 765 766 static inline void ionic_write_cmb_desc(struct ionic_queue *q, 767 void __iomem *cmb_desc, 768 void *desc) 769 { 770 if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS) 771 memcpy_toio(cmb_desc, desc, q->desc_size); 772 } 773 774 void ionic_rx_fill(struct ionic_queue *q) 775 { 776 struct net_device *netdev = q->lif->netdev; 777 struct ionic_desc_info *desc_info; 778 struct ionic_rxq_sg_desc *sg_desc; 779 struct ionic_rxq_sg_elem *sg_elem; 780 struct ionic_buf_info *buf_info; 781 unsigned int fill_threshold; 782 struct ionic_rxq_desc *desc; 783 unsigned int remain_len; 784 unsigned int frag_len; 785 unsigned int nfrags; 786 unsigned int n_fill; 787 unsigned int len; 788 unsigned int i; 789 unsigned int j; 790 791 n_fill = ionic_q_space_avail(q); 792 793 fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD, 794 q->num_descs / IONIC_RX_FILL_DIV); 795 if (n_fill < fill_threshold) 796 return; 797 798 len = netdev->mtu + VLAN_ETH_HLEN; 799 800 for (i = n_fill; i; i--) { 801 unsigned int headroom; 802 unsigned int buf_len; 803 804 nfrags = 0; 805 remain_len = len; 806 desc_info = &q->info[q->head_idx]; 807 desc = desc_info->desc; 808 buf_info = &desc_info->bufs[0]; 809 810 if (!buf_info->page) { /* alloc a new buffer? */ 811 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 812 desc->addr = 0; 813 desc->len = 0; 814 return; 815 } 816 } 817 818 /* fill main descriptor - buf[0] 819 * XDP uses space in the first buffer, so account for 820 * head room, tail room, and ip header in the first frag size. 821 */ 822 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 823 if (q->xdp_rxq_info) 824 buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN; 825 else 826 buf_len = ionic_rx_buf_size(buf_info); 827 frag_len = min_t(u16, len, buf_len); 828 829 desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom); 830 desc->len = cpu_to_le16(frag_len); 831 remain_len -= frag_len; 832 buf_info++; 833 nfrags++; 834 835 /* fill sg descriptors - buf[1..n] */ 836 sg_desc = desc_info->sg_desc; 837 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) { 838 sg_elem = &sg_desc->elems[j]; 839 if (!buf_info->page) { /* alloc a new sg buffer? */ 840 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 841 sg_elem->addr = 0; 842 sg_elem->len = 0; 843 return; 844 } 845 } 846 847 sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info)); 848 frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info)); 849 sg_elem->len = cpu_to_le16(frag_len); 850 remain_len -= frag_len; 851 buf_info++; 852 nfrags++; 853 } 854 855 /* clear end sg element as a sentinel */ 856 if (j < q->max_sg_elems) { 857 sg_elem = &sg_desc->elems[j]; 858 memset(sg_elem, 0, sizeof(*sg_elem)); 859 } 860 861 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 862 IONIC_RXQ_DESC_OPCODE_SIMPLE; 863 desc_info->nbufs = nfrags; 864 865 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 866 867 ionic_rxq_post(q, false, ionic_rx_clean, NULL); 868 } 869 870 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 871 q->dbval | q->head_idx); 872 873 q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE; 874 q->dbell_jiffies = jiffies; 875 876 mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline, 877 jiffies + IONIC_NAPI_DEADLINE); 878 } 879 880 void ionic_rx_empty(struct ionic_queue *q) 881 { 882 struct ionic_desc_info *desc_info; 883 struct ionic_buf_info *buf_info; 884 unsigned int i, j; 885 886 for (i = 0; i < q->num_descs; i++) { 887 desc_info = &q->info[i]; 888 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { 889 buf_info = &desc_info->bufs[j]; 890 if (buf_info->page) 891 ionic_rx_page_free(q, buf_info); 892 } 893 894 desc_info->nbufs = 0; 895 desc_info->cb = NULL; 896 desc_info->cb_arg = NULL; 897 } 898 899 q->head_idx = 0; 900 q->tail_idx = 0; 901 } 902 903 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode) 904 { 905 struct dim_sample dim_sample; 906 struct ionic_lif *lif; 907 unsigned int qi; 908 u64 pkts, bytes; 909 910 if (!qcq->intr.dim_coal_hw) 911 return; 912 913 lif = qcq->q.lif; 914 qi = qcq->cq.bound_q->index; 915 916 switch (napi_mode) { 917 case IONIC_LIF_F_TX_DIM_INTR: 918 pkts = lif->txqstats[qi].pkts; 919 bytes = lif->txqstats[qi].bytes; 920 break; 921 case IONIC_LIF_F_RX_DIM_INTR: 922 pkts = lif->rxqstats[qi].pkts; 923 bytes = lif->rxqstats[qi].bytes; 924 break; 925 default: 926 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts; 927 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes; 928 break; 929 } 930 931 dim_update_sample(qcq->cq.bound_intr->rearm_count, 932 pkts, bytes, &dim_sample); 933 934 net_dim(&qcq->dim, dim_sample); 935 } 936 937 int ionic_tx_napi(struct napi_struct *napi, int budget) 938 { 939 struct ionic_qcq *qcq = napi_to_qcq(napi); 940 struct ionic_cq *cq = napi_to_cq(napi); 941 struct ionic_dev *idev; 942 struct ionic_lif *lif; 943 u32 work_done = 0; 944 u32 flags = 0; 945 946 lif = cq->bound_q->lif; 947 idev = &lif->ionic->idev; 948 949 work_done = ionic_cq_service(cq, budget, 950 ionic_tx_service, NULL, NULL); 951 952 if (unlikely(!budget)) 953 return budget; 954 955 if (work_done < budget && napi_complete_done(napi, work_done)) { 956 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR); 957 flags |= IONIC_INTR_CRED_UNMASK; 958 cq->bound_intr->rearm_count++; 959 } 960 961 if (work_done || flags) { 962 flags |= IONIC_INTR_CRED_RESET_COALESCE; 963 ionic_intr_credits(idev->intr_ctrl, 964 cq->bound_intr->index, 965 work_done, flags); 966 } 967 968 if (!work_done && ionic_txq_poke_doorbell(&qcq->q)) 969 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 970 971 return work_done; 972 } 973 974 static void ionic_xdp_do_flush(struct ionic_cq *cq) 975 { 976 if (cq->bound_q->xdp_flush) { 977 xdp_do_flush(); 978 cq->bound_q->xdp_flush = false; 979 } 980 } 981 982 int ionic_rx_napi(struct napi_struct *napi, int budget) 983 { 984 struct ionic_qcq *qcq = napi_to_qcq(napi); 985 struct ionic_cq *cq = napi_to_cq(napi); 986 struct ionic_dev *idev; 987 struct ionic_lif *lif; 988 u32 work_done = 0; 989 u32 flags = 0; 990 991 if (unlikely(!budget)) 992 return budget; 993 994 lif = cq->bound_q->lif; 995 idev = &lif->ionic->idev; 996 997 work_done = ionic_cq_service(cq, budget, 998 ionic_rx_service, NULL, NULL); 999 1000 ionic_rx_fill(cq->bound_q); 1001 1002 ionic_xdp_do_flush(cq); 1003 if (work_done < budget && napi_complete_done(napi, work_done)) { 1004 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 1005 flags |= IONIC_INTR_CRED_UNMASK; 1006 cq->bound_intr->rearm_count++; 1007 } 1008 1009 if (work_done || flags) { 1010 flags |= IONIC_INTR_CRED_RESET_COALESCE; 1011 ionic_intr_credits(idev->intr_ctrl, 1012 cq->bound_intr->index, 1013 work_done, flags); 1014 } 1015 1016 if (!work_done && ionic_rxq_poke_doorbell(&qcq->q)) 1017 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 1018 1019 return work_done; 1020 } 1021 1022 int ionic_txrx_napi(struct napi_struct *napi, int budget) 1023 { 1024 struct ionic_qcq *rxqcq = napi_to_qcq(napi); 1025 struct ionic_cq *rxcq = napi_to_cq(napi); 1026 unsigned int qi = rxcq->bound_q->index; 1027 struct ionic_qcq *txqcq; 1028 struct ionic_dev *idev; 1029 struct ionic_lif *lif; 1030 struct ionic_cq *txcq; 1031 bool resched = false; 1032 u32 rx_work_done = 0; 1033 u32 tx_work_done = 0; 1034 u32 flags = 0; 1035 1036 lif = rxcq->bound_q->lif; 1037 idev = &lif->ionic->idev; 1038 txqcq = lif->txqcqs[qi]; 1039 txcq = &lif->txqcqs[qi]->cq; 1040 1041 tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, 1042 ionic_tx_service, NULL, NULL); 1043 1044 if (unlikely(!budget)) 1045 return budget; 1046 1047 rx_work_done = ionic_cq_service(rxcq, budget, 1048 ionic_rx_service, NULL, NULL); 1049 1050 ionic_rx_fill(rxcq->bound_q); 1051 1052 ionic_xdp_do_flush(rxcq); 1053 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 1054 ionic_dim_update(rxqcq, 0); 1055 flags |= IONIC_INTR_CRED_UNMASK; 1056 rxcq->bound_intr->rearm_count++; 1057 } 1058 1059 if (rx_work_done || flags) { 1060 flags |= IONIC_INTR_CRED_RESET_COALESCE; 1061 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, 1062 tx_work_done + rx_work_done, flags); 1063 } 1064 1065 if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q)) 1066 resched = true; 1067 if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q)) 1068 resched = true; 1069 if (resched) 1070 mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 1071 1072 return rx_work_done; 1073 } 1074 1075 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 1076 void *data, size_t len) 1077 { 1078 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1079 struct device *dev = q->dev; 1080 dma_addr_t dma_addr; 1081 1082 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 1083 if (dma_mapping_error(dev, dma_addr)) { 1084 net_warn_ratelimited("%s: DMA single map failed on %s!\n", 1085 q->lif->netdev->name, q->name); 1086 stats->dma_map_err++; 1087 return 0; 1088 } 1089 return dma_addr; 1090 } 1091 1092 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 1093 const skb_frag_t *frag, 1094 size_t offset, size_t len) 1095 { 1096 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1097 struct device *dev = q->dev; 1098 dma_addr_t dma_addr; 1099 1100 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 1101 if (dma_mapping_error(dev, dma_addr)) { 1102 net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 1103 q->lif->netdev->name, q->name); 1104 stats->dma_map_err++; 1105 } 1106 return dma_addr; 1107 } 1108 1109 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, 1110 struct ionic_desc_info *desc_info) 1111 { 1112 struct ionic_buf_info *buf_info = desc_info->bufs; 1113 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1114 struct device *dev = q->dev; 1115 dma_addr_t dma_addr; 1116 unsigned int nfrags; 1117 skb_frag_t *frag; 1118 int frag_idx; 1119 1120 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 1121 if (dma_mapping_error(dev, dma_addr)) { 1122 stats->dma_map_err++; 1123 return -EIO; 1124 } 1125 buf_info->dma_addr = dma_addr; 1126 buf_info->len = skb_headlen(skb); 1127 buf_info++; 1128 1129 frag = skb_shinfo(skb)->frags; 1130 nfrags = skb_shinfo(skb)->nr_frags; 1131 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { 1132 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 1133 if (dma_mapping_error(dev, dma_addr)) { 1134 stats->dma_map_err++; 1135 goto dma_fail; 1136 } 1137 buf_info->dma_addr = dma_addr; 1138 buf_info->len = skb_frag_size(frag); 1139 buf_info++; 1140 } 1141 1142 desc_info->nbufs = 1 + nfrags; 1143 1144 return 0; 1145 1146 dma_fail: 1147 /* unwind the frag mappings and the head mapping */ 1148 while (frag_idx > 0) { 1149 frag_idx--; 1150 buf_info--; 1151 dma_unmap_page(dev, buf_info->dma_addr, 1152 buf_info->len, DMA_TO_DEVICE); 1153 } 1154 dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); 1155 return -EIO; 1156 } 1157 1158 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 1159 struct ionic_desc_info *desc_info) 1160 { 1161 struct ionic_buf_info *buf_info = desc_info->bufs; 1162 struct device *dev = q->dev; 1163 unsigned int i; 1164 1165 if (!desc_info->nbufs) 1166 return; 1167 1168 dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr, 1169 buf_info->len, DMA_TO_DEVICE); 1170 buf_info++; 1171 for (i = 1; i < desc_info->nbufs; i++, buf_info++) 1172 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr, 1173 buf_info->len, DMA_TO_DEVICE); 1174 1175 desc_info->nbufs = 0; 1176 } 1177 1178 static void ionic_tx_clean(struct ionic_queue *q, 1179 struct ionic_desc_info *desc_info, 1180 struct ionic_cq_info *cq_info, 1181 void *cb_arg) 1182 { 1183 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1184 struct ionic_qcq *qcq = q_to_qcq(q); 1185 struct sk_buff *skb = cb_arg; 1186 u16 qi; 1187 1188 if (desc_info->xdpf) { 1189 ionic_xdp_tx_desc_clean(q->partner, desc_info); 1190 stats->clean++; 1191 1192 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index))) 1193 netif_wake_subqueue(q->lif->netdev, q->index); 1194 1195 return; 1196 } 1197 1198 ionic_tx_desc_unmap_bufs(q, desc_info); 1199 1200 if (!skb) 1201 return; 1202 1203 qi = skb_get_queue_mapping(skb); 1204 1205 if (ionic_txq_hwstamp_enabled(q)) { 1206 if (cq_info) { 1207 struct skb_shared_hwtstamps hwts = {}; 1208 __le64 *cq_desc_hwstamp; 1209 u64 hwstamp; 1210 1211 cq_desc_hwstamp = 1212 cq_info->cq_desc + 1213 qcq->cq.desc_size - 1214 sizeof(struct ionic_txq_comp) - 1215 IONIC_HWSTAMP_CQ_NEGOFFSET; 1216 1217 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 1218 1219 if (hwstamp != IONIC_HWSTAMP_INVALID) { 1220 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 1221 1222 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1223 skb_tstamp_tx(skb, &hwts); 1224 1225 stats->hwstamp_valid++; 1226 } else { 1227 stats->hwstamp_invalid++; 1228 } 1229 } 1230 1231 } else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) { 1232 netif_wake_subqueue(q->lif->netdev, qi); 1233 } 1234 1235 desc_info->bytes = skb->len; 1236 stats->clean++; 1237 1238 dev_consume_skb_any(skb); 1239 } 1240 1241 bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 1242 { 1243 struct ionic_queue *q = cq->bound_q; 1244 struct ionic_desc_info *desc_info; 1245 struct ionic_txq_comp *comp; 1246 int bytes = 0; 1247 int pkts = 0; 1248 u16 index; 1249 1250 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 1251 1252 if (!color_match(comp->color, cq->done_color)) 1253 return false; 1254 1255 /* clean the related q entries, there could be 1256 * several q entries completed for each cq completion 1257 */ 1258 do { 1259 desc_info = &q->info[q->tail_idx]; 1260 desc_info->bytes = 0; 1261 index = q->tail_idx; 1262 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1263 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); 1264 if (desc_info->cb_arg) { 1265 pkts++; 1266 bytes += desc_info->bytes; 1267 } 1268 desc_info->cb = NULL; 1269 desc_info->cb_arg = NULL; 1270 } while (index != le16_to_cpu(comp->comp_index)); 1271 1272 if (pkts && bytes && !ionic_txq_hwstamp_enabled(q)) 1273 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); 1274 1275 return true; 1276 } 1277 1278 void ionic_tx_flush(struct ionic_cq *cq) 1279 { 1280 struct ionic_dev *idev = &cq->lif->ionic->idev; 1281 u32 work_done; 1282 1283 work_done = ionic_cq_service(cq, cq->num_descs, 1284 ionic_tx_service, NULL, NULL); 1285 if (work_done) 1286 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 1287 work_done, IONIC_INTR_CRED_RESET_COALESCE); 1288 } 1289 1290 void ionic_tx_empty(struct ionic_queue *q) 1291 { 1292 struct ionic_desc_info *desc_info; 1293 int bytes = 0; 1294 int pkts = 0; 1295 1296 /* walk the not completed tx entries, if any */ 1297 while (q->head_idx != q->tail_idx) { 1298 desc_info = &q->info[q->tail_idx]; 1299 desc_info->bytes = 0; 1300 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1301 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); 1302 if (desc_info->cb_arg) { 1303 pkts++; 1304 bytes += desc_info->bytes; 1305 } 1306 desc_info->cb = NULL; 1307 desc_info->cb_arg = NULL; 1308 } 1309 1310 if (pkts && bytes && !ionic_txq_hwstamp_enabled(q)) 1311 netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes); 1312 } 1313 1314 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 1315 { 1316 int err; 1317 1318 err = skb_cow_head(skb, 0); 1319 if (err) 1320 return err; 1321 1322 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 1323 inner_ip_hdr(skb)->check = 0; 1324 inner_tcp_hdr(skb)->check = 1325 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 1326 inner_ip_hdr(skb)->daddr, 1327 0, IPPROTO_TCP, 0); 1328 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 1329 inner_tcp_hdr(skb)->check = 1330 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 1331 &inner_ipv6_hdr(skb)->daddr, 1332 0, IPPROTO_TCP, 0); 1333 } 1334 1335 return 0; 1336 } 1337 1338 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 1339 { 1340 int err; 1341 1342 err = skb_cow_head(skb, 0); 1343 if (err) 1344 return err; 1345 1346 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 1347 ip_hdr(skb)->check = 0; 1348 tcp_hdr(skb)->check = 1349 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 1350 ip_hdr(skb)->daddr, 1351 0, IPPROTO_TCP, 0); 1352 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 1353 tcp_v6_gso_csum_prep(skb); 1354 } 1355 1356 return 0; 1357 } 1358 1359 static void ionic_tx_tso_post(struct ionic_queue *q, 1360 struct ionic_desc_info *desc_info, 1361 struct sk_buff *skb, 1362 dma_addr_t addr, u8 nsge, u16 len, 1363 unsigned int hdrlen, unsigned int mss, 1364 bool outer_csum, 1365 u16 vlan_tci, bool has_vlan, 1366 bool start, bool done) 1367 { 1368 struct ionic_txq_desc *desc = desc_info->desc; 1369 u8 flags = 0; 1370 u64 cmd; 1371 1372 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1373 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1374 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 1375 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 1376 1377 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 1378 desc->cmd = cpu_to_le64(cmd); 1379 desc->len = cpu_to_le16(len); 1380 desc->vlan_tci = cpu_to_le16(vlan_tci); 1381 desc->hdr_len = cpu_to_le16(hdrlen); 1382 desc->mss = cpu_to_le16(mss); 1383 1384 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 1385 1386 if (start) { 1387 skb_tx_timestamp(skb); 1388 if (!ionic_txq_hwstamp_enabled(q)) 1389 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 1390 ionic_txq_post(q, false, ionic_tx_clean, skb); 1391 } else { 1392 ionic_txq_post(q, done, NULL, NULL); 1393 } 1394 } 1395 1396 static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb) 1397 { 1398 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1399 struct ionic_desc_info *desc_info; 1400 struct ionic_buf_info *buf_info; 1401 struct ionic_txq_sg_elem *elem; 1402 struct ionic_txq_desc *desc; 1403 unsigned int chunk_len; 1404 unsigned int frag_rem; 1405 unsigned int tso_rem; 1406 unsigned int seg_rem; 1407 dma_addr_t desc_addr; 1408 dma_addr_t frag_addr; 1409 unsigned int hdrlen; 1410 unsigned int len; 1411 unsigned int mss; 1412 bool start, done; 1413 bool outer_csum; 1414 bool has_vlan; 1415 u16 desc_len; 1416 u8 desc_nsge; 1417 u16 vlan_tci; 1418 bool encap; 1419 int err; 1420 1421 desc_info = &q->info[q->head_idx]; 1422 buf_info = desc_info->bufs; 1423 1424 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1425 return -EIO; 1426 1427 len = skb->len; 1428 mss = skb_shinfo(skb)->gso_size; 1429 outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 1430 SKB_GSO_GRE_CSUM | 1431 SKB_GSO_IPXIP4 | 1432 SKB_GSO_IPXIP6 | 1433 SKB_GSO_UDP_TUNNEL | 1434 SKB_GSO_UDP_TUNNEL_CSUM)); 1435 has_vlan = !!skb_vlan_tag_present(skb); 1436 vlan_tci = skb_vlan_tag_get(skb); 1437 encap = skb->encapsulation; 1438 1439 /* Preload inner-most TCP csum field with IP pseudo hdr 1440 * calculated with IP length set to zero. HW will later 1441 * add in length to each TCP segment resulting from the TSO. 1442 */ 1443 1444 if (encap) 1445 err = ionic_tx_tcp_inner_pseudo_csum(skb); 1446 else 1447 err = ionic_tx_tcp_pseudo_csum(skb); 1448 if (err) { 1449 /* clean up mapping from ionic_tx_map_skb */ 1450 ionic_tx_desc_unmap_bufs(q, desc_info); 1451 return err; 1452 } 1453 1454 if (encap) 1455 hdrlen = skb_inner_tcp_all_headers(skb); 1456 else 1457 hdrlen = skb_tcp_all_headers(skb); 1458 1459 tso_rem = len; 1460 seg_rem = min(tso_rem, hdrlen + mss); 1461 1462 frag_addr = 0; 1463 frag_rem = 0; 1464 1465 start = true; 1466 1467 while (tso_rem > 0) { 1468 desc = NULL; 1469 elem = NULL; 1470 desc_addr = 0; 1471 desc_len = 0; 1472 desc_nsge = 0; 1473 /* use fragments until we have enough to post a single descriptor */ 1474 while (seg_rem > 0) { 1475 /* if the fragment is exhausted then move to the next one */ 1476 if (frag_rem == 0) { 1477 /* grab the next fragment */ 1478 frag_addr = buf_info->dma_addr; 1479 frag_rem = buf_info->len; 1480 buf_info++; 1481 } 1482 chunk_len = min(frag_rem, seg_rem); 1483 if (!desc) { 1484 /* fill main descriptor */ 1485 desc = desc_info->txq_desc; 1486 elem = desc_info->txq_sg_desc->elems; 1487 desc_addr = frag_addr; 1488 desc_len = chunk_len; 1489 } else { 1490 /* fill sg descriptor */ 1491 elem->addr = cpu_to_le64(frag_addr); 1492 elem->len = cpu_to_le16(chunk_len); 1493 elem++; 1494 desc_nsge++; 1495 } 1496 frag_addr += chunk_len; 1497 frag_rem -= chunk_len; 1498 tso_rem -= chunk_len; 1499 seg_rem -= chunk_len; 1500 } 1501 seg_rem = min(tso_rem, mss); 1502 done = (tso_rem == 0); 1503 /* post descriptor */ 1504 ionic_tx_tso_post(q, desc_info, skb, 1505 desc_addr, desc_nsge, desc_len, 1506 hdrlen, mss, outer_csum, vlan_tci, has_vlan, 1507 start, done); 1508 start = false; 1509 /* Buffer information is stored with the first tso descriptor */ 1510 desc_info = &q->info[q->head_idx]; 1511 desc_info->nbufs = 0; 1512 } 1513 1514 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); 1515 stats->bytes += len; 1516 stats->tso++; 1517 stats->tso_bytes = len; 1518 1519 return 0; 1520 } 1521 1522 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, 1523 struct ionic_desc_info *desc_info) 1524 { 1525 struct ionic_txq_desc *desc = desc_info->txq_desc; 1526 struct ionic_buf_info *buf_info = desc_info->bufs; 1527 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1528 bool has_vlan; 1529 u8 flags = 0; 1530 bool encap; 1531 u64 cmd; 1532 1533 has_vlan = !!skb_vlan_tag_present(skb); 1534 encap = skb->encapsulation; 1535 1536 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1537 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1538 1539 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 1540 flags, skb_shinfo(skb)->nr_frags, 1541 buf_info->dma_addr); 1542 desc->cmd = cpu_to_le64(cmd); 1543 desc->len = cpu_to_le16(buf_info->len); 1544 if (has_vlan) { 1545 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1546 stats->vlan_inserted++; 1547 } else { 1548 desc->vlan_tci = 0; 1549 } 1550 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 1551 desc->csum_offset = cpu_to_le16(skb->csum_offset); 1552 1553 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 1554 1555 if (skb_csum_is_sctp(skb)) 1556 stats->crc32_csum++; 1557 else 1558 stats->csum++; 1559 } 1560 1561 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, 1562 struct ionic_desc_info *desc_info) 1563 { 1564 struct ionic_txq_desc *desc = desc_info->txq_desc; 1565 struct ionic_buf_info *buf_info = desc_info->bufs; 1566 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1567 bool has_vlan; 1568 u8 flags = 0; 1569 bool encap; 1570 u64 cmd; 1571 1572 has_vlan = !!skb_vlan_tag_present(skb); 1573 encap = skb->encapsulation; 1574 1575 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1576 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1577 1578 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1579 flags, skb_shinfo(skb)->nr_frags, 1580 buf_info->dma_addr); 1581 desc->cmd = cpu_to_le64(cmd); 1582 desc->len = cpu_to_le16(buf_info->len); 1583 if (has_vlan) { 1584 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1585 stats->vlan_inserted++; 1586 } else { 1587 desc->vlan_tci = 0; 1588 } 1589 desc->csum_start = 0; 1590 desc->csum_offset = 0; 1591 1592 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 1593 1594 stats->csum_none++; 1595 } 1596 1597 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, 1598 struct ionic_desc_info *desc_info) 1599 { 1600 struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc; 1601 struct ionic_buf_info *buf_info = &desc_info->bufs[1]; 1602 struct ionic_txq_sg_elem *elem = sg_desc->elems; 1603 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1604 unsigned int i; 1605 1606 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { 1607 elem->addr = cpu_to_le64(buf_info->dma_addr); 1608 elem->len = cpu_to_le16(buf_info->len); 1609 } 1610 1611 stats->frags += skb_shinfo(skb)->nr_frags; 1612 } 1613 1614 static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb) 1615 { 1616 struct ionic_desc_info *desc_info = &q->info[q->head_idx]; 1617 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1618 1619 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1620 return -EIO; 1621 1622 /* set up the initial descriptor */ 1623 if (skb->ip_summed == CHECKSUM_PARTIAL) 1624 ionic_tx_calc_csum(q, skb, desc_info); 1625 else 1626 ionic_tx_calc_no_csum(q, skb, desc_info); 1627 1628 /* add frags */ 1629 ionic_tx_skb_frags(q, skb, desc_info); 1630 1631 skb_tx_timestamp(skb); 1632 stats->pkts++; 1633 stats->bytes += skb->len; 1634 1635 if (!ionic_txq_hwstamp_enabled(q)) 1636 netdev_tx_sent_queue(q_to_ndq(q), skb->len); 1637 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb); 1638 1639 return 0; 1640 } 1641 1642 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 1643 { 1644 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1645 bool too_many_frags = false; 1646 skb_frag_t *frag; 1647 int desc_bufs; 1648 int chunk_len; 1649 int frag_rem; 1650 int tso_rem; 1651 int seg_rem; 1652 bool encap; 1653 int hdrlen; 1654 int ndescs; 1655 int err; 1656 1657 /* Each desc is mss long max, so a descriptor for each gso_seg */ 1658 if (skb_is_gso(skb)) { 1659 ndescs = skb_shinfo(skb)->gso_segs; 1660 } else { 1661 ndescs = 1; 1662 if (skb_shinfo(skb)->nr_frags > q->max_sg_elems) { 1663 too_many_frags = true; 1664 goto linearize; 1665 } 1666 } 1667 1668 /* If non-TSO, or no frags to check, we're done */ 1669 if (!skb_is_gso(skb) || !skb_shinfo(skb)->nr_frags) 1670 return ndescs; 1671 1672 /* We need to scan the skb to be sure that none of the MTU sized 1673 * packets in the TSO will require more sgs per descriptor than we 1674 * can support. We loop through the frags, add up the lengths for 1675 * a packet, and count the number of sgs used per packet. 1676 */ 1677 tso_rem = skb->len; 1678 frag = skb_shinfo(skb)->frags; 1679 encap = skb->encapsulation; 1680 1681 /* start with just hdr in first part of first descriptor */ 1682 if (encap) 1683 hdrlen = skb_inner_tcp_all_headers(skb); 1684 else 1685 hdrlen = skb_tcp_all_headers(skb); 1686 seg_rem = min_t(int, tso_rem, hdrlen + skb_shinfo(skb)->gso_size); 1687 frag_rem = hdrlen; 1688 1689 while (tso_rem > 0) { 1690 desc_bufs = 0; 1691 while (seg_rem > 0) { 1692 desc_bufs++; 1693 1694 /* We add the +1 because we can take buffers for one 1695 * more than we have SGs: one for the initial desc data 1696 * in addition to the SG segments that might follow. 1697 */ 1698 if (desc_bufs > q->max_sg_elems + 1) { 1699 too_many_frags = true; 1700 goto linearize; 1701 } 1702 1703 if (frag_rem == 0) { 1704 frag_rem = skb_frag_size(frag); 1705 frag++; 1706 } 1707 chunk_len = min(frag_rem, seg_rem); 1708 frag_rem -= chunk_len; 1709 tso_rem -= chunk_len; 1710 seg_rem -= chunk_len; 1711 } 1712 1713 seg_rem = min_t(int, tso_rem, skb_shinfo(skb)->gso_size); 1714 } 1715 1716 linearize: 1717 if (too_many_frags) { 1718 err = skb_linearize(skb); 1719 if (err) 1720 return err; 1721 stats->linearize++; 1722 } 1723 1724 return ndescs; 1725 } 1726 1727 static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs) 1728 { 1729 int stopped = 0; 1730 1731 if (unlikely(!ionic_q_has_space(q, ndescs))) { 1732 netif_stop_subqueue(q->lif->netdev, q->index); 1733 stopped = 1; 1734 1735 /* Might race with ionic_tx_clean, check again */ 1736 smp_rmb(); 1737 if (ionic_q_has_space(q, ndescs)) { 1738 netif_wake_subqueue(q->lif->netdev, q->index); 1739 stopped = 0; 1740 } 1741 } 1742 1743 return stopped; 1744 } 1745 1746 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, 1747 struct net_device *netdev) 1748 { 1749 struct ionic_lif *lif = netdev_priv(netdev); 1750 struct ionic_queue *q = &lif->hwstamp_txq->q; 1751 int err, ndescs; 1752 1753 /* Does not stop/start txq, because we post to a separate tx queue 1754 * for timestamping, and if a packet can't be posted immediately to 1755 * the timestamping queue, it is dropped. 1756 */ 1757 1758 ndescs = ionic_tx_descs_needed(q, skb); 1759 if (unlikely(ndescs < 0)) 1760 goto err_out_drop; 1761 1762 if (unlikely(!ionic_q_has_space(q, ndescs))) 1763 goto err_out_drop; 1764 1765 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP; 1766 if (skb_is_gso(skb)) 1767 err = ionic_tx_tso(q, skb); 1768 else 1769 err = ionic_tx(q, skb); 1770 1771 if (err) 1772 goto err_out_drop; 1773 1774 return NETDEV_TX_OK; 1775 1776 err_out_drop: 1777 q->drop++; 1778 dev_kfree_skb(skb); 1779 return NETDEV_TX_OK; 1780 } 1781 1782 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1783 { 1784 u16 queue_index = skb_get_queue_mapping(skb); 1785 struct ionic_lif *lif = netdev_priv(netdev); 1786 struct ionic_queue *q; 1787 int ndescs; 1788 int err; 1789 1790 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 1791 dev_kfree_skb(skb); 1792 return NETDEV_TX_OK; 1793 } 1794 1795 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 1796 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode) 1797 return ionic_start_hwstamp_xmit(skb, netdev); 1798 1799 if (unlikely(queue_index >= lif->nxqs)) 1800 queue_index = 0; 1801 q = &lif->txqcqs[queue_index]->q; 1802 1803 ndescs = ionic_tx_descs_needed(q, skb); 1804 if (ndescs < 0) 1805 goto err_out_drop; 1806 1807 if (unlikely(ionic_maybe_stop_tx(q, ndescs))) 1808 return NETDEV_TX_BUSY; 1809 1810 if (skb_is_gso(skb)) 1811 err = ionic_tx_tso(q, skb); 1812 else 1813 err = ionic_tx(q, skb); 1814 1815 if (err) 1816 goto err_out_drop; 1817 1818 /* Stop the queue if there aren't descriptors for the next packet. 1819 * Since our SG lists per descriptor take care of most of the possible 1820 * fragmentation, we don't need to have many descriptors available. 1821 */ 1822 ionic_maybe_stop_tx(q, 4); 1823 1824 return NETDEV_TX_OK; 1825 1826 err_out_drop: 1827 q->drop++; 1828 dev_kfree_skb(skb); 1829 return NETDEV_TX_OK; 1830 } 1831