1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip6_checksum.h> 8 #include <net/netdev_queues.h> 9 10 #include "ionic.h" 11 #include "ionic_lif.h" 12 #include "ionic_txrx.h" 13 14 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 15 void *data, size_t len); 16 17 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 18 const skb_frag_t *frag, 19 size_t offset, size_t len); 20 21 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 22 struct ionic_tx_desc_info *desc_info); 23 24 static void ionic_tx_clean(struct ionic_queue *q, 25 struct ionic_tx_desc_info *desc_info, 26 struct ionic_txq_comp *comp); 27 28 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell) 29 { 30 ionic_q_post(q, ring_dbell); 31 } 32 33 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell) 34 { 35 ionic_q_post(q, ring_dbell); 36 } 37 38 bool ionic_txq_poke_doorbell(struct ionic_queue *q) 39 { 40 struct netdev_queue *netdev_txq; 41 unsigned long now, then, dif; 42 struct net_device *netdev; 43 44 netdev = q->lif->netdev; 45 netdev_txq = netdev_get_tx_queue(netdev, q->index); 46 47 HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id()); 48 49 if (q->tail_idx == q->head_idx) { 50 HARD_TX_UNLOCK(netdev, netdev_txq); 51 return false; 52 } 53 54 now = READ_ONCE(jiffies); 55 then = q->dbell_jiffies; 56 dif = now - then; 57 58 if (dif > q->dbell_deadline) { 59 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 60 q->dbval | q->head_idx); 61 62 q->dbell_jiffies = now; 63 } 64 65 HARD_TX_UNLOCK(netdev, netdev_txq); 66 67 return true; 68 } 69 70 bool ionic_rxq_poke_doorbell(struct ionic_queue *q) 71 { 72 unsigned long now, then, dif; 73 74 /* no lock, called from rx napi or txrx napi, nothing else can fill */ 75 76 if (q->tail_idx == q->head_idx) 77 return false; 78 79 now = READ_ONCE(jiffies); 80 then = q->dbell_jiffies; 81 dif = now - then; 82 83 if (dif > q->dbell_deadline) { 84 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 85 q->dbval | q->head_idx); 86 87 q->dbell_jiffies = now; 88 89 dif = 2 * q->dbell_deadline; 90 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE) 91 dif = IONIC_RX_MAX_DOORBELL_DEADLINE; 92 93 q->dbell_deadline = dif; 94 } 95 96 return true; 97 } 98 99 static inline struct ionic_txq_sg_elem *ionic_tx_sg_elems(struct ionic_queue *q) 100 { 101 if (likely(q->sg_desc_size == sizeof(struct ionic_txq_sg_desc_v1))) 102 return q->txq_sgl_v1[q->head_idx].elems; 103 else 104 return q->txq_sgl[q->head_idx].elems; 105 } 106 107 static inline struct netdev_queue *q_to_ndq(struct net_device *netdev, 108 struct ionic_queue *q) 109 { 110 return netdev_get_tx_queue(netdev, q->index); 111 } 112 113 static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info) 114 { 115 return page_address(buf_info->page) + buf_info->page_offset; 116 } 117 118 static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info) 119 { 120 return buf_info->dma_addr + buf_info->page_offset; 121 } 122 123 static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info) 124 { 125 return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset); 126 } 127 128 static int ionic_rx_page_alloc(struct ionic_queue *q, 129 struct ionic_buf_info *buf_info) 130 { 131 struct device *dev = q->dev; 132 dma_addr_t dma_addr; 133 struct page *page; 134 135 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 136 if (unlikely(!page)) { 137 net_err_ratelimited("%s: %s page alloc failed\n", 138 dev_name(dev), q->name); 139 q_to_rx_stats(q)->alloc_err++; 140 return -ENOMEM; 141 } 142 143 dma_addr = dma_map_page(dev, page, 0, 144 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 145 if (unlikely(dma_mapping_error(dev, dma_addr))) { 146 __free_pages(page, 0); 147 net_err_ratelimited("%s: %s dma map failed\n", 148 dev_name(dev), q->name); 149 q_to_rx_stats(q)->dma_map_err++; 150 return -EIO; 151 } 152 153 buf_info->dma_addr = dma_addr; 154 buf_info->page = page; 155 buf_info->page_offset = 0; 156 157 return 0; 158 } 159 160 static void ionic_rx_page_free(struct ionic_queue *q, 161 struct ionic_buf_info *buf_info) 162 { 163 struct device *dev = q->dev; 164 165 if (unlikely(!buf_info)) { 166 net_err_ratelimited("%s: %s invalid buf_info in free\n", 167 dev_name(dev), q->name); 168 return; 169 } 170 171 if (!buf_info->page) 172 return; 173 174 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 175 __free_pages(buf_info->page, 0); 176 buf_info->page = NULL; 177 } 178 179 static bool ionic_rx_buf_recycle(struct ionic_queue *q, 180 struct ionic_buf_info *buf_info, u32 len) 181 { 182 u32 size; 183 184 /* don't re-use pages allocated in low-mem condition */ 185 if (page_is_pfmemalloc(buf_info->page)) 186 return false; 187 188 /* don't re-use buffers from non-local numa nodes */ 189 if (page_to_nid(buf_info->page) != numa_mem_id()) 190 return false; 191 192 size = ALIGN(len, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ); 193 buf_info->page_offset += size; 194 if (buf_info->page_offset >= IONIC_PAGE_SIZE) 195 return false; 196 197 get_page(buf_info->page); 198 199 return true; 200 } 201 202 static void ionic_rx_add_skb_frag(struct ionic_queue *q, 203 struct sk_buff *skb, 204 struct ionic_buf_info *buf_info, 205 u32 off, u32 len, 206 bool synced) 207 { 208 if (!synced) 209 dma_sync_single_range_for_cpu(q->dev, ionic_rx_buf_pa(buf_info), 210 off, len, DMA_FROM_DEVICE); 211 212 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 213 buf_info->page, buf_info->page_offset + off, 214 len, 215 IONIC_PAGE_SIZE); 216 217 if (!ionic_rx_buf_recycle(q, buf_info, len)) { 218 dma_unmap_page(q->dev, buf_info->dma_addr, 219 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 220 buf_info->page = NULL; 221 } 222 } 223 224 static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q, 225 struct ionic_rx_desc_info *desc_info, 226 unsigned int headroom, 227 unsigned int len, 228 unsigned int num_sg_elems, 229 bool synced) 230 { 231 struct ionic_buf_info *buf_info; 232 struct sk_buff *skb; 233 unsigned int i; 234 u16 frag_len; 235 236 buf_info = &desc_info->bufs[0]; 237 prefetchw(buf_info->page); 238 239 skb = napi_get_frags(&q_to_qcq(q)->napi); 240 if (unlikely(!skb)) { 241 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 242 dev_name(q->dev), q->name); 243 q_to_rx_stats(q)->alloc_err++; 244 return NULL; 245 } 246 247 if (headroom) 248 frag_len = min_t(u16, len, 249 IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 250 else 251 frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 252 253 if (unlikely(!buf_info->page)) 254 goto err_bad_buf_page; 255 ionic_rx_add_skb_frag(q, skb, buf_info, headroom, frag_len, synced); 256 len -= frag_len; 257 buf_info++; 258 259 for (i = 0; i < num_sg_elems; i++, buf_info++) { 260 if (unlikely(!buf_info->page)) 261 goto err_bad_buf_page; 262 frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 263 ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced); 264 len -= frag_len; 265 } 266 267 return skb; 268 269 err_bad_buf_page: 270 dev_kfree_skb(skb); 271 return NULL; 272 } 273 274 static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev, 275 struct ionic_queue *q, 276 struct ionic_rx_desc_info *desc_info, 277 unsigned int headroom, 278 unsigned int len, 279 bool synced) 280 { 281 struct ionic_buf_info *buf_info; 282 struct device *dev = q->dev; 283 struct sk_buff *skb; 284 285 buf_info = &desc_info->bufs[0]; 286 287 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 288 if (unlikely(!skb)) { 289 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 290 dev_name(dev), q->name); 291 q_to_rx_stats(q)->alloc_err++; 292 return NULL; 293 } 294 295 if (unlikely(!buf_info->page)) { 296 dev_kfree_skb(skb); 297 return NULL; 298 } 299 300 if (!synced) 301 dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 302 headroom, len, DMA_FROM_DEVICE); 303 skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); 304 dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info), 305 headroom, len, DMA_FROM_DEVICE); 306 307 skb_put(skb, len); 308 skb->protocol = eth_type_trans(skb, netdev); 309 310 return skb; 311 } 312 313 static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, 314 struct ionic_tx_desc_info *desc_info) 315 { 316 unsigned int nbufs = desc_info->nbufs; 317 struct ionic_buf_info *buf_info; 318 struct device *dev = q->dev; 319 int i; 320 321 if (!nbufs) 322 return; 323 324 buf_info = desc_info->bufs; 325 dma_unmap_single(dev, buf_info->dma_addr, 326 buf_info->len, DMA_TO_DEVICE); 327 if (desc_info->act == XDP_TX) 328 __free_pages(buf_info->page, 0); 329 buf_info->page = NULL; 330 331 buf_info++; 332 for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) { 333 dma_unmap_page(dev, buf_info->dma_addr, 334 buf_info->len, DMA_TO_DEVICE); 335 if (desc_info->act == XDP_TX) 336 __free_pages(buf_info->page, 0); 337 buf_info->page = NULL; 338 } 339 340 if (desc_info->act == XDP_REDIRECT) 341 xdp_return_frame(desc_info->xdpf); 342 343 desc_info->nbufs = 0; 344 desc_info->xdpf = NULL; 345 desc_info->act = 0; 346 } 347 348 static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame, 349 enum xdp_action act, struct page *page, int off, 350 bool ring_doorbell) 351 { 352 struct ionic_tx_desc_info *desc_info; 353 struct ionic_buf_info *buf_info; 354 struct ionic_tx_stats *stats; 355 struct ionic_txq_desc *desc; 356 size_t len = frame->len; 357 dma_addr_t dma_addr; 358 u64 cmd; 359 360 desc_info = &q->tx_info[q->head_idx]; 361 desc = &q->txq[q->head_idx]; 362 buf_info = desc_info->bufs; 363 stats = q_to_tx_stats(q); 364 365 dma_addr = ionic_tx_map_single(q, frame->data, len); 366 if (!dma_addr) 367 return -EIO; 368 buf_info->dma_addr = dma_addr; 369 buf_info->len = len; 370 buf_info->page = page; 371 buf_info->page_offset = off; 372 373 desc_info->nbufs = 1; 374 desc_info->xdpf = frame; 375 desc_info->act = act; 376 377 if (xdp_frame_has_frags(frame)) { 378 struct ionic_txq_sg_elem *elem; 379 struct skb_shared_info *sinfo; 380 struct ionic_buf_info *bi; 381 skb_frag_t *frag; 382 int i; 383 384 bi = &buf_info[1]; 385 sinfo = xdp_get_shared_info_from_frame(frame); 386 frag = sinfo->frags; 387 elem = ionic_tx_sg_elems(q); 388 for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) { 389 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 390 if (!dma_addr) { 391 ionic_tx_desc_unmap_bufs(q, desc_info); 392 return -EIO; 393 } 394 bi->dma_addr = dma_addr; 395 bi->len = skb_frag_size(frag); 396 bi->page = skb_frag_page(frag); 397 398 elem->addr = cpu_to_le64(bi->dma_addr); 399 elem->len = cpu_to_le16(bi->len); 400 elem++; 401 402 desc_info->nbufs++; 403 } 404 } 405 406 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 407 0, (desc_info->nbufs - 1), buf_info->dma_addr); 408 desc->cmd = cpu_to_le64(cmd); 409 desc->len = cpu_to_le16(len); 410 desc->csum_start = 0; 411 desc->csum_offset = 0; 412 413 stats->xdp_frames++; 414 stats->pkts++; 415 stats->bytes += len; 416 417 ionic_txq_post(q, ring_doorbell); 418 419 return 0; 420 } 421 422 int ionic_xdp_xmit(struct net_device *netdev, int n, 423 struct xdp_frame **xdp_frames, u32 flags) 424 { 425 struct ionic_lif *lif = netdev_priv(netdev); 426 struct ionic_queue *txq; 427 struct netdev_queue *nq; 428 int nxmit; 429 int space; 430 int cpu; 431 int qi; 432 433 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) 434 return -ENETDOWN; 435 436 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 437 return -EINVAL; 438 439 /* AdminQ is assumed on cpu 0, while we attempt to affinitize the 440 * TxRx queue pairs 0..n-1 on cpus 1..n. We try to keep with that 441 * affinitization here, but of course irqbalance and friends might 442 * have juggled things anyway, so we have to check for the 0 case. 443 */ 444 cpu = smp_processor_id(); 445 qi = cpu ? (cpu - 1) % lif->nxqs : cpu; 446 447 txq = &lif->txqcqs[qi]->q; 448 nq = netdev_get_tx_queue(netdev, txq->index); 449 __netif_tx_lock(nq, cpu); 450 txq_trans_cond_update(nq); 451 452 if (netif_tx_queue_stopped(nq) || 453 !netif_txq_maybe_stop(q_to_ndq(netdev, txq), 454 ionic_q_space_avail(txq), 455 1, 1)) { 456 __netif_tx_unlock(nq); 457 return -EIO; 458 } 459 460 space = min_t(int, n, ionic_q_space_avail(txq)); 461 for (nxmit = 0; nxmit < space ; nxmit++) { 462 if (ionic_xdp_post_frame(txq, xdp_frames[nxmit], 463 XDP_REDIRECT, 464 virt_to_page(xdp_frames[nxmit]->data), 465 0, false)) { 466 nxmit--; 467 break; 468 } 469 } 470 471 if (flags & XDP_XMIT_FLUSH) 472 ionic_dbell_ring(lif->kern_dbpage, txq->hw_type, 473 txq->dbval | txq->head_idx); 474 475 netif_txq_maybe_stop(q_to_ndq(netdev, txq), 476 ionic_q_space_avail(txq), 477 4, 4); 478 __netif_tx_unlock(nq); 479 480 return nxmit; 481 } 482 483 static bool ionic_run_xdp(struct ionic_rx_stats *stats, 484 struct net_device *netdev, 485 struct bpf_prog *xdp_prog, 486 struct ionic_queue *rxq, 487 struct ionic_buf_info *buf_info, 488 int len) 489 { 490 u32 xdp_action = XDP_ABORTED; 491 struct xdp_buff xdp_buf; 492 struct ionic_queue *txq; 493 struct netdev_queue *nq; 494 struct xdp_frame *xdpf; 495 int remain_len; 496 int frag_len; 497 int err = 0; 498 499 xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info); 500 frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 501 xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), 502 XDP_PACKET_HEADROOM, frag_len, false); 503 504 dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info), 505 XDP_PACKET_HEADROOM, len, 506 DMA_FROM_DEVICE); 507 508 prefetchw(&xdp_buf.data_hard_start); 509 510 /* We limit MTU size to one buffer if !xdp_has_frags, so 511 * if the recv len is bigger than one buffer 512 * then we know we have frag info to gather 513 */ 514 remain_len = len - frag_len; 515 if (remain_len) { 516 struct skb_shared_info *sinfo; 517 struct ionic_buf_info *bi; 518 skb_frag_t *frag; 519 520 bi = buf_info; 521 sinfo = xdp_get_shared_info_from_buff(&xdp_buf); 522 sinfo->nr_frags = 0; 523 sinfo->xdp_frags_size = 0; 524 xdp_buff_set_frags_flag(&xdp_buf); 525 526 do { 527 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) { 528 err = -ENOSPC; 529 goto out_xdp_abort; 530 } 531 532 frag = &sinfo->frags[sinfo->nr_frags]; 533 sinfo->nr_frags++; 534 bi++; 535 frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi)); 536 dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi), 537 0, frag_len, DMA_FROM_DEVICE); 538 skb_frag_fill_page_desc(frag, bi->page, 0, frag_len); 539 sinfo->xdp_frags_size += frag_len; 540 remain_len -= frag_len; 541 542 if (page_is_pfmemalloc(bi->page)) 543 xdp_buff_set_frag_pfmemalloc(&xdp_buf); 544 } while (remain_len > 0); 545 } 546 547 xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf); 548 549 switch (xdp_action) { 550 case XDP_PASS: 551 stats->xdp_pass++; 552 return false; /* false = we didn't consume the packet */ 553 554 case XDP_DROP: 555 ionic_rx_page_free(rxq, buf_info); 556 stats->xdp_drop++; 557 break; 558 559 case XDP_TX: 560 xdpf = xdp_convert_buff_to_frame(&xdp_buf); 561 if (!xdpf) 562 goto out_xdp_abort; 563 564 txq = rxq->partner; 565 nq = netdev_get_tx_queue(netdev, txq->index); 566 __netif_tx_lock(nq, smp_processor_id()); 567 txq_trans_cond_update(nq); 568 569 if (netif_tx_queue_stopped(nq) || 570 !netif_txq_maybe_stop(q_to_ndq(netdev, txq), 571 ionic_q_space_avail(txq), 572 1, 1)) { 573 __netif_tx_unlock(nq); 574 goto out_xdp_abort; 575 } 576 577 dma_unmap_page(rxq->dev, buf_info->dma_addr, 578 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 579 580 err = ionic_xdp_post_frame(txq, xdpf, XDP_TX, 581 buf_info->page, 582 buf_info->page_offset, 583 true); 584 __netif_tx_unlock(nq); 585 if (err) { 586 netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); 587 goto out_xdp_abort; 588 } 589 stats->xdp_tx++; 590 591 /* the Tx completion will free the buffers */ 592 break; 593 594 case XDP_REDIRECT: 595 /* unmap the pages before handing them to a different device */ 596 dma_unmap_page(rxq->dev, buf_info->dma_addr, 597 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 598 599 err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog); 600 if (err) { 601 netdev_dbg(netdev, "xdp_do_redirect err %d\n", err); 602 goto out_xdp_abort; 603 } 604 buf_info->page = NULL; 605 rxq->xdp_flush = true; 606 stats->xdp_redirect++; 607 break; 608 609 case XDP_ABORTED: 610 default: 611 goto out_xdp_abort; 612 } 613 614 return true; 615 616 out_xdp_abort: 617 trace_xdp_exception(netdev, xdp_prog, xdp_action); 618 ionic_rx_page_free(rxq, buf_info); 619 stats->xdp_aborted++; 620 621 return true; 622 } 623 624 static void ionic_rx_clean(struct ionic_queue *q, 625 struct ionic_rx_desc_info *desc_info, 626 struct ionic_rxq_comp *comp) 627 { 628 struct net_device *netdev = q->lif->netdev; 629 struct ionic_qcq *qcq = q_to_qcq(q); 630 struct ionic_rx_stats *stats; 631 struct bpf_prog *xdp_prog; 632 unsigned int headroom; 633 struct sk_buff *skb; 634 bool synced = false; 635 bool use_copybreak; 636 u16 len; 637 638 stats = q_to_rx_stats(q); 639 640 if (comp->status) { 641 stats->dropped++; 642 return; 643 } 644 645 len = le16_to_cpu(comp->len); 646 stats->pkts++; 647 stats->bytes += len; 648 649 xdp_prog = READ_ONCE(q->lif->xdp_prog); 650 if (xdp_prog) { 651 if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len)) 652 return; 653 synced = true; 654 } 655 656 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 657 use_copybreak = len <= q->lif->rx_copybreak; 658 if (use_copybreak) 659 skb = ionic_rx_copybreak(netdev, q, desc_info, 660 headroom, len, synced); 661 else 662 skb = ionic_rx_build_skb(q, desc_info, headroom, len, 663 comp->num_sg_elems, synced); 664 665 if (unlikely(!skb)) { 666 stats->dropped++; 667 return; 668 } 669 670 skb_record_rx_queue(skb, q->index); 671 672 if (likely(netdev->features & NETIF_F_RXHASH)) { 673 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 674 case IONIC_PKT_TYPE_IPV4: 675 case IONIC_PKT_TYPE_IPV6: 676 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 677 PKT_HASH_TYPE_L3); 678 break; 679 case IONIC_PKT_TYPE_IPV4_TCP: 680 case IONIC_PKT_TYPE_IPV6_TCP: 681 case IONIC_PKT_TYPE_IPV4_UDP: 682 case IONIC_PKT_TYPE_IPV6_UDP: 683 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 684 PKT_HASH_TYPE_L4); 685 break; 686 } 687 } 688 689 if (likely(netdev->features & NETIF_F_RXCSUM) && 690 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) { 691 skb->ip_summed = CHECKSUM_COMPLETE; 692 skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 693 stats->csum_complete++; 694 } else { 695 stats->csum_none++; 696 } 697 698 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 699 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 700 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 701 stats->csum_error++; 702 703 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 704 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 705 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 706 le16_to_cpu(comp->vlan_tci)); 707 stats->vlan_stripped++; 708 } 709 710 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) { 711 __le64 *cq_desc_hwstamp; 712 u64 hwstamp; 713 714 cq_desc_hwstamp = 715 (void *)comp + 716 qcq->cq.desc_size - 717 sizeof(struct ionic_rxq_comp) - 718 IONIC_HWSTAMP_CQ_NEGOFFSET; 719 720 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 721 722 if (hwstamp != IONIC_HWSTAMP_INVALID) { 723 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 724 stats->hwstamp_valid++; 725 } else { 726 stats->hwstamp_invalid++; 727 } 728 } 729 730 if (use_copybreak) 731 napi_gro_receive(&qcq->napi, skb); 732 else 733 napi_gro_frags(&qcq->napi); 734 } 735 736 bool ionic_rx_service(struct ionic_cq *cq) 737 { 738 struct ionic_rx_desc_info *desc_info; 739 struct ionic_queue *q = cq->bound_q; 740 struct ionic_rxq_comp *comp; 741 742 comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; 743 744 if (!color_match(comp->pkt_type_color, cq->done_color)) 745 return false; 746 747 /* check for empty queue */ 748 if (q->tail_idx == q->head_idx) 749 return false; 750 751 if (q->tail_idx != le16_to_cpu(comp->comp_index)) 752 return false; 753 754 desc_info = &q->rx_info[q->tail_idx]; 755 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 756 757 /* clean the related q entry, only one per qc completion */ 758 ionic_rx_clean(q, desc_info, comp); 759 760 return true; 761 } 762 763 static inline void ionic_write_cmb_desc(struct ionic_queue *q, 764 void *desc) 765 { 766 /* Since Rx and Tx descriptors are the same size, we can 767 * save an instruction or two and skip the qtype check. 768 */ 769 if (unlikely(q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS)) 770 memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0])); 771 } 772 773 void ionic_rx_fill(struct ionic_queue *q) 774 { 775 struct net_device *netdev = q->lif->netdev; 776 struct ionic_rx_desc_info *desc_info; 777 struct ionic_rxq_sg_elem *sg_elem; 778 struct ionic_buf_info *buf_info; 779 unsigned int fill_threshold; 780 struct ionic_rxq_desc *desc; 781 unsigned int remain_len; 782 unsigned int frag_len; 783 unsigned int nfrags; 784 unsigned int n_fill; 785 unsigned int len; 786 unsigned int i; 787 unsigned int j; 788 789 n_fill = ionic_q_space_avail(q); 790 791 fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD, 792 q->num_descs / IONIC_RX_FILL_DIV); 793 if (n_fill < fill_threshold) 794 return; 795 796 len = netdev->mtu + VLAN_ETH_HLEN; 797 798 for (i = n_fill; i; i--) { 799 unsigned int headroom; 800 unsigned int buf_len; 801 802 nfrags = 0; 803 remain_len = len; 804 desc = &q->rxq[q->head_idx]; 805 desc_info = &q->rx_info[q->head_idx]; 806 buf_info = &desc_info->bufs[0]; 807 808 if (!buf_info->page) { /* alloc a new buffer? */ 809 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 810 desc->addr = 0; 811 desc->len = 0; 812 return; 813 } 814 } 815 816 /* fill main descriptor - buf[0] 817 * XDP uses space in the first buffer, so account for 818 * head room, tail room, and ip header in the first frag size. 819 */ 820 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 821 if (q->xdp_rxq_info) 822 buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN; 823 else 824 buf_len = ionic_rx_buf_size(buf_info); 825 frag_len = min_t(u16, len, buf_len); 826 827 desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom); 828 desc->len = cpu_to_le16(frag_len); 829 remain_len -= frag_len; 830 buf_info++; 831 nfrags++; 832 833 /* fill sg descriptors - buf[1..n] */ 834 sg_elem = q->rxq_sgl[q->head_idx].elems; 835 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) { 836 if (!buf_info->page) { /* alloc a new sg buffer? */ 837 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 838 sg_elem->addr = 0; 839 sg_elem->len = 0; 840 return; 841 } 842 } 843 844 sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info)); 845 frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info)); 846 sg_elem->len = cpu_to_le16(frag_len); 847 remain_len -= frag_len; 848 buf_info++; 849 nfrags++; 850 } 851 852 /* clear end sg element as a sentinel */ 853 if (j < q->max_sg_elems) 854 memset(sg_elem, 0, sizeof(*sg_elem)); 855 856 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 857 IONIC_RXQ_DESC_OPCODE_SIMPLE; 858 desc_info->nbufs = nfrags; 859 860 ionic_write_cmb_desc(q, desc); 861 862 ionic_rxq_post(q, false); 863 } 864 865 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 866 q->dbval | q->head_idx); 867 868 q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE; 869 q->dbell_jiffies = jiffies; 870 871 mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline, 872 jiffies + IONIC_NAPI_DEADLINE); 873 } 874 875 void ionic_rx_empty(struct ionic_queue *q) 876 { 877 struct ionic_rx_desc_info *desc_info; 878 struct ionic_buf_info *buf_info; 879 unsigned int i, j; 880 881 for (i = 0; i < q->num_descs; i++) { 882 desc_info = &q->rx_info[i]; 883 for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) { 884 buf_info = &desc_info->bufs[j]; 885 if (buf_info->page) 886 ionic_rx_page_free(q, buf_info); 887 } 888 889 desc_info->nbufs = 0; 890 } 891 892 q->head_idx = 0; 893 q->tail_idx = 0; 894 } 895 896 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode) 897 { 898 struct dim_sample dim_sample; 899 struct ionic_lif *lif; 900 unsigned int qi; 901 u64 pkts, bytes; 902 903 if (!qcq->intr.dim_coal_hw) 904 return; 905 906 lif = qcq->q.lif; 907 qi = qcq->cq.bound_q->index; 908 909 switch (napi_mode) { 910 case IONIC_LIF_F_TX_DIM_INTR: 911 pkts = lif->txqstats[qi].pkts; 912 bytes = lif->txqstats[qi].bytes; 913 break; 914 case IONIC_LIF_F_RX_DIM_INTR: 915 pkts = lif->rxqstats[qi].pkts; 916 bytes = lif->rxqstats[qi].bytes; 917 break; 918 default: 919 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts; 920 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes; 921 break; 922 } 923 924 dim_update_sample(qcq->cq.bound_intr->rearm_count, 925 pkts, bytes, &dim_sample); 926 927 net_dim(&qcq->dim, dim_sample); 928 } 929 930 int ionic_tx_napi(struct napi_struct *napi, int budget) 931 { 932 struct ionic_qcq *qcq = napi_to_qcq(napi); 933 struct ionic_cq *cq = napi_to_cq(napi); 934 u32 work_done = 0; 935 u32 flags = 0; 936 937 work_done = ionic_tx_cq_service(cq, budget); 938 939 if (unlikely(!budget)) 940 return budget; 941 942 if (work_done < budget && napi_complete_done(napi, work_done)) { 943 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR); 944 flags |= IONIC_INTR_CRED_UNMASK; 945 cq->bound_intr->rearm_count++; 946 } 947 948 if (work_done || flags) { 949 flags |= IONIC_INTR_CRED_RESET_COALESCE; 950 ionic_intr_credits(cq->idev->intr_ctrl, 951 cq->bound_intr->index, 952 work_done, flags); 953 } 954 955 if (!work_done && ionic_txq_poke_doorbell(&qcq->q)) 956 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 957 958 return work_done; 959 } 960 961 static void ionic_xdp_do_flush(struct ionic_cq *cq) 962 { 963 if (cq->bound_q->xdp_flush) { 964 xdp_do_flush(); 965 cq->bound_q->xdp_flush = false; 966 } 967 } 968 969 int ionic_rx_napi(struct napi_struct *napi, int budget) 970 { 971 struct ionic_qcq *qcq = napi_to_qcq(napi); 972 struct ionic_cq *cq = napi_to_cq(napi); 973 u32 work_done = 0; 974 u32 flags = 0; 975 976 if (unlikely(!budget)) 977 return budget; 978 979 work_done = ionic_cq_service(cq, budget, 980 ionic_rx_service, NULL, NULL); 981 982 ionic_rx_fill(cq->bound_q); 983 984 ionic_xdp_do_flush(cq); 985 if (work_done < budget && napi_complete_done(napi, work_done)) { 986 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 987 flags |= IONIC_INTR_CRED_UNMASK; 988 cq->bound_intr->rearm_count++; 989 } 990 991 if (work_done || flags) { 992 flags |= IONIC_INTR_CRED_RESET_COALESCE; 993 ionic_intr_credits(cq->idev->intr_ctrl, 994 cq->bound_intr->index, 995 work_done, flags); 996 } 997 998 if (!work_done && ionic_rxq_poke_doorbell(&qcq->q)) 999 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 1000 1001 return work_done; 1002 } 1003 1004 int ionic_txrx_napi(struct napi_struct *napi, int budget) 1005 { 1006 struct ionic_qcq *rxqcq = napi_to_qcq(napi); 1007 struct ionic_cq *rxcq = napi_to_cq(napi); 1008 unsigned int qi = rxcq->bound_q->index; 1009 struct ionic_qcq *txqcq; 1010 struct ionic_lif *lif; 1011 struct ionic_cq *txcq; 1012 bool resched = false; 1013 u32 rx_work_done = 0; 1014 u32 tx_work_done = 0; 1015 u32 flags = 0; 1016 1017 lif = rxcq->bound_q->lif; 1018 txqcq = lif->txqcqs[qi]; 1019 txcq = &lif->txqcqs[qi]->cq; 1020 1021 tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT); 1022 1023 if (unlikely(!budget)) 1024 return budget; 1025 1026 rx_work_done = ionic_cq_service(rxcq, budget, 1027 ionic_rx_service, NULL, NULL); 1028 1029 ionic_rx_fill(rxcq->bound_q); 1030 1031 ionic_xdp_do_flush(rxcq); 1032 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 1033 ionic_dim_update(rxqcq, 0); 1034 flags |= IONIC_INTR_CRED_UNMASK; 1035 rxcq->bound_intr->rearm_count++; 1036 } 1037 1038 if (rx_work_done || flags) { 1039 flags |= IONIC_INTR_CRED_RESET_COALESCE; 1040 ionic_intr_credits(rxcq->idev->intr_ctrl, rxcq->bound_intr->index, 1041 tx_work_done + rx_work_done, flags); 1042 } 1043 1044 if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q)) 1045 resched = true; 1046 if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q)) 1047 resched = true; 1048 if (resched) 1049 mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 1050 1051 return rx_work_done; 1052 } 1053 1054 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 1055 void *data, size_t len) 1056 { 1057 struct device *dev = q->dev; 1058 dma_addr_t dma_addr; 1059 1060 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 1061 if (dma_mapping_error(dev, dma_addr)) { 1062 net_warn_ratelimited("%s: DMA single map failed on %s!\n", 1063 dev_name(dev), q->name); 1064 q_to_tx_stats(q)->dma_map_err++; 1065 return 0; 1066 } 1067 return dma_addr; 1068 } 1069 1070 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 1071 const skb_frag_t *frag, 1072 size_t offset, size_t len) 1073 { 1074 struct device *dev = q->dev; 1075 dma_addr_t dma_addr; 1076 1077 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 1078 if (dma_mapping_error(dev, dma_addr)) { 1079 net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 1080 dev_name(dev), q->name); 1081 q_to_tx_stats(q)->dma_map_err++; 1082 return 0; 1083 } 1084 return dma_addr; 1085 } 1086 1087 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, 1088 struct ionic_tx_desc_info *desc_info) 1089 { 1090 struct ionic_buf_info *buf_info = desc_info->bufs; 1091 struct device *dev = q->dev; 1092 dma_addr_t dma_addr; 1093 unsigned int nfrags; 1094 skb_frag_t *frag; 1095 int frag_idx; 1096 1097 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 1098 if (!dma_addr) 1099 return -EIO; 1100 buf_info->dma_addr = dma_addr; 1101 buf_info->len = skb_headlen(skb); 1102 buf_info++; 1103 1104 frag = skb_shinfo(skb)->frags; 1105 nfrags = skb_shinfo(skb)->nr_frags; 1106 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { 1107 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 1108 if (!dma_addr) 1109 goto dma_fail; 1110 buf_info->dma_addr = dma_addr; 1111 buf_info->len = skb_frag_size(frag); 1112 buf_info++; 1113 } 1114 1115 desc_info->nbufs = 1 + nfrags; 1116 1117 return 0; 1118 1119 dma_fail: 1120 /* unwind the frag mappings and the head mapping */ 1121 while (frag_idx > 0) { 1122 frag_idx--; 1123 buf_info--; 1124 dma_unmap_page(dev, buf_info->dma_addr, 1125 buf_info->len, DMA_TO_DEVICE); 1126 } 1127 dma_unmap_single(dev, desc_info->bufs[0].dma_addr, 1128 desc_info->bufs[0].len, DMA_TO_DEVICE); 1129 return -EIO; 1130 } 1131 1132 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 1133 struct ionic_tx_desc_info *desc_info) 1134 { 1135 struct ionic_buf_info *buf_info = desc_info->bufs; 1136 struct device *dev = q->dev; 1137 unsigned int i; 1138 1139 if (!desc_info->nbufs) 1140 return; 1141 1142 dma_unmap_single(dev, buf_info->dma_addr, 1143 buf_info->len, DMA_TO_DEVICE); 1144 buf_info++; 1145 for (i = 1; i < desc_info->nbufs; i++, buf_info++) 1146 dma_unmap_page(dev, buf_info->dma_addr, 1147 buf_info->len, DMA_TO_DEVICE); 1148 1149 desc_info->nbufs = 0; 1150 } 1151 1152 static void ionic_tx_clean(struct ionic_queue *q, 1153 struct ionic_tx_desc_info *desc_info, 1154 struct ionic_txq_comp *comp) 1155 { 1156 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1157 struct ionic_qcq *qcq = q_to_qcq(q); 1158 struct sk_buff *skb; 1159 1160 if (desc_info->xdpf) { 1161 ionic_xdp_tx_desc_clean(q->partner, desc_info); 1162 stats->clean++; 1163 1164 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index))) 1165 netif_wake_subqueue(q->lif->netdev, q->index); 1166 1167 return; 1168 } 1169 1170 ionic_tx_desc_unmap_bufs(q, desc_info); 1171 1172 skb = desc_info->skb; 1173 if (!skb) 1174 return; 1175 1176 if (unlikely(ionic_txq_hwstamp_enabled(q))) { 1177 if (comp) { 1178 struct skb_shared_hwtstamps hwts = {}; 1179 __le64 *cq_desc_hwstamp; 1180 u64 hwstamp; 1181 1182 cq_desc_hwstamp = 1183 (void *)comp + 1184 qcq->cq.desc_size - 1185 sizeof(struct ionic_txq_comp) - 1186 IONIC_HWSTAMP_CQ_NEGOFFSET; 1187 1188 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 1189 1190 if (hwstamp != IONIC_HWSTAMP_INVALID) { 1191 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 1192 1193 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1194 skb_tstamp_tx(skb, &hwts); 1195 1196 stats->hwstamp_valid++; 1197 } else { 1198 stats->hwstamp_invalid++; 1199 } 1200 } 1201 } 1202 1203 desc_info->bytes = skb->len; 1204 stats->clean++; 1205 1206 napi_consume_skb(skb, 1); 1207 } 1208 1209 static bool ionic_tx_service(struct ionic_cq *cq, 1210 unsigned int *total_pkts, unsigned int *total_bytes) 1211 { 1212 struct ionic_tx_desc_info *desc_info; 1213 struct ionic_queue *q = cq->bound_q; 1214 struct ionic_txq_comp *comp; 1215 unsigned int bytes = 0; 1216 unsigned int pkts = 0; 1217 u16 index; 1218 1219 comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; 1220 1221 if (!color_match(comp->color, cq->done_color)) 1222 return false; 1223 1224 /* clean the related q entries, there could be 1225 * several q entries completed for each cq completion 1226 */ 1227 do { 1228 desc_info = &q->tx_info[q->tail_idx]; 1229 desc_info->bytes = 0; 1230 index = q->tail_idx; 1231 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1232 ionic_tx_clean(q, desc_info, comp); 1233 if (desc_info->skb) { 1234 pkts++; 1235 bytes += desc_info->bytes; 1236 desc_info->skb = NULL; 1237 } 1238 } while (index != le16_to_cpu(comp->comp_index)); 1239 1240 (*total_pkts) += pkts; 1241 (*total_bytes) += bytes; 1242 1243 return true; 1244 } 1245 1246 unsigned int ionic_tx_cq_service(struct ionic_cq *cq, unsigned int work_to_do) 1247 { 1248 unsigned int work_done = 0; 1249 unsigned int bytes = 0; 1250 unsigned int pkts = 0; 1251 1252 if (work_to_do == 0) 1253 return 0; 1254 1255 while (ionic_tx_service(cq, &pkts, &bytes)) { 1256 if (cq->tail_idx == cq->num_descs - 1) 1257 cq->done_color = !cq->done_color; 1258 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 1259 1260 if (++work_done >= work_to_do) 1261 break; 1262 } 1263 1264 if (work_done) { 1265 struct ionic_queue *q = cq->bound_q; 1266 1267 if (likely(!ionic_txq_hwstamp_enabled(q))) 1268 netif_txq_completed_wake(q_to_ndq(q->lif->netdev, q), 1269 pkts, bytes, 1270 ionic_q_space_avail(q), 1271 IONIC_TSO_DESCS_NEEDED); 1272 } 1273 1274 return work_done; 1275 } 1276 1277 void ionic_tx_flush(struct ionic_cq *cq) 1278 { 1279 u32 work_done; 1280 1281 work_done = ionic_tx_cq_service(cq, cq->num_descs); 1282 if (work_done) 1283 ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index, 1284 work_done, IONIC_INTR_CRED_RESET_COALESCE); 1285 } 1286 1287 void ionic_tx_empty(struct ionic_queue *q) 1288 { 1289 struct ionic_tx_desc_info *desc_info; 1290 int bytes = 0; 1291 int pkts = 0; 1292 1293 /* walk the not completed tx entries, if any */ 1294 while (q->head_idx != q->tail_idx) { 1295 desc_info = &q->tx_info[q->tail_idx]; 1296 desc_info->bytes = 0; 1297 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1298 ionic_tx_clean(q, desc_info, NULL); 1299 if (desc_info->skb) { 1300 pkts++; 1301 bytes += desc_info->bytes; 1302 desc_info->skb = NULL; 1303 } 1304 } 1305 1306 if (likely(!ionic_txq_hwstamp_enabled(q))) { 1307 struct netdev_queue *ndq = q_to_ndq(q->lif->netdev, q); 1308 1309 netdev_tx_completed_queue(ndq, pkts, bytes); 1310 netdev_tx_reset_queue(ndq); 1311 } 1312 } 1313 1314 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 1315 { 1316 int err; 1317 1318 err = skb_cow_head(skb, 0); 1319 if (err) 1320 return err; 1321 1322 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 1323 inner_ip_hdr(skb)->check = 0; 1324 inner_tcp_hdr(skb)->check = 1325 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 1326 inner_ip_hdr(skb)->daddr, 1327 0, IPPROTO_TCP, 0); 1328 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 1329 inner_tcp_hdr(skb)->check = 1330 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 1331 &inner_ipv6_hdr(skb)->daddr, 1332 0, IPPROTO_TCP, 0); 1333 } 1334 1335 return 0; 1336 } 1337 1338 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 1339 { 1340 int err; 1341 1342 err = skb_cow_head(skb, 0); 1343 if (err) 1344 return err; 1345 1346 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 1347 ip_hdr(skb)->check = 0; 1348 tcp_hdr(skb)->check = 1349 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 1350 ip_hdr(skb)->daddr, 1351 0, IPPROTO_TCP, 0); 1352 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 1353 tcp_v6_gso_csum_prep(skb); 1354 } 1355 1356 return 0; 1357 } 1358 1359 static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q, 1360 struct ionic_tx_desc_info *desc_info, 1361 struct sk_buff *skb, 1362 dma_addr_t addr, u8 nsge, u16 len, 1363 unsigned int hdrlen, unsigned int mss, 1364 bool outer_csum, 1365 u16 vlan_tci, bool has_vlan, 1366 bool start, bool done) 1367 { 1368 struct ionic_txq_desc *desc = &q->txq[q->head_idx]; 1369 u8 flags = 0; 1370 u64 cmd; 1371 1372 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1373 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1374 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 1375 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 1376 1377 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 1378 desc->cmd = cpu_to_le64(cmd); 1379 desc->len = cpu_to_le16(len); 1380 desc->vlan_tci = cpu_to_le16(vlan_tci); 1381 desc->hdr_len = cpu_to_le16(hdrlen); 1382 desc->mss = cpu_to_le16(mss); 1383 1384 ionic_write_cmb_desc(q, desc); 1385 1386 if (start) { 1387 skb_tx_timestamp(skb); 1388 if (likely(!ionic_txq_hwstamp_enabled(q))) 1389 netdev_tx_sent_queue(q_to_ndq(netdev, q), skb->len); 1390 ionic_txq_post(q, false); 1391 } else { 1392 ionic_txq_post(q, done); 1393 } 1394 } 1395 1396 static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q, 1397 struct sk_buff *skb) 1398 { 1399 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1400 struct ionic_tx_desc_info *desc_info; 1401 struct ionic_buf_info *buf_info; 1402 struct ionic_txq_sg_elem *elem; 1403 struct ionic_txq_desc *desc; 1404 unsigned int chunk_len; 1405 unsigned int frag_rem; 1406 unsigned int tso_rem; 1407 unsigned int seg_rem; 1408 dma_addr_t desc_addr; 1409 dma_addr_t frag_addr; 1410 unsigned int hdrlen; 1411 unsigned int len; 1412 unsigned int mss; 1413 bool start, done; 1414 bool outer_csum; 1415 bool has_vlan; 1416 u16 desc_len; 1417 u8 desc_nsge; 1418 u16 vlan_tci; 1419 bool encap; 1420 int err; 1421 1422 desc_info = &q->tx_info[q->head_idx]; 1423 1424 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1425 return -EIO; 1426 1427 len = skb->len; 1428 mss = skb_shinfo(skb)->gso_size; 1429 outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 1430 SKB_GSO_GRE_CSUM | 1431 SKB_GSO_IPXIP4 | 1432 SKB_GSO_IPXIP6 | 1433 SKB_GSO_UDP_TUNNEL | 1434 SKB_GSO_UDP_TUNNEL_CSUM)); 1435 has_vlan = !!skb_vlan_tag_present(skb); 1436 vlan_tci = skb_vlan_tag_get(skb); 1437 encap = skb->encapsulation; 1438 1439 /* Preload inner-most TCP csum field with IP pseudo hdr 1440 * calculated with IP length set to zero. HW will later 1441 * add in length to each TCP segment resulting from the TSO. 1442 */ 1443 1444 if (encap) 1445 err = ionic_tx_tcp_inner_pseudo_csum(skb); 1446 else 1447 err = ionic_tx_tcp_pseudo_csum(skb); 1448 if (err) { 1449 /* clean up mapping from ionic_tx_map_skb */ 1450 ionic_tx_desc_unmap_bufs(q, desc_info); 1451 return err; 1452 } 1453 1454 if (encap) 1455 hdrlen = skb_inner_tcp_all_headers(skb); 1456 else 1457 hdrlen = skb_tcp_all_headers(skb); 1458 1459 desc_info->skb = skb; 1460 buf_info = desc_info->bufs; 1461 tso_rem = len; 1462 seg_rem = min(tso_rem, hdrlen + mss); 1463 1464 frag_addr = 0; 1465 frag_rem = 0; 1466 1467 start = true; 1468 1469 while (tso_rem > 0) { 1470 desc = NULL; 1471 elem = NULL; 1472 desc_addr = 0; 1473 desc_len = 0; 1474 desc_nsge = 0; 1475 /* use fragments until we have enough to post a single descriptor */ 1476 while (seg_rem > 0) { 1477 /* if the fragment is exhausted then move to the next one */ 1478 if (frag_rem == 0) { 1479 /* grab the next fragment */ 1480 frag_addr = buf_info->dma_addr; 1481 frag_rem = buf_info->len; 1482 buf_info++; 1483 } 1484 chunk_len = min(frag_rem, seg_rem); 1485 if (!desc) { 1486 /* fill main descriptor */ 1487 desc = &q->txq[q->head_idx]; 1488 elem = ionic_tx_sg_elems(q); 1489 desc_addr = frag_addr; 1490 desc_len = chunk_len; 1491 } else { 1492 /* fill sg descriptor */ 1493 elem->addr = cpu_to_le64(frag_addr); 1494 elem->len = cpu_to_le16(chunk_len); 1495 elem++; 1496 desc_nsge++; 1497 } 1498 frag_addr += chunk_len; 1499 frag_rem -= chunk_len; 1500 tso_rem -= chunk_len; 1501 seg_rem -= chunk_len; 1502 } 1503 seg_rem = min(tso_rem, mss); 1504 done = (tso_rem == 0); 1505 /* post descriptor */ 1506 ionic_tx_tso_post(netdev, q, desc_info, skb, 1507 desc_addr, desc_nsge, desc_len, 1508 hdrlen, mss, outer_csum, vlan_tci, has_vlan, 1509 start, done); 1510 start = false; 1511 /* Buffer information is stored with the first tso descriptor */ 1512 desc_info = &q->tx_info[q->head_idx]; 1513 desc_info->nbufs = 0; 1514 } 1515 1516 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); 1517 stats->bytes += len; 1518 stats->tso++; 1519 stats->tso_bytes = len; 1520 1521 return 0; 1522 } 1523 1524 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, 1525 struct ionic_tx_desc_info *desc_info) 1526 { 1527 struct ionic_txq_desc *desc = &q->txq[q->head_idx]; 1528 struct ionic_buf_info *buf_info = desc_info->bufs; 1529 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1530 bool has_vlan; 1531 u8 flags = 0; 1532 bool encap; 1533 u64 cmd; 1534 1535 has_vlan = !!skb_vlan_tag_present(skb); 1536 encap = skb->encapsulation; 1537 1538 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1539 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1540 1541 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 1542 flags, skb_shinfo(skb)->nr_frags, 1543 buf_info->dma_addr); 1544 desc->cmd = cpu_to_le64(cmd); 1545 desc->len = cpu_to_le16(buf_info->len); 1546 if (has_vlan) { 1547 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1548 stats->vlan_inserted++; 1549 } else { 1550 desc->vlan_tci = 0; 1551 } 1552 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 1553 desc->csum_offset = cpu_to_le16(skb->csum_offset); 1554 1555 ionic_write_cmb_desc(q, desc); 1556 1557 if (skb_csum_is_sctp(skb)) 1558 stats->crc32_csum++; 1559 else 1560 stats->csum++; 1561 } 1562 1563 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, 1564 struct ionic_tx_desc_info *desc_info) 1565 { 1566 struct ionic_txq_desc *desc = &q->txq[q->head_idx]; 1567 struct ionic_buf_info *buf_info = desc_info->bufs; 1568 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1569 bool has_vlan; 1570 u8 flags = 0; 1571 bool encap; 1572 u64 cmd; 1573 1574 has_vlan = !!skb_vlan_tag_present(skb); 1575 encap = skb->encapsulation; 1576 1577 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1578 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1579 1580 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1581 flags, skb_shinfo(skb)->nr_frags, 1582 buf_info->dma_addr); 1583 desc->cmd = cpu_to_le64(cmd); 1584 desc->len = cpu_to_le16(buf_info->len); 1585 if (has_vlan) { 1586 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1587 stats->vlan_inserted++; 1588 } else { 1589 desc->vlan_tci = 0; 1590 } 1591 desc->csum_start = 0; 1592 desc->csum_offset = 0; 1593 1594 ionic_write_cmb_desc(q, desc); 1595 1596 stats->csum_none++; 1597 } 1598 1599 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, 1600 struct ionic_tx_desc_info *desc_info) 1601 { 1602 struct ionic_buf_info *buf_info = &desc_info->bufs[1]; 1603 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1604 struct ionic_txq_sg_elem *elem; 1605 unsigned int i; 1606 1607 elem = ionic_tx_sg_elems(q); 1608 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { 1609 elem->addr = cpu_to_le64(buf_info->dma_addr); 1610 elem->len = cpu_to_le16(buf_info->len); 1611 } 1612 1613 stats->frags += skb_shinfo(skb)->nr_frags; 1614 } 1615 1616 static int ionic_tx(struct net_device *netdev, struct ionic_queue *q, 1617 struct sk_buff *skb) 1618 { 1619 struct ionic_tx_desc_info *desc_info = &q->tx_info[q->head_idx]; 1620 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1621 bool ring_dbell = true; 1622 1623 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1624 return -EIO; 1625 1626 desc_info->skb = skb; 1627 1628 /* set up the initial descriptor */ 1629 if (skb->ip_summed == CHECKSUM_PARTIAL) 1630 ionic_tx_calc_csum(q, skb, desc_info); 1631 else 1632 ionic_tx_calc_no_csum(q, skb, desc_info); 1633 1634 /* add frags */ 1635 ionic_tx_skb_frags(q, skb, desc_info); 1636 1637 skb_tx_timestamp(skb); 1638 stats->pkts++; 1639 stats->bytes += skb->len; 1640 1641 if (likely(!ionic_txq_hwstamp_enabled(q))) { 1642 struct netdev_queue *ndq = q_to_ndq(netdev, q); 1643 1644 if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1))) 1645 netif_tx_stop_queue(ndq); 1646 ring_dbell = __netdev_tx_sent_queue(ndq, skb->len, 1647 netdev_xmit_more()); 1648 } 1649 ionic_txq_post(q, ring_dbell); 1650 1651 return 0; 1652 } 1653 1654 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 1655 { 1656 int nr_frags = skb_shinfo(skb)->nr_frags; 1657 bool too_many_frags = false; 1658 skb_frag_t *frag; 1659 int desc_bufs; 1660 int chunk_len; 1661 int frag_rem; 1662 int tso_rem; 1663 int seg_rem; 1664 bool encap; 1665 int hdrlen; 1666 int ndescs; 1667 int err; 1668 1669 /* Each desc is mss long max, so a descriptor for each gso_seg */ 1670 if (skb_is_gso(skb)) { 1671 ndescs = skb_shinfo(skb)->gso_segs; 1672 if (!nr_frags) 1673 return ndescs; 1674 } else { 1675 ndescs = 1; 1676 if (!nr_frags) 1677 return ndescs; 1678 1679 if (unlikely(nr_frags > q->max_sg_elems)) { 1680 too_many_frags = true; 1681 goto linearize; 1682 } 1683 1684 return ndescs; 1685 } 1686 1687 /* We need to scan the skb to be sure that none of the MTU sized 1688 * packets in the TSO will require more sgs per descriptor than we 1689 * can support. We loop through the frags, add up the lengths for 1690 * a packet, and count the number of sgs used per packet. 1691 */ 1692 tso_rem = skb->len; 1693 frag = skb_shinfo(skb)->frags; 1694 encap = skb->encapsulation; 1695 1696 /* start with just hdr in first part of first descriptor */ 1697 if (encap) 1698 hdrlen = skb_inner_tcp_all_headers(skb); 1699 else 1700 hdrlen = skb_tcp_all_headers(skb); 1701 seg_rem = min_t(int, tso_rem, hdrlen + skb_shinfo(skb)->gso_size); 1702 frag_rem = hdrlen; 1703 1704 while (tso_rem > 0) { 1705 desc_bufs = 0; 1706 while (seg_rem > 0) { 1707 desc_bufs++; 1708 1709 /* We add the +1 because we can take buffers for one 1710 * more than we have SGs: one for the initial desc data 1711 * in addition to the SG segments that might follow. 1712 */ 1713 if (desc_bufs > q->max_sg_elems + 1) { 1714 too_many_frags = true; 1715 goto linearize; 1716 } 1717 1718 if (frag_rem == 0) { 1719 frag_rem = skb_frag_size(frag); 1720 frag++; 1721 } 1722 chunk_len = min(frag_rem, seg_rem); 1723 frag_rem -= chunk_len; 1724 tso_rem -= chunk_len; 1725 seg_rem -= chunk_len; 1726 } 1727 1728 seg_rem = min_t(int, tso_rem, skb_shinfo(skb)->gso_size); 1729 } 1730 1731 linearize: 1732 if (too_many_frags) { 1733 err = skb_linearize(skb); 1734 if (err) 1735 return err; 1736 q_to_tx_stats(q)->linearize++; 1737 } 1738 1739 return ndescs; 1740 } 1741 1742 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, 1743 struct net_device *netdev) 1744 { 1745 struct ionic_lif *lif = netdev_priv(netdev); 1746 struct ionic_queue *q; 1747 int err, ndescs; 1748 1749 /* Does not stop/start txq, because we post to a separate tx queue 1750 * for timestamping, and if a packet can't be posted immediately to 1751 * the timestamping queue, it is dropped. 1752 */ 1753 1754 q = &lif->hwstamp_txq->q; 1755 ndescs = ionic_tx_descs_needed(q, skb); 1756 if (unlikely(ndescs < 0)) 1757 goto err_out_drop; 1758 1759 if (unlikely(!ionic_q_has_space(q, ndescs))) 1760 goto err_out_drop; 1761 1762 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP; 1763 if (skb_is_gso(skb)) 1764 err = ionic_tx_tso(netdev, q, skb); 1765 else 1766 err = ionic_tx(netdev, q, skb); 1767 1768 if (err) 1769 goto err_out_drop; 1770 1771 return NETDEV_TX_OK; 1772 1773 err_out_drop: 1774 q->drop++; 1775 dev_kfree_skb(skb); 1776 return NETDEV_TX_OK; 1777 } 1778 1779 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1780 { 1781 u16 queue_index = skb_get_queue_mapping(skb); 1782 struct ionic_lif *lif = netdev_priv(netdev); 1783 struct ionic_queue *q; 1784 int ndescs; 1785 int err; 1786 1787 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 1788 dev_kfree_skb(skb); 1789 return NETDEV_TX_OK; 1790 } 1791 1792 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 1793 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode) 1794 return ionic_start_hwstamp_xmit(skb, netdev); 1795 1796 if (unlikely(queue_index >= lif->nxqs)) 1797 queue_index = 0; 1798 q = &lif->txqcqs[queue_index]->q; 1799 1800 ndescs = ionic_tx_descs_needed(q, skb); 1801 if (ndescs < 0) 1802 goto err_out_drop; 1803 1804 if (!netif_txq_maybe_stop(q_to_ndq(netdev, q), 1805 ionic_q_space_avail(q), 1806 ndescs, ndescs)) 1807 return NETDEV_TX_BUSY; 1808 1809 if (skb_is_gso(skb)) 1810 err = ionic_tx_tso(netdev, q, skb); 1811 else 1812 err = ionic_tx(netdev, q, skb); 1813 1814 if (err) 1815 goto err_out_drop; 1816 1817 return NETDEV_TX_OK; 1818 1819 err_out_drop: 1820 q->drop++; 1821 dev_kfree_skb(skb); 1822 return NETDEV_TX_OK; 1823 } 1824