1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip6_checksum.h> 8 #include <net/netdev_queues.h> 9 10 #include "ionic.h" 11 #include "ionic_lif.h" 12 #include "ionic_txrx.h" 13 14 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 15 void *data, size_t len); 16 17 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 18 const skb_frag_t *frag, 19 size_t offset, size_t len); 20 21 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 22 struct ionic_tx_desc_info *desc_info); 23 24 static void ionic_tx_clean(struct ionic_queue *q, 25 struct ionic_tx_desc_info *desc_info, 26 struct ionic_txq_comp *comp, 27 bool in_napi); 28 29 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell) 30 { 31 ionic_q_post(q, ring_dbell); 32 } 33 34 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell) 35 { 36 ionic_q_post(q, ring_dbell); 37 } 38 39 bool ionic_txq_poke_doorbell(struct ionic_queue *q) 40 { 41 struct netdev_queue *netdev_txq; 42 unsigned long now, then, dif; 43 struct net_device *netdev; 44 45 netdev = q->lif->netdev; 46 netdev_txq = netdev_get_tx_queue(netdev, q->index); 47 48 HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id()); 49 50 if (q->tail_idx == q->head_idx) { 51 HARD_TX_UNLOCK(netdev, netdev_txq); 52 return false; 53 } 54 55 now = READ_ONCE(jiffies); 56 then = q->dbell_jiffies; 57 dif = now - then; 58 59 if (dif > q->dbell_deadline) { 60 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 61 q->dbval | q->head_idx); 62 63 q->dbell_jiffies = now; 64 } 65 66 HARD_TX_UNLOCK(netdev, netdev_txq); 67 68 return true; 69 } 70 71 bool ionic_rxq_poke_doorbell(struct ionic_queue *q) 72 { 73 unsigned long now, then, dif; 74 75 /* no lock, called from rx napi or txrx napi, nothing else can fill */ 76 77 if (q->tail_idx == q->head_idx) 78 return false; 79 80 now = READ_ONCE(jiffies); 81 then = q->dbell_jiffies; 82 dif = now - then; 83 84 if (dif > q->dbell_deadline) { 85 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 86 q->dbval | q->head_idx); 87 88 q->dbell_jiffies = now; 89 90 dif = 2 * q->dbell_deadline; 91 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE) 92 dif = IONIC_RX_MAX_DOORBELL_DEADLINE; 93 94 q->dbell_deadline = dif; 95 } 96 97 return true; 98 } 99 100 static inline struct ionic_txq_sg_elem *ionic_tx_sg_elems(struct ionic_queue *q) 101 { 102 if (likely(q->sg_desc_size == sizeof(struct ionic_txq_sg_desc_v1))) 103 return q->txq_sgl_v1[q->head_idx].elems; 104 else 105 return q->txq_sgl[q->head_idx].elems; 106 } 107 108 static inline struct netdev_queue *q_to_ndq(struct net_device *netdev, 109 struct ionic_queue *q) 110 { 111 return netdev_get_tx_queue(netdev, q->index); 112 } 113 114 static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info) 115 { 116 return page_address(buf_info->page) + buf_info->page_offset; 117 } 118 119 static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info) 120 { 121 return buf_info->dma_addr + buf_info->page_offset; 122 } 123 124 static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info) 125 { 126 return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset); 127 } 128 129 static int ionic_rx_page_alloc(struct ionic_queue *q, 130 struct ionic_buf_info *buf_info) 131 { 132 struct device *dev = q->dev; 133 dma_addr_t dma_addr; 134 struct page *page; 135 136 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 137 if (unlikely(!page)) { 138 net_err_ratelimited("%s: %s page alloc failed\n", 139 dev_name(dev), q->name); 140 q_to_rx_stats(q)->alloc_err++; 141 return -ENOMEM; 142 } 143 144 dma_addr = dma_map_page(dev, page, 0, 145 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 146 if (unlikely(dma_mapping_error(dev, dma_addr))) { 147 __free_pages(page, 0); 148 net_err_ratelimited("%s: %s dma map failed\n", 149 dev_name(dev), q->name); 150 q_to_rx_stats(q)->dma_map_err++; 151 return -EIO; 152 } 153 154 buf_info->dma_addr = dma_addr; 155 buf_info->page = page; 156 buf_info->page_offset = 0; 157 158 return 0; 159 } 160 161 static void ionic_rx_page_free(struct ionic_queue *q, 162 struct ionic_buf_info *buf_info) 163 { 164 struct device *dev = q->dev; 165 166 if (unlikely(!buf_info)) { 167 net_err_ratelimited("%s: %s invalid buf_info in free\n", 168 dev_name(dev), q->name); 169 return; 170 } 171 172 if (!buf_info->page) 173 return; 174 175 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 176 __free_pages(buf_info->page, 0); 177 buf_info->page = NULL; 178 } 179 180 static bool ionic_rx_buf_recycle(struct ionic_queue *q, 181 struct ionic_buf_info *buf_info, u32 len) 182 { 183 u32 size; 184 185 /* don't re-use pages allocated in low-mem condition */ 186 if (page_is_pfmemalloc(buf_info->page)) 187 return false; 188 189 /* don't re-use buffers from non-local numa nodes */ 190 if (page_to_nid(buf_info->page) != numa_mem_id()) 191 return false; 192 193 size = ALIGN(len, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ); 194 buf_info->page_offset += size; 195 if (buf_info->page_offset >= IONIC_PAGE_SIZE) 196 return false; 197 198 get_page(buf_info->page); 199 200 return true; 201 } 202 203 static void ionic_rx_add_skb_frag(struct ionic_queue *q, 204 struct sk_buff *skb, 205 struct ionic_buf_info *buf_info, 206 u32 off, u32 len, 207 bool synced) 208 { 209 if (!synced) 210 dma_sync_single_range_for_cpu(q->dev, ionic_rx_buf_pa(buf_info), 211 off, len, DMA_FROM_DEVICE); 212 213 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 214 buf_info->page, buf_info->page_offset + off, 215 len, 216 IONIC_PAGE_SIZE); 217 218 if (!ionic_rx_buf_recycle(q, buf_info, len)) { 219 dma_unmap_page(q->dev, buf_info->dma_addr, 220 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 221 buf_info->page = NULL; 222 } 223 } 224 225 static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q, 226 struct ionic_rx_desc_info *desc_info, 227 unsigned int headroom, 228 unsigned int len, 229 unsigned int num_sg_elems, 230 bool synced) 231 { 232 struct ionic_buf_info *buf_info; 233 struct sk_buff *skb; 234 unsigned int i; 235 u16 frag_len; 236 237 buf_info = &desc_info->bufs[0]; 238 prefetchw(buf_info->page); 239 240 skb = napi_get_frags(&q_to_qcq(q)->napi); 241 if (unlikely(!skb)) { 242 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 243 dev_name(q->dev), q->name); 244 q_to_rx_stats(q)->alloc_err++; 245 return NULL; 246 } 247 248 if (headroom) 249 frag_len = min_t(u16, len, 250 IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 251 else 252 frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 253 254 if (unlikely(!buf_info->page)) 255 goto err_bad_buf_page; 256 ionic_rx_add_skb_frag(q, skb, buf_info, headroom, frag_len, synced); 257 len -= frag_len; 258 buf_info++; 259 260 for (i = 0; i < num_sg_elems; i++, buf_info++) { 261 if (unlikely(!buf_info->page)) 262 goto err_bad_buf_page; 263 frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 264 ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced); 265 len -= frag_len; 266 } 267 268 return skb; 269 270 err_bad_buf_page: 271 dev_kfree_skb(skb); 272 return NULL; 273 } 274 275 static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev, 276 struct ionic_queue *q, 277 struct ionic_rx_desc_info *desc_info, 278 unsigned int headroom, 279 unsigned int len, 280 bool synced) 281 { 282 struct ionic_buf_info *buf_info; 283 struct device *dev = q->dev; 284 struct sk_buff *skb; 285 286 buf_info = &desc_info->bufs[0]; 287 288 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 289 if (unlikely(!skb)) { 290 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 291 dev_name(dev), q->name); 292 q_to_rx_stats(q)->alloc_err++; 293 return NULL; 294 } 295 296 if (unlikely(!buf_info->page)) { 297 dev_kfree_skb(skb); 298 return NULL; 299 } 300 301 if (!synced) 302 dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 303 headroom, len, DMA_FROM_DEVICE); 304 skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); 305 dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info), 306 headroom, len, DMA_FROM_DEVICE); 307 308 skb_put(skb, len); 309 skb->protocol = eth_type_trans(skb, netdev); 310 311 return skb; 312 } 313 314 static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, 315 struct ionic_tx_desc_info *desc_info) 316 { 317 unsigned int nbufs = desc_info->nbufs; 318 struct ionic_buf_info *buf_info; 319 struct device *dev = q->dev; 320 int i; 321 322 if (!nbufs) 323 return; 324 325 buf_info = desc_info->bufs; 326 dma_unmap_single(dev, buf_info->dma_addr, 327 buf_info->len, DMA_TO_DEVICE); 328 if (desc_info->act == XDP_TX) 329 __free_pages(buf_info->page, 0); 330 buf_info->page = NULL; 331 332 buf_info++; 333 for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) { 334 dma_unmap_page(dev, buf_info->dma_addr, 335 buf_info->len, DMA_TO_DEVICE); 336 if (desc_info->act == XDP_TX) 337 __free_pages(buf_info->page, 0); 338 buf_info->page = NULL; 339 } 340 341 if (desc_info->act == XDP_REDIRECT) 342 xdp_return_frame(desc_info->xdpf); 343 344 desc_info->nbufs = 0; 345 desc_info->xdpf = NULL; 346 desc_info->act = 0; 347 } 348 349 static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame, 350 enum xdp_action act, struct page *page, int off, 351 bool ring_doorbell) 352 { 353 struct ionic_tx_desc_info *desc_info; 354 struct ionic_buf_info *buf_info; 355 struct ionic_tx_stats *stats; 356 struct ionic_txq_desc *desc; 357 size_t len = frame->len; 358 dma_addr_t dma_addr; 359 u64 cmd; 360 361 desc_info = &q->tx_info[q->head_idx]; 362 desc = &q->txq[q->head_idx]; 363 buf_info = desc_info->bufs; 364 stats = q_to_tx_stats(q); 365 366 dma_addr = ionic_tx_map_single(q, frame->data, len); 367 if (!dma_addr) 368 return -EIO; 369 buf_info->dma_addr = dma_addr; 370 buf_info->len = len; 371 buf_info->page = page; 372 buf_info->page_offset = off; 373 374 desc_info->nbufs = 1; 375 desc_info->xdpf = frame; 376 desc_info->act = act; 377 378 if (xdp_frame_has_frags(frame)) { 379 struct ionic_txq_sg_elem *elem; 380 struct skb_shared_info *sinfo; 381 struct ionic_buf_info *bi; 382 skb_frag_t *frag; 383 int i; 384 385 bi = &buf_info[1]; 386 sinfo = xdp_get_shared_info_from_frame(frame); 387 frag = sinfo->frags; 388 elem = ionic_tx_sg_elems(q); 389 for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) { 390 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 391 if (!dma_addr) { 392 ionic_tx_desc_unmap_bufs(q, desc_info); 393 return -EIO; 394 } 395 bi->dma_addr = dma_addr; 396 bi->len = skb_frag_size(frag); 397 bi->page = skb_frag_page(frag); 398 399 elem->addr = cpu_to_le64(bi->dma_addr); 400 elem->len = cpu_to_le16(bi->len); 401 elem++; 402 403 desc_info->nbufs++; 404 } 405 } 406 407 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 408 0, (desc_info->nbufs - 1), buf_info->dma_addr); 409 desc->cmd = cpu_to_le64(cmd); 410 desc->len = cpu_to_le16(len); 411 desc->csum_start = 0; 412 desc->csum_offset = 0; 413 414 stats->xdp_frames++; 415 stats->pkts++; 416 stats->bytes += len; 417 418 ionic_txq_post(q, ring_doorbell); 419 420 return 0; 421 } 422 423 int ionic_xdp_xmit(struct net_device *netdev, int n, 424 struct xdp_frame **xdp_frames, u32 flags) 425 { 426 struct ionic_lif *lif = netdev_priv(netdev); 427 struct ionic_queue *txq; 428 struct netdev_queue *nq; 429 int nxmit; 430 int space; 431 int cpu; 432 int qi; 433 434 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) 435 return -ENETDOWN; 436 437 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 438 return -EINVAL; 439 440 /* AdminQ is assumed on cpu 0, while we attempt to affinitize the 441 * TxRx queue pairs 0..n-1 on cpus 1..n. We try to keep with that 442 * affinitization here, but of course irqbalance and friends might 443 * have juggled things anyway, so we have to check for the 0 case. 444 */ 445 cpu = smp_processor_id(); 446 qi = cpu ? (cpu - 1) % lif->nxqs : cpu; 447 448 txq = &lif->txqcqs[qi]->q; 449 nq = netdev_get_tx_queue(netdev, txq->index); 450 __netif_tx_lock(nq, cpu); 451 txq_trans_cond_update(nq); 452 453 if (netif_tx_queue_stopped(nq) || 454 !netif_txq_maybe_stop(q_to_ndq(netdev, txq), 455 ionic_q_space_avail(txq), 456 1, 1)) { 457 __netif_tx_unlock(nq); 458 return -EIO; 459 } 460 461 space = min_t(int, n, ionic_q_space_avail(txq)); 462 for (nxmit = 0; nxmit < space ; nxmit++) { 463 if (ionic_xdp_post_frame(txq, xdp_frames[nxmit], 464 XDP_REDIRECT, 465 virt_to_page(xdp_frames[nxmit]->data), 466 0, false)) { 467 nxmit--; 468 break; 469 } 470 } 471 472 if (flags & XDP_XMIT_FLUSH) 473 ionic_dbell_ring(lif->kern_dbpage, txq->hw_type, 474 txq->dbval | txq->head_idx); 475 476 netif_txq_maybe_stop(q_to_ndq(netdev, txq), 477 ionic_q_space_avail(txq), 478 4, 4); 479 __netif_tx_unlock(nq); 480 481 return nxmit; 482 } 483 484 static void ionic_xdp_rx_put_bufs(struct ionic_queue *q, 485 struct ionic_buf_info *buf_info, 486 int nbufs) 487 { 488 int i; 489 490 for (i = 0; i < nbufs; i++) { 491 dma_unmap_page(q->dev, buf_info->dma_addr, 492 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 493 buf_info->page = NULL; 494 buf_info++; 495 } 496 } 497 498 static bool ionic_run_xdp(struct ionic_rx_stats *stats, 499 struct net_device *netdev, 500 struct bpf_prog *xdp_prog, 501 struct ionic_queue *rxq, 502 struct ionic_buf_info *buf_info, 503 int len) 504 { 505 u32 xdp_action = XDP_ABORTED; 506 struct xdp_buff xdp_buf; 507 struct ionic_queue *txq; 508 struct netdev_queue *nq; 509 struct xdp_frame *xdpf; 510 int remain_len; 511 int nbufs = 1; 512 int frag_len; 513 int err = 0; 514 515 xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info); 516 frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 517 xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), 518 XDP_PACKET_HEADROOM, frag_len, false); 519 520 dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info), 521 XDP_PACKET_HEADROOM, frag_len, 522 DMA_FROM_DEVICE); 523 524 prefetchw(&xdp_buf.data_hard_start); 525 526 /* We limit MTU size to one buffer if !xdp_has_frags, so 527 * if the recv len is bigger than one buffer 528 * then we know we have frag info to gather 529 */ 530 remain_len = len - frag_len; 531 if (remain_len) { 532 struct skb_shared_info *sinfo; 533 struct ionic_buf_info *bi; 534 skb_frag_t *frag; 535 536 bi = buf_info; 537 sinfo = xdp_get_shared_info_from_buff(&xdp_buf); 538 sinfo->nr_frags = 0; 539 sinfo->xdp_frags_size = 0; 540 xdp_buff_set_frags_flag(&xdp_buf); 541 542 do { 543 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) { 544 err = -ENOSPC; 545 goto out_xdp_abort; 546 } 547 548 frag = &sinfo->frags[sinfo->nr_frags]; 549 sinfo->nr_frags++; 550 bi++; 551 frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi)); 552 dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi), 553 0, frag_len, DMA_FROM_DEVICE); 554 skb_frag_fill_page_desc(frag, bi->page, 0, frag_len); 555 sinfo->xdp_frags_size += frag_len; 556 remain_len -= frag_len; 557 558 if (page_is_pfmemalloc(bi->page)) 559 xdp_buff_set_frag_pfmemalloc(&xdp_buf); 560 } while (remain_len > 0); 561 nbufs += sinfo->nr_frags; 562 } 563 564 xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf); 565 566 switch (xdp_action) { 567 case XDP_PASS: 568 stats->xdp_pass++; 569 return false; /* false = we didn't consume the packet */ 570 571 case XDP_DROP: 572 ionic_rx_page_free(rxq, buf_info); 573 stats->xdp_drop++; 574 break; 575 576 case XDP_TX: 577 xdpf = xdp_convert_buff_to_frame(&xdp_buf); 578 if (!xdpf) 579 goto out_xdp_abort; 580 581 txq = rxq->partner; 582 nq = netdev_get_tx_queue(netdev, txq->index); 583 __netif_tx_lock(nq, smp_processor_id()); 584 txq_trans_cond_update(nq); 585 586 if (netif_tx_queue_stopped(nq) || 587 !netif_txq_maybe_stop(q_to_ndq(netdev, txq), 588 ionic_q_space_avail(txq), 589 1, 1)) { 590 __netif_tx_unlock(nq); 591 goto out_xdp_abort; 592 } 593 594 err = ionic_xdp_post_frame(txq, xdpf, XDP_TX, 595 buf_info->page, 596 buf_info->page_offset, 597 true); 598 __netif_tx_unlock(nq); 599 if (unlikely(err)) { 600 netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); 601 goto out_xdp_abort; 602 } 603 ionic_xdp_rx_put_bufs(rxq, buf_info, nbufs); 604 stats->xdp_tx++; 605 606 /* the Tx completion will free the buffers */ 607 break; 608 609 case XDP_REDIRECT: 610 err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog); 611 if (unlikely(err)) { 612 netdev_dbg(netdev, "xdp_do_redirect err %d\n", err); 613 goto out_xdp_abort; 614 } 615 ionic_xdp_rx_put_bufs(rxq, buf_info, nbufs); 616 rxq->xdp_flush = true; 617 stats->xdp_redirect++; 618 break; 619 620 case XDP_ABORTED: 621 default: 622 goto out_xdp_abort; 623 } 624 625 return true; 626 627 out_xdp_abort: 628 trace_xdp_exception(netdev, xdp_prog, xdp_action); 629 ionic_rx_page_free(rxq, buf_info); 630 stats->xdp_aborted++; 631 632 return true; 633 } 634 635 static void ionic_rx_clean(struct ionic_queue *q, 636 struct ionic_rx_desc_info *desc_info, 637 struct ionic_rxq_comp *comp) 638 { 639 struct net_device *netdev = q->lif->netdev; 640 struct ionic_qcq *qcq = q_to_qcq(q); 641 struct ionic_rx_stats *stats; 642 struct bpf_prog *xdp_prog; 643 unsigned int headroom; 644 struct sk_buff *skb; 645 bool synced = false; 646 bool use_copybreak; 647 u16 len; 648 649 stats = q_to_rx_stats(q); 650 651 if (comp->status) { 652 stats->dropped++; 653 return; 654 } 655 656 len = le16_to_cpu(comp->len); 657 stats->pkts++; 658 stats->bytes += len; 659 660 xdp_prog = READ_ONCE(q->lif->xdp_prog); 661 if (xdp_prog) { 662 if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len)) 663 return; 664 synced = true; 665 } 666 667 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 668 use_copybreak = len <= q->lif->rx_copybreak; 669 if (use_copybreak) 670 skb = ionic_rx_copybreak(netdev, q, desc_info, 671 headroom, len, synced); 672 else 673 skb = ionic_rx_build_skb(q, desc_info, headroom, len, 674 comp->num_sg_elems, synced); 675 676 if (unlikely(!skb)) { 677 stats->dropped++; 678 return; 679 } 680 681 skb_record_rx_queue(skb, q->index); 682 683 if (likely(netdev->features & NETIF_F_RXHASH)) { 684 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 685 case IONIC_PKT_TYPE_IPV4: 686 case IONIC_PKT_TYPE_IPV6: 687 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 688 PKT_HASH_TYPE_L3); 689 break; 690 case IONIC_PKT_TYPE_IPV4_TCP: 691 case IONIC_PKT_TYPE_IPV6_TCP: 692 case IONIC_PKT_TYPE_IPV4_UDP: 693 case IONIC_PKT_TYPE_IPV6_UDP: 694 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 695 PKT_HASH_TYPE_L4); 696 break; 697 } 698 } 699 700 if (likely(netdev->features & NETIF_F_RXCSUM) && 701 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) { 702 skb->ip_summed = CHECKSUM_COMPLETE; 703 skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 704 stats->csum_complete++; 705 } else { 706 stats->csum_none++; 707 } 708 709 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 710 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 711 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 712 stats->csum_error++; 713 714 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 715 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 716 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 717 le16_to_cpu(comp->vlan_tci)); 718 stats->vlan_stripped++; 719 } 720 721 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) { 722 __le64 *cq_desc_hwstamp; 723 u64 hwstamp; 724 725 cq_desc_hwstamp = 726 (void *)comp + 727 qcq->cq.desc_size - 728 sizeof(struct ionic_rxq_comp) - 729 IONIC_HWSTAMP_CQ_NEGOFFSET; 730 731 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 732 733 if (hwstamp != IONIC_HWSTAMP_INVALID) { 734 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 735 stats->hwstamp_valid++; 736 } else { 737 stats->hwstamp_invalid++; 738 } 739 } 740 741 if (use_copybreak) 742 napi_gro_receive(&qcq->napi, skb); 743 else 744 napi_gro_frags(&qcq->napi); 745 } 746 747 bool ionic_rx_service(struct ionic_cq *cq) 748 { 749 struct ionic_rx_desc_info *desc_info; 750 struct ionic_queue *q = cq->bound_q; 751 struct ionic_rxq_comp *comp; 752 753 comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; 754 755 if (!color_match(comp->pkt_type_color, cq->done_color)) 756 return false; 757 758 /* check for empty queue */ 759 if (q->tail_idx == q->head_idx) 760 return false; 761 762 if (q->tail_idx != le16_to_cpu(comp->comp_index)) 763 return false; 764 765 desc_info = &q->rx_info[q->tail_idx]; 766 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 767 768 /* clean the related q entry, only one per qc completion */ 769 ionic_rx_clean(q, desc_info, comp); 770 771 return true; 772 } 773 774 static inline void ionic_write_cmb_desc(struct ionic_queue *q, 775 void *desc) 776 { 777 /* Since Rx and Tx descriptors are the same size, we can 778 * save an instruction or two and skip the qtype check. 779 */ 780 if (unlikely(q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS)) 781 memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0])); 782 } 783 784 void ionic_rx_fill(struct ionic_queue *q) 785 { 786 struct net_device *netdev = q->lif->netdev; 787 struct ionic_rx_desc_info *desc_info; 788 struct ionic_rxq_sg_elem *sg_elem; 789 struct ionic_buf_info *buf_info; 790 unsigned int fill_threshold; 791 struct ionic_rxq_desc *desc; 792 unsigned int remain_len; 793 unsigned int frag_len; 794 unsigned int nfrags; 795 unsigned int n_fill; 796 unsigned int len; 797 unsigned int i; 798 unsigned int j; 799 800 n_fill = ionic_q_space_avail(q); 801 802 fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD, 803 q->num_descs / IONIC_RX_FILL_DIV); 804 if (n_fill < fill_threshold) 805 return; 806 807 len = netdev->mtu + VLAN_ETH_HLEN; 808 809 for (i = n_fill; i; i--) { 810 unsigned int headroom; 811 unsigned int buf_len; 812 813 nfrags = 0; 814 remain_len = len; 815 desc = &q->rxq[q->head_idx]; 816 desc_info = &q->rx_info[q->head_idx]; 817 buf_info = &desc_info->bufs[0]; 818 819 if (!buf_info->page) { /* alloc a new buffer? */ 820 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 821 desc->addr = 0; 822 desc->len = 0; 823 return; 824 } 825 } 826 827 /* fill main descriptor - buf[0] 828 * XDP uses space in the first buffer, so account for 829 * head room, tail room, and ip header in the first frag size. 830 */ 831 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 832 if (q->xdp_rxq_info) 833 buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN; 834 else 835 buf_len = ionic_rx_buf_size(buf_info); 836 frag_len = min_t(u16, len, buf_len); 837 838 desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom); 839 desc->len = cpu_to_le16(frag_len); 840 remain_len -= frag_len; 841 buf_info++; 842 nfrags++; 843 844 /* fill sg descriptors - buf[1..n] */ 845 sg_elem = q->rxq_sgl[q->head_idx].elems; 846 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) { 847 if (!buf_info->page) { /* alloc a new sg buffer? */ 848 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 849 sg_elem->addr = 0; 850 sg_elem->len = 0; 851 return; 852 } 853 } 854 855 sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info)); 856 frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info)); 857 sg_elem->len = cpu_to_le16(frag_len); 858 remain_len -= frag_len; 859 buf_info++; 860 nfrags++; 861 } 862 863 /* clear end sg element as a sentinel */ 864 if (j < q->max_sg_elems) 865 memset(sg_elem, 0, sizeof(*sg_elem)); 866 867 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 868 IONIC_RXQ_DESC_OPCODE_SIMPLE; 869 desc_info->nbufs = nfrags; 870 871 ionic_write_cmb_desc(q, desc); 872 873 ionic_rxq_post(q, false); 874 } 875 876 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 877 q->dbval | q->head_idx); 878 879 q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE; 880 q->dbell_jiffies = jiffies; 881 } 882 883 void ionic_rx_empty(struct ionic_queue *q) 884 { 885 struct ionic_rx_desc_info *desc_info; 886 struct ionic_buf_info *buf_info; 887 unsigned int i, j; 888 889 for (i = 0; i < q->num_descs; i++) { 890 desc_info = &q->rx_info[i]; 891 for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) { 892 buf_info = &desc_info->bufs[j]; 893 if (buf_info->page) 894 ionic_rx_page_free(q, buf_info); 895 } 896 897 desc_info->nbufs = 0; 898 } 899 900 q->head_idx = 0; 901 q->tail_idx = 0; 902 } 903 904 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode) 905 { 906 struct dim_sample dim_sample; 907 struct ionic_lif *lif; 908 unsigned int qi; 909 u64 pkts, bytes; 910 911 if (!qcq->intr.dim_coal_hw) 912 return; 913 914 lif = qcq->q.lif; 915 qi = qcq->cq.bound_q->index; 916 917 switch (napi_mode) { 918 case IONIC_LIF_F_TX_DIM_INTR: 919 pkts = lif->txqstats[qi].pkts; 920 bytes = lif->txqstats[qi].bytes; 921 break; 922 case IONIC_LIF_F_RX_DIM_INTR: 923 pkts = lif->rxqstats[qi].pkts; 924 bytes = lif->rxqstats[qi].bytes; 925 break; 926 default: 927 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts; 928 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes; 929 break; 930 } 931 932 dim_update_sample(qcq->cq.bound_intr->rearm_count, 933 pkts, bytes, &dim_sample); 934 935 net_dim(&qcq->dim, dim_sample); 936 } 937 938 int ionic_tx_napi(struct napi_struct *napi, int budget) 939 { 940 struct ionic_qcq *qcq = napi_to_qcq(napi); 941 struct ionic_cq *cq = napi_to_cq(napi); 942 u32 work_done = 0; 943 u32 flags = 0; 944 945 work_done = ionic_tx_cq_service(cq, budget, !!budget); 946 947 if (unlikely(!budget)) 948 return budget; 949 950 if (work_done < budget && napi_complete_done(napi, work_done)) { 951 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR); 952 flags |= IONIC_INTR_CRED_UNMASK; 953 cq->bound_intr->rearm_count++; 954 } 955 956 if (work_done || flags) { 957 flags |= IONIC_INTR_CRED_RESET_COALESCE; 958 ionic_intr_credits(cq->idev->intr_ctrl, 959 cq->bound_intr->index, 960 work_done, flags); 961 } 962 963 if (!work_done && cq->bound_q->lif->doorbell_wa) 964 ionic_txq_poke_doorbell(&qcq->q); 965 966 return work_done; 967 } 968 969 static void ionic_xdp_do_flush(struct ionic_cq *cq) 970 { 971 if (cq->bound_q->xdp_flush) { 972 xdp_do_flush(); 973 cq->bound_q->xdp_flush = false; 974 } 975 } 976 977 int ionic_rx_napi(struct napi_struct *napi, int budget) 978 { 979 struct ionic_qcq *qcq = napi_to_qcq(napi); 980 struct ionic_cq *cq = napi_to_cq(napi); 981 u32 work_done = 0; 982 u32 flags = 0; 983 984 if (unlikely(!budget)) 985 return budget; 986 987 work_done = ionic_cq_service(cq, budget, 988 ionic_rx_service, NULL, NULL); 989 990 ionic_rx_fill(cq->bound_q); 991 992 ionic_xdp_do_flush(cq); 993 if (work_done < budget && napi_complete_done(napi, work_done)) { 994 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 995 flags |= IONIC_INTR_CRED_UNMASK; 996 cq->bound_intr->rearm_count++; 997 } 998 999 if (work_done || flags) { 1000 flags |= IONIC_INTR_CRED_RESET_COALESCE; 1001 ionic_intr_credits(cq->idev->intr_ctrl, 1002 cq->bound_intr->index, 1003 work_done, flags); 1004 } 1005 1006 if (!work_done && cq->bound_q->lif->doorbell_wa) 1007 ionic_rxq_poke_doorbell(&qcq->q); 1008 1009 return work_done; 1010 } 1011 1012 int ionic_txrx_napi(struct napi_struct *napi, int budget) 1013 { 1014 struct ionic_qcq *rxqcq = napi_to_qcq(napi); 1015 struct ionic_cq *rxcq = napi_to_cq(napi); 1016 unsigned int qi = rxcq->bound_q->index; 1017 struct ionic_qcq *txqcq; 1018 struct ionic_lif *lif; 1019 struct ionic_cq *txcq; 1020 u32 rx_work_done = 0; 1021 u32 tx_work_done = 0; 1022 u32 flags = 0; 1023 1024 lif = rxcq->bound_q->lif; 1025 txqcq = lif->txqcqs[qi]; 1026 txcq = &lif->txqcqs[qi]->cq; 1027 1028 tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, !!budget); 1029 1030 if (unlikely(!budget)) 1031 return budget; 1032 1033 rx_work_done = ionic_cq_service(rxcq, budget, 1034 ionic_rx_service, NULL, NULL); 1035 1036 ionic_rx_fill(rxcq->bound_q); 1037 1038 ionic_xdp_do_flush(rxcq); 1039 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 1040 ionic_dim_update(rxqcq, 0); 1041 flags |= IONIC_INTR_CRED_UNMASK; 1042 rxcq->bound_intr->rearm_count++; 1043 } 1044 1045 if (rx_work_done || flags) { 1046 flags |= IONIC_INTR_CRED_RESET_COALESCE; 1047 ionic_intr_credits(rxcq->idev->intr_ctrl, rxcq->bound_intr->index, 1048 tx_work_done + rx_work_done, flags); 1049 } 1050 1051 if (lif->doorbell_wa) { 1052 if (!rx_work_done) 1053 ionic_rxq_poke_doorbell(&rxqcq->q); 1054 if (!tx_work_done) 1055 ionic_txq_poke_doorbell(&txqcq->q); 1056 } 1057 1058 return rx_work_done; 1059 } 1060 1061 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 1062 void *data, size_t len) 1063 { 1064 struct device *dev = q->dev; 1065 dma_addr_t dma_addr; 1066 1067 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 1068 if (unlikely(dma_mapping_error(dev, dma_addr))) { 1069 net_warn_ratelimited("%s: DMA single map failed on %s!\n", 1070 dev_name(dev), q->name); 1071 q_to_tx_stats(q)->dma_map_err++; 1072 return 0; 1073 } 1074 return dma_addr; 1075 } 1076 1077 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 1078 const skb_frag_t *frag, 1079 size_t offset, size_t len) 1080 { 1081 struct device *dev = q->dev; 1082 dma_addr_t dma_addr; 1083 1084 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 1085 if (unlikely(dma_mapping_error(dev, dma_addr))) { 1086 net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 1087 dev_name(dev), q->name); 1088 q_to_tx_stats(q)->dma_map_err++; 1089 return 0; 1090 } 1091 return dma_addr; 1092 } 1093 1094 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, 1095 struct ionic_tx_desc_info *desc_info) 1096 { 1097 struct ionic_buf_info *buf_info = desc_info->bufs; 1098 struct device *dev = q->dev; 1099 dma_addr_t dma_addr; 1100 unsigned int nfrags; 1101 skb_frag_t *frag; 1102 int frag_idx; 1103 1104 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 1105 if (!dma_addr) 1106 return -EIO; 1107 buf_info->dma_addr = dma_addr; 1108 buf_info->len = skb_headlen(skb); 1109 buf_info++; 1110 1111 frag = skb_shinfo(skb)->frags; 1112 nfrags = skb_shinfo(skb)->nr_frags; 1113 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { 1114 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 1115 if (!dma_addr) 1116 goto dma_fail; 1117 buf_info->dma_addr = dma_addr; 1118 buf_info->len = skb_frag_size(frag); 1119 buf_info++; 1120 } 1121 1122 desc_info->nbufs = 1 + nfrags; 1123 1124 return 0; 1125 1126 dma_fail: 1127 /* unwind the frag mappings and the head mapping */ 1128 while (frag_idx > 0) { 1129 frag_idx--; 1130 buf_info--; 1131 dma_unmap_page(dev, buf_info->dma_addr, 1132 buf_info->len, DMA_TO_DEVICE); 1133 } 1134 dma_unmap_single(dev, desc_info->bufs[0].dma_addr, 1135 desc_info->bufs[0].len, DMA_TO_DEVICE); 1136 return -EIO; 1137 } 1138 1139 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 1140 struct ionic_tx_desc_info *desc_info) 1141 { 1142 struct ionic_buf_info *buf_info = desc_info->bufs; 1143 struct device *dev = q->dev; 1144 unsigned int i; 1145 1146 if (!desc_info->nbufs) 1147 return; 1148 1149 dma_unmap_single(dev, buf_info->dma_addr, 1150 buf_info->len, DMA_TO_DEVICE); 1151 buf_info++; 1152 for (i = 1; i < desc_info->nbufs; i++, buf_info++) 1153 dma_unmap_page(dev, buf_info->dma_addr, 1154 buf_info->len, DMA_TO_DEVICE); 1155 1156 desc_info->nbufs = 0; 1157 } 1158 1159 static void ionic_tx_clean(struct ionic_queue *q, 1160 struct ionic_tx_desc_info *desc_info, 1161 struct ionic_txq_comp *comp, 1162 bool in_napi) 1163 { 1164 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1165 struct ionic_qcq *qcq = q_to_qcq(q); 1166 struct sk_buff *skb; 1167 1168 if (desc_info->xdpf) { 1169 ionic_xdp_tx_desc_clean(q->partner, desc_info); 1170 stats->clean++; 1171 1172 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index))) 1173 netif_wake_subqueue(q->lif->netdev, q->index); 1174 1175 return; 1176 } 1177 1178 ionic_tx_desc_unmap_bufs(q, desc_info); 1179 1180 skb = desc_info->skb; 1181 if (!skb) 1182 return; 1183 1184 if (unlikely(ionic_txq_hwstamp_enabled(q))) { 1185 if (comp) { 1186 struct skb_shared_hwtstamps hwts = {}; 1187 __le64 *cq_desc_hwstamp; 1188 u64 hwstamp; 1189 1190 cq_desc_hwstamp = 1191 (void *)comp + 1192 qcq->cq.desc_size - 1193 sizeof(struct ionic_txq_comp) - 1194 IONIC_HWSTAMP_CQ_NEGOFFSET; 1195 1196 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 1197 1198 if (hwstamp != IONIC_HWSTAMP_INVALID) { 1199 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 1200 1201 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1202 skb_tstamp_tx(skb, &hwts); 1203 1204 stats->hwstamp_valid++; 1205 } else { 1206 stats->hwstamp_invalid++; 1207 } 1208 } 1209 } 1210 1211 desc_info->bytes = skb->len; 1212 stats->clean++; 1213 1214 napi_consume_skb(skb, likely(in_napi) ? 1 : 0); 1215 } 1216 1217 static bool ionic_tx_service(struct ionic_cq *cq, 1218 unsigned int *total_pkts, 1219 unsigned int *total_bytes, 1220 bool in_napi) 1221 { 1222 struct ionic_tx_desc_info *desc_info; 1223 struct ionic_queue *q = cq->bound_q; 1224 struct ionic_txq_comp *comp; 1225 unsigned int bytes = 0; 1226 unsigned int pkts = 0; 1227 u16 index; 1228 1229 comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; 1230 1231 if (!color_match(comp->color, cq->done_color)) 1232 return false; 1233 1234 /* clean the related q entries, there could be 1235 * several q entries completed for each cq completion 1236 */ 1237 do { 1238 desc_info = &q->tx_info[q->tail_idx]; 1239 desc_info->bytes = 0; 1240 index = q->tail_idx; 1241 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1242 ionic_tx_clean(q, desc_info, comp, in_napi); 1243 if (desc_info->skb) { 1244 pkts++; 1245 bytes += desc_info->bytes; 1246 desc_info->skb = NULL; 1247 } 1248 } while (index != le16_to_cpu(comp->comp_index)); 1249 1250 (*total_pkts) += pkts; 1251 (*total_bytes) += bytes; 1252 1253 return true; 1254 } 1255 1256 unsigned int ionic_tx_cq_service(struct ionic_cq *cq, 1257 unsigned int work_to_do, 1258 bool in_napi) 1259 { 1260 unsigned int work_done = 0; 1261 unsigned int bytes = 0; 1262 unsigned int pkts = 0; 1263 1264 if (work_to_do == 0) 1265 return 0; 1266 1267 while (ionic_tx_service(cq, &pkts, &bytes, in_napi)) { 1268 if (cq->tail_idx == cq->num_descs - 1) 1269 cq->done_color = !cq->done_color; 1270 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 1271 1272 if (++work_done >= work_to_do) 1273 break; 1274 } 1275 1276 if (work_done) { 1277 struct ionic_queue *q = cq->bound_q; 1278 1279 if (likely(!ionic_txq_hwstamp_enabled(q))) 1280 netif_txq_completed_wake(q_to_ndq(q->lif->netdev, q), 1281 pkts, bytes, 1282 ionic_q_space_avail(q), 1283 IONIC_TSO_DESCS_NEEDED); 1284 } 1285 1286 return work_done; 1287 } 1288 1289 void ionic_tx_flush(struct ionic_cq *cq) 1290 { 1291 u32 work_done; 1292 1293 work_done = ionic_tx_cq_service(cq, cq->num_descs, false); 1294 if (work_done) 1295 ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index, 1296 work_done, IONIC_INTR_CRED_RESET_COALESCE); 1297 } 1298 1299 void ionic_tx_empty(struct ionic_queue *q) 1300 { 1301 struct ionic_tx_desc_info *desc_info; 1302 int bytes = 0; 1303 int pkts = 0; 1304 1305 /* walk the not completed tx entries, if any */ 1306 while (q->head_idx != q->tail_idx) { 1307 desc_info = &q->tx_info[q->tail_idx]; 1308 desc_info->bytes = 0; 1309 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1310 ionic_tx_clean(q, desc_info, NULL, false); 1311 if (desc_info->skb) { 1312 pkts++; 1313 bytes += desc_info->bytes; 1314 desc_info->skb = NULL; 1315 } 1316 } 1317 1318 if (likely(!ionic_txq_hwstamp_enabled(q))) { 1319 struct netdev_queue *ndq = q_to_ndq(q->lif->netdev, q); 1320 1321 netdev_tx_completed_queue(ndq, pkts, bytes); 1322 netdev_tx_reset_queue(ndq); 1323 } 1324 } 1325 1326 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 1327 { 1328 int err; 1329 1330 err = skb_cow_head(skb, 0); 1331 if (unlikely(err)) 1332 return err; 1333 1334 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 1335 inner_ip_hdr(skb)->check = 0; 1336 inner_tcp_hdr(skb)->check = 1337 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 1338 inner_ip_hdr(skb)->daddr, 1339 0, IPPROTO_TCP, 0); 1340 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 1341 inner_tcp_hdr(skb)->check = 1342 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 1343 &inner_ipv6_hdr(skb)->daddr, 1344 0, IPPROTO_TCP, 0); 1345 } 1346 1347 return 0; 1348 } 1349 1350 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 1351 { 1352 int err; 1353 1354 err = skb_cow_head(skb, 0); 1355 if (unlikely(err)) 1356 return err; 1357 1358 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 1359 ip_hdr(skb)->check = 0; 1360 tcp_hdr(skb)->check = 1361 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 1362 ip_hdr(skb)->daddr, 1363 0, IPPROTO_TCP, 0); 1364 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 1365 tcp_v6_gso_csum_prep(skb); 1366 } 1367 1368 return 0; 1369 } 1370 1371 static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q, 1372 struct ionic_txq_desc *desc, 1373 struct sk_buff *skb, 1374 dma_addr_t addr, u8 nsge, u16 len, 1375 unsigned int hdrlen, unsigned int mss, 1376 bool outer_csum, 1377 u16 vlan_tci, bool has_vlan, 1378 bool start, bool done) 1379 { 1380 u8 flags = 0; 1381 u64 cmd; 1382 1383 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1384 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1385 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 1386 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 1387 1388 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 1389 desc->cmd = cpu_to_le64(cmd); 1390 desc->len = cpu_to_le16(len); 1391 desc->vlan_tci = cpu_to_le16(vlan_tci); 1392 desc->hdr_len = cpu_to_le16(hdrlen); 1393 desc->mss = cpu_to_le16(mss); 1394 1395 ionic_write_cmb_desc(q, desc); 1396 1397 if (start) { 1398 skb_tx_timestamp(skb); 1399 if (likely(!ionic_txq_hwstamp_enabled(q))) 1400 netdev_tx_sent_queue(q_to_ndq(netdev, q), skb->len); 1401 ionic_txq_post(q, false); 1402 } else { 1403 ionic_txq_post(q, done); 1404 } 1405 } 1406 1407 static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q, 1408 struct sk_buff *skb) 1409 { 1410 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1411 struct ionic_tx_desc_info *desc_info; 1412 struct ionic_buf_info *buf_info; 1413 struct ionic_txq_sg_elem *elem; 1414 struct ionic_txq_desc *desc; 1415 unsigned int chunk_len; 1416 unsigned int frag_rem; 1417 unsigned int tso_rem; 1418 unsigned int seg_rem; 1419 dma_addr_t desc_addr; 1420 dma_addr_t frag_addr; 1421 unsigned int hdrlen; 1422 unsigned int len; 1423 unsigned int mss; 1424 bool start, done; 1425 bool outer_csum; 1426 bool has_vlan; 1427 u16 desc_len; 1428 u8 desc_nsge; 1429 u16 vlan_tci; 1430 bool encap; 1431 int err; 1432 1433 desc_info = &q->tx_info[q->head_idx]; 1434 1435 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1436 return -EIO; 1437 1438 len = skb->len; 1439 mss = skb_shinfo(skb)->gso_size; 1440 outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 1441 SKB_GSO_GRE_CSUM | 1442 SKB_GSO_IPXIP4 | 1443 SKB_GSO_IPXIP6 | 1444 SKB_GSO_UDP_TUNNEL | 1445 SKB_GSO_UDP_TUNNEL_CSUM)); 1446 has_vlan = !!skb_vlan_tag_present(skb); 1447 vlan_tci = skb_vlan_tag_get(skb); 1448 encap = skb->encapsulation; 1449 1450 /* Preload inner-most TCP csum field with IP pseudo hdr 1451 * calculated with IP length set to zero. HW will later 1452 * add in length to each TCP segment resulting from the TSO. 1453 */ 1454 1455 if (encap) 1456 err = ionic_tx_tcp_inner_pseudo_csum(skb); 1457 else 1458 err = ionic_tx_tcp_pseudo_csum(skb); 1459 if (unlikely(err)) { 1460 /* clean up mapping from ionic_tx_map_skb */ 1461 ionic_tx_desc_unmap_bufs(q, desc_info); 1462 return err; 1463 } 1464 1465 if (encap) 1466 hdrlen = skb_inner_tcp_all_headers(skb); 1467 else 1468 hdrlen = skb_tcp_all_headers(skb); 1469 1470 desc_info->skb = skb; 1471 buf_info = desc_info->bufs; 1472 tso_rem = len; 1473 seg_rem = min(tso_rem, hdrlen + mss); 1474 1475 frag_addr = 0; 1476 frag_rem = 0; 1477 1478 start = true; 1479 1480 while (tso_rem > 0) { 1481 desc = NULL; 1482 elem = NULL; 1483 desc_addr = 0; 1484 desc_len = 0; 1485 desc_nsge = 0; 1486 /* use fragments until we have enough to post a single descriptor */ 1487 while (seg_rem > 0) { 1488 /* if the fragment is exhausted then move to the next one */ 1489 if (frag_rem == 0) { 1490 /* grab the next fragment */ 1491 frag_addr = buf_info->dma_addr; 1492 frag_rem = buf_info->len; 1493 buf_info++; 1494 } 1495 chunk_len = min(frag_rem, seg_rem); 1496 if (!desc) { 1497 /* fill main descriptor */ 1498 desc = &q->txq[q->head_idx]; 1499 elem = ionic_tx_sg_elems(q); 1500 desc_addr = frag_addr; 1501 desc_len = chunk_len; 1502 } else { 1503 /* fill sg descriptor */ 1504 elem->addr = cpu_to_le64(frag_addr); 1505 elem->len = cpu_to_le16(chunk_len); 1506 elem++; 1507 desc_nsge++; 1508 } 1509 frag_addr += chunk_len; 1510 frag_rem -= chunk_len; 1511 tso_rem -= chunk_len; 1512 seg_rem -= chunk_len; 1513 } 1514 seg_rem = min(tso_rem, mss); 1515 done = (tso_rem == 0); 1516 /* post descriptor */ 1517 ionic_tx_tso_post(netdev, q, desc, skb, desc_addr, desc_nsge, 1518 desc_len, hdrlen, mss, outer_csum, vlan_tci, 1519 has_vlan, start, done); 1520 start = false; 1521 /* Buffer information is stored with the first tso descriptor */ 1522 desc_info = &q->tx_info[q->head_idx]; 1523 desc_info->nbufs = 0; 1524 } 1525 1526 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); 1527 stats->bytes += len; 1528 stats->tso++; 1529 stats->tso_bytes = len; 1530 1531 return 0; 1532 } 1533 1534 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, 1535 struct ionic_tx_desc_info *desc_info) 1536 { 1537 struct ionic_txq_desc *desc = &q->txq[q->head_idx]; 1538 struct ionic_buf_info *buf_info = desc_info->bufs; 1539 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1540 bool has_vlan; 1541 u8 flags = 0; 1542 bool encap; 1543 u64 cmd; 1544 1545 has_vlan = !!skb_vlan_tag_present(skb); 1546 encap = skb->encapsulation; 1547 1548 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1549 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1550 1551 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 1552 flags, skb_shinfo(skb)->nr_frags, 1553 buf_info->dma_addr); 1554 desc->cmd = cpu_to_le64(cmd); 1555 desc->len = cpu_to_le16(buf_info->len); 1556 if (has_vlan) { 1557 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1558 stats->vlan_inserted++; 1559 } else { 1560 desc->vlan_tci = 0; 1561 } 1562 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 1563 desc->csum_offset = cpu_to_le16(skb->csum_offset); 1564 1565 ionic_write_cmb_desc(q, desc); 1566 1567 if (skb_csum_is_sctp(skb)) 1568 stats->crc32_csum++; 1569 else 1570 stats->csum++; 1571 } 1572 1573 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, 1574 struct ionic_tx_desc_info *desc_info) 1575 { 1576 struct ionic_txq_desc *desc = &q->txq[q->head_idx]; 1577 struct ionic_buf_info *buf_info = desc_info->bufs; 1578 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1579 bool has_vlan; 1580 u8 flags = 0; 1581 bool encap; 1582 u64 cmd; 1583 1584 has_vlan = !!skb_vlan_tag_present(skb); 1585 encap = skb->encapsulation; 1586 1587 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1588 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1589 1590 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1591 flags, skb_shinfo(skb)->nr_frags, 1592 buf_info->dma_addr); 1593 desc->cmd = cpu_to_le64(cmd); 1594 desc->len = cpu_to_le16(buf_info->len); 1595 if (has_vlan) { 1596 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1597 stats->vlan_inserted++; 1598 } else { 1599 desc->vlan_tci = 0; 1600 } 1601 desc->csum_start = 0; 1602 desc->csum_offset = 0; 1603 1604 ionic_write_cmb_desc(q, desc); 1605 1606 stats->csum_none++; 1607 } 1608 1609 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, 1610 struct ionic_tx_desc_info *desc_info) 1611 { 1612 struct ionic_buf_info *buf_info = &desc_info->bufs[1]; 1613 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1614 struct ionic_txq_sg_elem *elem; 1615 unsigned int i; 1616 1617 elem = ionic_tx_sg_elems(q); 1618 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { 1619 elem->addr = cpu_to_le64(buf_info->dma_addr); 1620 elem->len = cpu_to_le16(buf_info->len); 1621 } 1622 1623 stats->frags += skb_shinfo(skb)->nr_frags; 1624 } 1625 1626 static int ionic_tx(struct net_device *netdev, struct ionic_queue *q, 1627 struct sk_buff *skb) 1628 { 1629 struct ionic_tx_desc_info *desc_info = &q->tx_info[q->head_idx]; 1630 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1631 bool ring_dbell = true; 1632 1633 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1634 return -EIO; 1635 1636 desc_info->skb = skb; 1637 1638 /* set up the initial descriptor */ 1639 if (skb->ip_summed == CHECKSUM_PARTIAL) 1640 ionic_tx_calc_csum(q, skb, desc_info); 1641 else 1642 ionic_tx_calc_no_csum(q, skb, desc_info); 1643 1644 /* add frags */ 1645 ionic_tx_skb_frags(q, skb, desc_info); 1646 1647 skb_tx_timestamp(skb); 1648 stats->pkts++; 1649 stats->bytes += skb->len; 1650 1651 if (likely(!ionic_txq_hwstamp_enabled(q))) { 1652 struct netdev_queue *ndq = q_to_ndq(netdev, q); 1653 1654 if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1))) 1655 netif_tx_stop_queue(ndq); 1656 ring_dbell = __netdev_tx_sent_queue(ndq, skb->len, 1657 netdev_xmit_more()); 1658 } 1659 ionic_txq_post(q, ring_dbell); 1660 1661 return 0; 1662 } 1663 1664 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 1665 { 1666 int nr_frags = skb_shinfo(skb)->nr_frags; 1667 bool too_many_frags = false; 1668 skb_frag_t *frag; 1669 int desc_bufs; 1670 int chunk_len; 1671 int frag_rem; 1672 int tso_rem; 1673 int seg_rem; 1674 bool encap; 1675 int hdrlen; 1676 int ndescs; 1677 int err; 1678 1679 /* Each desc is mss long max, so a descriptor for each gso_seg */ 1680 if (skb_is_gso(skb)) { 1681 ndescs = skb_shinfo(skb)->gso_segs; 1682 if (!nr_frags) 1683 return ndescs; 1684 } else { 1685 ndescs = 1; 1686 if (!nr_frags) 1687 return ndescs; 1688 1689 if (unlikely(nr_frags > q->max_sg_elems)) { 1690 too_many_frags = true; 1691 goto linearize; 1692 } 1693 1694 return ndescs; 1695 } 1696 1697 /* We need to scan the skb to be sure that none of the MTU sized 1698 * packets in the TSO will require more sgs per descriptor than we 1699 * can support. We loop through the frags, add up the lengths for 1700 * a packet, and count the number of sgs used per packet. 1701 */ 1702 tso_rem = skb->len; 1703 frag = skb_shinfo(skb)->frags; 1704 encap = skb->encapsulation; 1705 1706 /* start with just hdr in first part of first descriptor */ 1707 if (encap) 1708 hdrlen = skb_inner_tcp_all_headers(skb); 1709 else 1710 hdrlen = skb_tcp_all_headers(skb); 1711 seg_rem = min_t(int, tso_rem, hdrlen + skb_shinfo(skb)->gso_size); 1712 frag_rem = hdrlen; 1713 1714 while (tso_rem > 0) { 1715 desc_bufs = 0; 1716 while (seg_rem > 0) { 1717 desc_bufs++; 1718 1719 /* We add the +1 because we can take buffers for one 1720 * more than we have SGs: one for the initial desc data 1721 * in addition to the SG segments that might follow. 1722 */ 1723 if (desc_bufs > q->max_sg_elems + 1) { 1724 too_many_frags = true; 1725 goto linearize; 1726 } 1727 1728 if (frag_rem == 0) { 1729 frag_rem = skb_frag_size(frag); 1730 frag++; 1731 } 1732 chunk_len = min(frag_rem, seg_rem); 1733 frag_rem -= chunk_len; 1734 tso_rem -= chunk_len; 1735 seg_rem -= chunk_len; 1736 } 1737 1738 seg_rem = min_t(int, tso_rem, skb_shinfo(skb)->gso_size); 1739 } 1740 1741 linearize: 1742 if (too_many_frags) { 1743 err = skb_linearize(skb); 1744 if (unlikely(err)) 1745 return err; 1746 q_to_tx_stats(q)->linearize++; 1747 } 1748 1749 return ndescs; 1750 } 1751 1752 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, 1753 struct net_device *netdev) 1754 { 1755 struct ionic_lif *lif = netdev_priv(netdev); 1756 struct ionic_queue *q; 1757 int err, ndescs; 1758 1759 /* Does not stop/start txq, because we post to a separate tx queue 1760 * for timestamping, and if a packet can't be posted immediately to 1761 * the timestamping queue, it is dropped. 1762 */ 1763 1764 q = &lif->hwstamp_txq->q; 1765 ndescs = ionic_tx_descs_needed(q, skb); 1766 if (unlikely(ndescs < 0)) 1767 goto err_out_drop; 1768 1769 if (unlikely(!ionic_q_has_space(q, ndescs))) 1770 goto err_out_drop; 1771 1772 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP; 1773 if (skb_is_gso(skb)) 1774 err = ionic_tx_tso(netdev, q, skb); 1775 else 1776 err = ionic_tx(netdev, q, skb); 1777 1778 if (unlikely(err)) 1779 goto err_out_drop; 1780 1781 return NETDEV_TX_OK; 1782 1783 err_out_drop: 1784 q->drop++; 1785 dev_kfree_skb(skb); 1786 return NETDEV_TX_OK; 1787 } 1788 1789 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1790 { 1791 u16 queue_index = skb_get_queue_mapping(skb); 1792 struct ionic_lif *lif = netdev_priv(netdev); 1793 struct ionic_queue *q; 1794 int ndescs; 1795 int err; 1796 1797 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 1798 dev_kfree_skb(skb); 1799 return NETDEV_TX_OK; 1800 } 1801 1802 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 1803 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode) 1804 return ionic_start_hwstamp_xmit(skb, netdev); 1805 1806 if (unlikely(queue_index >= lif->nxqs)) 1807 queue_index = 0; 1808 q = &lif->txqcqs[queue_index]->q; 1809 1810 ndescs = ionic_tx_descs_needed(q, skb); 1811 if (ndescs < 0) 1812 goto err_out_drop; 1813 1814 if (!netif_txq_maybe_stop(q_to_ndq(netdev, q), 1815 ionic_q_space_avail(q), 1816 ndescs, ndescs)) 1817 return NETDEV_TX_BUSY; 1818 1819 if (skb_is_gso(skb)) 1820 err = ionic_tx_tso(netdev, q, skb); 1821 else 1822 err = ionic_tx(netdev, q, skb); 1823 1824 if (unlikely(err)) 1825 goto err_out_drop; 1826 1827 return NETDEV_TX_OK; 1828 1829 err_out_drop: 1830 q->drop++; 1831 dev_kfree_skb(skb); 1832 return NETDEV_TX_OK; 1833 } 1834