1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/ip.h> 5 #include <linux/ipv6.h> 6 #include <linux/if_vlan.h> 7 #include <net/ip6_checksum.h> 8 #include <net/netdev_queues.h> 9 10 #include "ionic.h" 11 #include "ionic_lif.h" 12 #include "ionic_txrx.h" 13 14 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 15 void *data, size_t len); 16 17 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 18 const skb_frag_t *frag, 19 size_t offset, size_t len); 20 21 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 22 struct ionic_desc_info *desc_info); 23 24 static void ionic_tx_clean(struct ionic_queue *q, 25 struct ionic_desc_info *desc_info, 26 struct ionic_cq_info *cq_info, 27 void *cb_arg); 28 29 static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, 30 ionic_desc_cb cb_func, void *cb_arg) 31 { 32 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 33 } 34 35 static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell, 36 ionic_desc_cb cb_func, void *cb_arg) 37 { 38 ionic_q_post(q, ring_dbell, cb_func, cb_arg); 39 } 40 41 bool ionic_txq_poke_doorbell(struct ionic_queue *q) 42 { 43 struct netdev_queue *netdev_txq; 44 unsigned long now, then, dif; 45 struct net_device *netdev; 46 47 netdev = q->lif->netdev; 48 netdev_txq = netdev_get_tx_queue(netdev, q->index); 49 50 HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id()); 51 52 if (q->tail_idx == q->head_idx) { 53 HARD_TX_UNLOCK(netdev, netdev_txq); 54 return false; 55 } 56 57 now = READ_ONCE(jiffies); 58 then = q->dbell_jiffies; 59 dif = now - then; 60 61 if (dif > q->dbell_deadline) { 62 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 63 q->dbval | q->head_idx); 64 65 q->dbell_jiffies = now; 66 } 67 68 HARD_TX_UNLOCK(netdev, netdev_txq); 69 70 return true; 71 } 72 73 bool ionic_rxq_poke_doorbell(struct ionic_queue *q) 74 { 75 unsigned long now, then, dif; 76 77 /* no lock, called from rx napi or txrx napi, nothing else can fill */ 78 79 if (q->tail_idx == q->head_idx) 80 return false; 81 82 now = READ_ONCE(jiffies); 83 then = q->dbell_jiffies; 84 dif = now - then; 85 86 if (dif > q->dbell_deadline) { 87 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 88 q->dbval | q->head_idx); 89 90 q->dbell_jiffies = now; 91 92 dif = 2 * q->dbell_deadline; 93 if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE) 94 dif = IONIC_RX_MAX_DOORBELL_DEADLINE; 95 96 q->dbell_deadline = dif; 97 } 98 99 return true; 100 } 101 102 static inline struct netdev_queue *q_to_ndq(struct net_device *netdev, 103 struct ionic_queue *q) 104 { 105 return netdev_get_tx_queue(netdev, q->index); 106 } 107 108 static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info) 109 { 110 return page_address(buf_info->page) + buf_info->page_offset; 111 } 112 113 static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info) 114 { 115 return buf_info->dma_addr + buf_info->page_offset; 116 } 117 118 static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info) 119 { 120 return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset); 121 } 122 123 static int ionic_rx_page_alloc(struct ionic_queue *q, 124 struct ionic_buf_info *buf_info) 125 { 126 struct ionic_rx_stats *stats; 127 struct device *dev; 128 struct page *page; 129 130 dev = q->dev; 131 stats = q_to_rx_stats(q); 132 133 if (unlikely(!buf_info)) { 134 net_err_ratelimited("%s: %s invalid buf_info in alloc\n", 135 dev_name(dev), q->name); 136 return -EINVAL; 137 } 138 139 page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); 140 if (unlikely(!page)) { 141 net_err_ratelimited("%s: %s page alloc failed\n", 142 dev_name(dev), q->name); 143 stats->alloc_err++; 144 return -ENOMEM; 145 } 146 147 buf_info->dma_addr = dma_map_page(dev, page, 0, 148 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 149 if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) { 150 __free_pages(page, 0); 151 net_err_ratelimited("%s: %s dma map failed\n", 152 dev_name(dev), q->name); 153 stats->dma_map_err++; 154 return -EIO; 155 } 156 157 buf_info->page = page; 158 buf_info->page_offset = 0; 159 160 return 0; 161 } 162 163 static void ionic_rx_page_free(struct ionic_queue *q, 164 struct ionic_buf_info *buf_info) 165 { 166 struct device *dev = q->dev; 167 168 if (unlikely(!buf_info)) { 169 net_err_ratelimited("%s: %s invalid buf_info in free\n", 170 dev_name(dev), q->name); 171 return; 172 } 173 174 if (!buf_info->page) 175 return; 176 177 dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 178 __free_pages(buf_info->page, 0); 179 buf_info->page = NULL; 180 } 181 182 static bool ionic_rx_buf_recycle(struct ionic_queue *q, 183 struct ionic_buf_info *buf_info, u32 used) 184 { 185 u32 size; 186 187 /* don't re-use pages allocated in low-mem condition */ 188 if (page_is_pfmemalloc(buf_info->page)) 189 return false; 190 191 /* don't re-use buffers from non-local numa nodes */ 192 if (page_to_nid(buf_info->page) != numa_mem_id()) 193 return false; 194 195 size = ALIGN(used, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ); 196 buf_info->page_offset += size; 197 if (buf_info->page_offset >= IONIC_PAGE_SIZE) 198 return false; 199 200 get_page(buf_info->page); 201 202 return true; 203 } 204 205 static struct sk_buff *ionic_rx_frags(struct net_device *netdev, 206 struct ionic_queue *q, 207 struct ionic_desc_info *desc_info, 208 unsigned int headroom, 209 unsigned int len, 210 unsigned int num_sg_elems, 211 bool synced) 212 { 213 struct ionic_buf_info *buf_info; 214 struct ionic_rx_stats *stats; 215 struct device *dev = q->dev; 216 struct sk_buff *skb; 217 unsigned int i; 218 u16 frag_len; 219 220 stats = q_to_rx_stats(q); 221 222 buf_info = &desc_info->bufs[0]; 223 224 prefetchw(buf_info->page); 225 226 skb = napi_get_frags(&q_to_qcq(q)->napi); 227 if (unlikely(!skb)) { 228 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 229 dev_name(dev), q->name); 230 stats->alloc_err++; 231 return NULL; 232 } 233 234 i = num_sg_elems + 1; 235 do { 236 if (unlikely(!buf_info->page)) { 237 dev_kfree_skb(skb); 238 return NULL; 239 } 240 241 if (headroom) 242 frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 243 else 244 frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); 245 len -= frag_len; 246 247 if (!synced) 248 dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 249 headroom, frag_len, DMA_FROM_DEVICE); 250 251 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 252 buf_info->page, buf_info->page_offset + headroom, 253 frag_len, IONIC_PAGE_SIZE); 254 255 if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { 256 dma_unmap_page(dev, buf_info->dma_addr, 257 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 258 buf_info->page = NULL; 259 } 260 261 /* only needed on the first buffer */ 262 if (headroom) 263 headroom = 0; 264 265 buf_info++; 266 267 i--; 268 } while (i > 0); 269 270 return skb; 271 } 272 273 static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev, 274 struct ionic_queue *q, 275 struct ionic_desc_info *desc_info, 276 unsigned int headroom, 277 unsigned int len, 278 bool synced) 279 { 280 struct ionic_buf_info *buf_info; 281 struct ionic_rx_stats *stats; 282 struct device *dev = q->dev; 283 struct sk_buff *skb; 284 285 stats = q_to_rx_stats(q); 286 287 buf_info = &desc_info->bufs[0]; 288 289 skb = napi_alloc_skb(&q_to_qcq(q)->napi, len); 290 if (unlikely(!skb)) { 291 net_warn_ratelimited("%s: SKB alloc failed on %s!\n", 292 dev_name(dev), q->name); 293 stats->alloc_err++; 294 return NULL; 295 } 296 297 if (unlikely(!buf_info->page)) { 298 dev_kfree_skb(skb); 299 return NULL; 300 } 301 302 if (!synced) 303 dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), 304 headroom, len, DMA_FROM_DEVICE); 305 skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); 306 dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info), 307 headroom, len, DMA_FROM_DEVICE); 308 309 skb_put(skb, len); 310 skb->protocol = eth_type_trans(skb, netdev); 311 312 return skb; 313 } 314 315 static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, 316 struct ionic_desc_info *desc_info) 317 { 318 unsigned int nbufs = desc_info->nbufs; 319 struct ionic_buf_info *buf_info; 320 struct device *dev = q->dev; 321 int i; 322 323 if (!nbufs) 324 return; 325 326 buf_info = desc_info->bufs; 327 dma_unmap_single(dev, buf_info->dma_addr, 328 buf_info->len, DMA_TO_DEVICE); 329 if (desc_info->act == XDP_TX) 330 __free_pages(buf_info->page, 0); 331 buf_info->page = NULL; 332 333 buf_info++; 334 for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) { 335 dma_unmap_page(dev, buf_info->dma_addr, 336 buf_info->len, DMA_TO_DEVICE); 337 if (desc_info->act == XDP_TX) 338 __free_pages(buf_info->page, 0); 339 buf_info->page = NULL; 340 } 341 342 if (desc_info->act == XDP_REDIRECT) 343 xdp_return_frame(desc_info->xdpf); 344 345 desc_info->nbufs = 0; 346 desc_info->xdpf = NULL; 347 desc_info->act = 0; 348 } 349 350 static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame, 351 enum xdp_action act, struct page *page, int off, 352 bool ring_doorbell) 353 { 354 struct ionic_desc_info *desc_info; 355 struct ionic_buf_info *buf_info; 356 struct ionic_tx_stats *stats; 357 struct ionic_txq_desc *desc; 358 size_t len = frame->len; 359 dma_addr_t dma_addr; 360 u64 cmd; 361 362 desc_info = &q->info[q->head_idx]; 363 desc = desc_info->txq_desc; 364 buf_info = desc_info->bufs; 365 stats = q_to_tx_stats(q); 366 367 dma_addr = ionic_tx_map_single(q, frame->data, len); 368 if (dma_mapping_error(q->dev, dma_addr)) { 369 stats->dma_map_err++; 370 return -EIO; 371 } 372 buf_info->dma_addr = dma_addr; 373 buf_info->len = len; 374 buf_info->page = page; 375 buf_info->page_offset = off; 376 377 desc_info->nbufs = 1; 378 desc_info->xdpf = frame; 379 desc_info->act = act; 380 381 if (xdp_frame_has_frags(frame)) { 382 struct ionic_txq_sg_elem *elem; 383 struct skb_shared_info *sinfo; 384 struct ionic_buf_info *bi; 385 skb_frag_t *frag; 386 int i; 387 388 bi = &buf_info[1]; 389 sinfo = xdp_get_shared_info_from_frame(frame); 390 frag = sinfo->frags; 391 elem = desc_info->txq_sg_desc->elems; 392 for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) { 393 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 394 if (dma_mapping_error(q->dev, dma_addr)) { 395 stats->dma_map_err++; 396 ionic_tx_desc_unmap_bufs(q, desc_info); 397 return -EIO; 398 } 399 bi->dma_addr = dma_addr; 400 bi->len = skb_frag_size(frag); 401 bi->page = skb_frag_page(frag); 402 403 elem->addr = cpu_to_le64(bi->dma_addr); 404 elem->len = cpu_to_le16(bi->len); 405 elem++; 406 407 desc_info->nbufs++; 408 } 409 } 410 411 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 412 0, (desc_info->nbufs - 1), buf_info->dma_addr); 413 desc->cmd = cpu_to_le64(cmd); 414 desc->len = cpu_to_le16(len); 415 desc->csum_start = 0; 416 desc->csum_offset = 0; 417 418 stats->xdp_frames++; 419 stats->pkts++; 420 stats->bytes += len; 421 422 ionic_txq_post(q, ring_doorbell, ionic_tx_clean, NULL); 423 424 return 0; 425 } 426 427 int ionic_xdp_xmit(struct net_device *netdev, int n, 428 struct xdp_frame **xdp_frames, u32 flags) 429 { 430 struct ionic_lif *lif = netdev_priv(netdev); 431 struct ionic_queue *txq; 432 struct netdev_queue *nq; 433 int nxmit; 434 int space; 435 int cpu; 436 int qi; 437 438 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) 439 return -ENETDOWN; 440 441 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 442 return -EINVAL; 443 444 /* AdminQ is assumed on cpu 0, while we attempt to affinitize the 445 * TxRx queue pairs 0..n-1 on cpus 1..n. We try to keep with that 446 * affinitization here, but of course irqbalance and friends might 447 * have juggled things anyway, so we have to check for the 0 case. 448 */ 449 cpu = smp_processor_id(); 450 qi = cpu ? (cpu - 1) % lif->nxqs : cpu; 451 452 txq = &lif->txqcqs[qi]->q; 453 nq = netdev_get_tx_queue(netdev, txq->index); 454 __netif_tx_lock(nq, cpu); 455 txq_trans_cond_update(nq); 456 457 if (netif_tx_queue_stopped(nq) || 458 !netif_txq_maybe_stop(q_to_ndq(netdev, txq), 459 ionic_q_space_avail(txq), 460 1, 1)) { 461 __netif_tx_unlock(nq); 462 return -EIO; 463 } 464 465 space = min_t(int, n, ionic_q_space_avail(txq)); 466 for (nxmit = 0; nxmit < space ; nxmit++) { 467 if (ionic_xdp_post_frame(txq, xdp_frames[nxmit], 468 XDP_REDIRECT, 469 virt_to_page(xdp_frames[nxmit]->data), 470 0, false)) { 471 nxmit--; 472 break; 473 } 474 } 475 476 if (flags & XDP_XMIT_FLUSH) 477 ionic_dbell_ring(lif->kern_dbpage, txq->hw_type, 478 txq->dbval | txq->head_idx); 479 480 netif_txq_maybe_stop(q_to_ndq(netdev, txq), 481 ionic_q_space_avail(txq), 482 4, 4); 483 __netif_tx_unlock(nq); 484 485 return nxmit; 486 } 487 488 static bool ionic_run_xdp(struct ionic_rx_stats *stats, 489 struct net_device *netdev, 490 struct bpf_prog *xdp_prog, 491 struct ionic_queue *rxq, 492 struct ionic_buf_info *buf_info, 493 int len) 494 { 495 u32 xdp_action = XDP_ABORTED; 496 struct xdp_buff xdp_buf; 497 struct ionic_queue *txq; 498 struct netdev_queue *nq; 499 struct xdp_frame *xdpf; 500 int remain_len; 501 int frag_len; 502 int err = 0; 503 504 xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info); 505 frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); 506 xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), 507 XDP_PACKET_HEADROOM, frag_len, false); 508 509 dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info), 510 XDP_PACKET_HEADROOM, len, 511 DMA_FROM_DEVICE); 512 513 prefetchw(&xdp_buf.data_hard_start); 514 515 /* We limit MTU size to one buffer if !xdp_has_frags, so 516 * if the recv len is bigger than one buffer 517 * then we know we have frag info to gather 518 */ 519 remain_len = len - frag_len; 520 if (remain_len) { 521 struct skb_shared_info *sinfo; 522 struct ionic_buf_info *bi; 523 skb_frag_t *frag; 524 525 bi = buf_info; 526 sinfo = xdp_get_shared_info_from_buff(&xdp_buf); 527 sinfo->nr_frags = 0; 528 sinfo->xdp_frags_size = 0; 529 xdp_buff_set_frags_flag(&xdp_buf); 530 531 do { 532 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) { 533 err = -ENOSPC; 534 goto out_xdp_abort; 535 } 536 537 frag = &sinfo->frags[sinfo->nr_frags]; 538 sinfo->nr_frags++; 539 bi++; 540 frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi)); 541 dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi), 542 0, frag_len, DMA_FROM_DEVICE); 543 skb_frag_fill_page_desc(frag, bi->page, 0, frag_len); 544 sinfo->xdp_frags_size += frag_len; 545 remain_len -= frag_len; 546 547 if (page_is_pfmemalloc(bi->page)) 548 xdp_buff_set_frag_pfmemalloc(&xdp_buf); 549 } while (remain_len > 0); 550 } 551 552 xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf); 553 554 switch (xdp_action) { 555 case XDP_PASS: 556 stats->xdp_pass++; 557 return false; /* false = we didn't consume the packet */ 558 559 case XDP_DROP: 560 ionic_rx_page_free(rxq, buf_info); 561 stats->xdp_drop++; 562 break; 563 564 case XDP_TX: 565 xdpf = xdp_convert_buff_to_frame(&xdp_buf); 566 if (!xdpf) 567 goto out_xdp_abort; 568 569 txq = rxq->partner; 570 nq = netdev_get_tx_queue(netdev, txq->index); 571 __netif_tx_lock(nq, smp_processor_id()); 572 txq_trans_cond_update(nq); 573 574 if (netif_tx_queue_stopped(nq) || 575 !netif_txq_maybe_stop(q_to_ndq(netdev, txq), 576 ionic_q_space_avail(txq), 577 1, 1)) { 578 __netif_tx_unlock(nq); 579 goto out_xdp_abort; 580 } 581 582 dma_unmap_page(rxq->dev, buf_info->dma_addr, 583 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 584 585 err = ionic_xdp_post_frame(txq, xdpf, XDP_TX, 586 buf_info->page, 587 buf_info->page_offset, 588 true); 589 __netif_tx_unlock(nq); 590 if (err) { 591 netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); 592 goto out_xdp_abort; 593 } 594 stats->xdp_tx++; 595 596 /* the Tx completion will free the buffers */ 597 break; 598 599 case XDP_REDIRECT: 600 /* unmap the pages before handing them to a different device */ 601 dma_unmap_page(rxq->dev, buf_info->dma_addr, 602 IONIC_PAGE_SIZE, DMA_FROM_DEVICE); 603 604 err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog); 605 if (err) { 606 netdev_dbg(netdev, "xdp_do_redirect err %d\n", err); 607 goto out_xdp_abort; 608 } 609 buf_info->page = NULL; 610 rxq->xdp_flush = true; 611 stats->xdp_redirect++; 612 break; 613 614 case XDP_ABORTED: 615 default: 616 goto out_xdp_abort; 617 } 618 619 return true; 620 621 out_xdp_abort: 622 trace_xdp_exception(netdev, xdp_prog, xdp_action); 623 ionic_rx_page_free(rxq, buf_info); 624 stats->xdp_aborted++; 625 626 return true; 627 } 628 629 static void ionic_rx_clean(struct ionic_queue *q, 630 struct ionic_desc_info *desc_info, 631 struct ionic_cq_info *cq_info, 632 void *cb_arg) 633 { 634 struct net_device *netdev = q->lif->netdev; 635 struct ionic_qcq *qcq = q_to_qcq(q); 636 struct ionic_rx_stats *stats; 637 struct ionic_rxq_comp *comp; 638 struct bpf_prog *xdp_prog; 639 unsigned int headroom; 640 struct sk_buff *skb; 641 u16 len; 642 643 comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp); 644 645 stats = q_to_rx_stats(q); 646 647 if (comp->status) { 648 stats->dropped++; 649 return; 650 } 651 652 len = le16_to_cpu(comp->len); 653 stats->pkts++; 654 stats->bytes += len; 655 656 xdp_prog = READ_ONCE(q->lif->xdp_prog); 657 if (xdp_prog && 658 ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len)) 659 return; 660 661 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 662 if (len <= q->lif->rx_copybreak) 663 skb = ionic_rx_copybreak(netdev, q, desc_info, 664 headroom, len, !!xdp_prog); 665 else 666 skb = ionic_rx_frags(netdev, q, desc_info, headroom, len, 667 comp->num_sg_elems, !!xdp_prog); 668 669 if (unlikely(!skb)) { 670 stats->dropped++; 671 return; 672 } 673 674 skb_record_rx_queue(skb, q->index); 675 676 if (likely(netdev->features & NETIF_F_RXHASH)) { 677 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 678 case IONIC_PKT_TYPE_IPV4: 679 case IONIC_PKT_TYPE_IPV6: 680 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 681 PKT_HASH_TYPE_L3); 682 break; 683 case IONIC_PKT_TYPE_IPV4_TCP: 684 case IONIC_PKT_TYPE_IPV6_TCP: 685 case IONIC_PKT_TYPE_IPV4_UDP: 686 case IONIC_PKT_TYPE_IPV6_UDP: 687 skb_set_hash(skb, le32_to_cpu(comp->rss_hash), 688 PKT_HASH_TYPE_L4); 689 break; 690 } 691 } 692 693 if (likely(netdev->features & NETIF_F_RXCSUM) && 694 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC)) { 695 skb->ip_summed = CHECKSUM_COMPLETE; 696 skb->csum = (__force __wsum)le16_to_cpu(comp->csum); 697 stats->csum_complete++; 698 } else { 699 stats->csum_none++; 700 } 701 702 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 703 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) || 704 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD))) 705 stats->csum_error++; 706 707 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && 708 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) { 709 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 710 le16_to_cpu(comp->vlan_tci)); 711 stats->vlan_stripped++; 712 } 713 714 if (unlikely(q->features & IONIC_RXQ_F_HWSTAMP)) { 715 __le64 *cq_desc_hwstamp; 716 u64 hwstamp; 717 718 cq_desc_hwstamp = 719 cq_info->cq_desc + 720 qcq->cq.desc_size - 721 sizeof(struct ionic_rxq_comp) - 722 IONIC_HWSTAMP_CQ_NEGOFFSET; 723 724 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 725 726 if (hwstamp != IONIC_HWSTAMP_INVALID) { 727 skb_hwtstamps(skb)->hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 728 stats->hwstamp_valid++; 729 } else { 730 stats->hwstamp_invalid++; 731 } 732 } 733 734 if (len <= q->lif->rx_copybreak) 735 napi_gro_receive(&qcq->napi, skb); 736 else 737 napi_gro_frags(&qcq->napi); 738 } 739 740 bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info) 741 { 742 struct ionic_queue *q = cq->bound_q; 743 struct ionic_desc_info *desc_info; 744 struct ionic_rxq_comp *comp; 745 746 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 747 748 if (!color_match(comp->pkt_type_color, cq->done_color)) 749 return false; 750 751 /* check for empty queue */ 752 if (q->tail_idx == q->head_idx) 753 return false; 754 755 if (q->tail_idx != le16_to_cpu(comp->comp_index)) 756 return false; 757 758 desc_info = &q->info[q->tail_idx]; 759 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 760 761 /* clean the related q entry, only one per qc completion */ 762 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg); 763 764 desc_info->cb = NULL; 765 desc_info->cb_arg = NULL; 766 767 return true; 768 } 769 770 static inline void ionic_write_cmb_desc(struct ionic_queue *q, 771 void __iomem *cmb_desc, 772 void *desc) 773 { 774 if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS) 775 memcpy_toio(cmb_desc, desc, q->desc_size); 776 } 777 778 void ionic_rx_fill(struct ionic_queue *q) 779 { 780 struct net_device *netdev = q->lif->netdev; 781 struct ionic_desc_info *desc_info; 782 struct ionic_rxq_sg_desc *sg_desc; 783 struct ionic_rxq_sg_elem *sg_elem; 784 struct ionic_buf_info *buf_info; 785 unsigned int fill_threshold; 786 struct ionic_rxq_desc *desc; 787 unsigned int remain_len; 788 unsigned int frag_len; 789 unsigned int nfrags; 790 unsigned int n_fill; 791 unsigned int len; 792 unsigned int i; 793 unsigned int j; 794 795 n_fill = ionic_q_space_avail(q); 796 797 fill_threshold = min_t(unsigned int, IONIC_RX_FILL_THRESHOLD, 798 q->num_descs / IONIC_RX_FILL_DIV); 799 if (n_fill < fill_threshold) 800 return; 801 802 len = netdev->mtu + VLAN_ETH_HLEN; 803 804 for (i = n_fill; i; i--) { 805 unsigned int headroom; 806 unsigned int buf_len; 807 808 nfrags = 0; 809 remain_len = len; 810 desc_info = &q->info[q->head_idx]; 811 desc = desc_info->desc; 812 buf_info = &desc_info->bufs[0]; 813 814 if (!buf_info->page) { /* alloc a new buffer? */ 815 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 816 desc->addr = 0; 817 desc->len = 0; 818 return; 819 } 820 } 821 822 /* fill main descriptor - buf[0] 823 * XDP uses space in the first buffer, so account for 824 * head room, tail room, and ip header in the first frag size. 825 */ 826 headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; 827 if (q->xdp_rxq_info) 828 buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN; 829 else 830 buf_len = ionic_rx_buf_size(buf_info); 831 frag_len = min_t(u16, len, buf_len); 832 833 desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom); 834 desc->len = cpu_to_le16(frag_len); 835 remain_len -= frag_len; 836 buf_info++; 837 nfrags++; 838 839 /* fill sg descriptors - buf[1..n] */ 840 sg_desc = desc_info->sg_desc; 841 for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) { 842 sg_elem = &sg_desc->elems[j]; 843 if (!buf_info->page) { /* alloc a new sg buffer? */ 844 if (unlikely(ionic_rx_page_alloc(q, buf_info))) { 845 sg_elem->addr = 0; 846 sg_elem->len = 0; 847 return; 848 } 849 } 850 851 sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info)); 852 frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info)); 853 sg_elem->len = cpu_to_le16(frag_len); 854 remain_len -= frag_len; 855 buf_info++; 856 nfrags++; 857 } 858 859 /* clear end sg element as a sentinel */ 860 if (j < q->max_sg_elems) { 861 sg_elem = &sg_desc->elems[j]; 862 memset(sg_elem, 0, sizeof(*sg_elem)); 863 } 864 865 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 866 IONIC_RXQ_DESC_OPCODE_SIMPLE; 867 desc_info->nbufs = nfrags; 868 869 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 870 871 ionic_rxq_post(q, false, ionic_rx_clean, NULL); 872 } 873 874 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, 875 q->dbval | q->head_idx); 876 877 q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE; 878 q->dbell_jiffies = jiffies; 879 880 mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline, 881 jiffies + IONIC_NAPI_DEADLINE); 882 } 883 884 void ionic_rx_empty(struct ionic_queue *q) 885 { 886 struct ionic_desc_info *desc_info; 887 struct ionic_buf_info *buf_info; 888 unsigned int i, j; 889 890 for (i = 0; i < q->num_descs; i++) { 891 desc_info = &q->info[i]; 892 for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) { 893 buf_info = &desc_info->bufs[j]; 894 if (buf_info->page) 895 ionic_rx_page_free(q, buf_info); 896 } 897 898 desc_info->nbufs = 0; 899 desc_info->cb = NULL; 900 desc_info->cb_arg = NULL; 901 } 902 903 q->head_idx = 0; 904 q->tail_idx = 0; 905 } 906 907 static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode) 908 { 909 struct dim_sample dim_sample; 910 struct ionic_lif *lif; 911 unsigned int qi; 912 u64 pkts, bytes; 913 914 if (!qcq->intr.dim_coal_hw) 915 return; 916 917 lif = qcq->q.lif; 918 qi = qcq->cq.bound_q->index; 919 920 switch (napi_mode) { 921 case IONIC_LIF_F_TX_DIM_INTR: 922 pkts = lif->txqstats[qi].pkts; 923 bytes = lif->txqstats[qi].bytes; 924 break; 925 case IONIC_LIF_F_RX_DIM_INTR: 926 pkts = lif->rxqstats[qi].pkts; 927 bytes = lif->rxqstats[qi].bytes; 928 break; 929 default: 930 pkts = lif->txqstats[qi].pkts + lif->rxqstats[qi].pkts; 931 bytes = lif->txqstats[qi].bytes + lif->rxqstats[qi].bytes; 932 break; 933 } 934 935 dim_update_sample(qcq->cq.bound_intr->rearm_count, 936 pkts, bytes, &dim_sample); 937 938 net_dim(&qcq->dim, dim_sample); 939 } 940 941 int ionic_tx_napi(struct napi_struct *napi, int budget) 942 { 943 struct ionic_qcq *qcq = napi_to_qcq(napi); 944 struct ionic_cq *cq = napi_to_cq(napi); 945 struct ionic_dev *idev; 946 struct ionic_lif *lif; 947 u32 work_done = 0; 948 u32 flags = 0; 949 950 lif = cq->bound_q->lif; 951 idev = &lif->ionic->idev; 952 953 work_done = ionic_tx_cq_service(cq, budget); 954 955 if (unlikely(!budget)) 956 return budget; 957 958 if (work_done < budget && napi_complete_done(napi, work_done)) { 959 ionic_dim_update(qcq, IONIC_LIF_F_TX_DIM_INTR); 960 flags |= IONIC_INTR_CRED_UNMASK; 961 cq->bound_intr->rearm_count++; 962 } 963 964 if (work_done || flags) { 965 flags |= IONIC_INTR_CRED_RESET_COALESCE; 966 ionic_intr_credits(idev->intr_ctrl, 967 cq->bound_intr->index, 968 work_done, flags); 969 } 970 971 if (!work_done && ionic_txq_poke_doorbell(&qcq->q)) 972 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 973 974 return work_done; 975 } 976 977 static void ionic_xdp_do_flush(struct ionic_cq *cq) 978 { 979 if (cq->bound_q->xdp_flush) { 980 xdp_do_flush(); 981 cq->bound_q->xdp_flush = false; 982 } 983 } 984 985 int ionic_rx_napi(struct napi_struct *napi, int budget) 986 { 987 struct ionic_qcq *qcq = napi_to_qcq(napi); 988 struct ionic_cq *cq = napi_to_cq(napi); 989 struct ionic_dev *idev; 990 struct ionic_lif *lif; 991 u32 work_done = 0; 992 u32 flags = 0; 993 994 if (unlikely(!budget)) 995 return budget; 996 997 lif = cq->bound_q->lif; 998 idev = &lif->ionic->idev; 999 1000 work_done = ionic_cq_service(cq, budget, 1001 ionic_rx_service, NULL, NULL); 1002 1003 ionic_rx_fill(cq->bound_q); 1004 1005 ionic_xdp_do_flush(cq); 1006 if (work_done < budget && napi_complete_done(napi, work_done)) { 1007 ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); 1008 flags |= IONIC_INTR_CRED_UNMASK; 1009 cq->bound_intr->rearm_count++; 1010 } 1011 1012 if (work_done || flags) { 1013 flags |= IONIC_INTR_CRED_RESET_COALESCE; 1014 ionic_intr_credits(idev->intr_ctrl, 1015 cq->bound_intr->index, 1016 work_done, flags); 1017 } 1018 1019 if (!work_done && ionic_rxq_poke_doorbell(&qcq->q)) 1020 mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 1021 1022 return work_done; 1023 } 1024 1025 int ionic_txrx_napi(struct napi_struct *napi, int budget) 1026 { 1027 struct ionic_qcq *rxqcq = napi_to_qcq(napi); 1028 struct ionic_cq *rxcq = napi_to_cq(napi); 1029 unsigned int qi = rxcq->bound_q->index; 1030 struct ionic_qcq *txqcq; 1031 struct ionic_dev *idev; 1032 struct ionic_lif *lif; 1033 struct ionic_cq *txcq; 1034 bool resched = false; 1035 u32 rx_work_done = 0; 1036 u32 tx_work_done = 0; 1037 u32 flags = 0; 1038 1039 lif = rxcq->bound_q->lif; 1040 idev = &lif->ionic->idev; 1041 txqcq = lif->txqcqs[qi]; 1042 txcq = &lif->txqcqs[qi]->cq; 1043 1044 tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT); 1045 1046 if (unlikely(!budget)) 1047 return budget; 1048 1049 rx_work_done = ionic_cq_service(rxcq, budget, 1050 ionic_rx_service, NULL, NULL); 1051 1052 ionic_rx_fill(rxcq->bound_q); 1053 1054 ionic_xdp_do_flush(rxcq); 1055 if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { 1056 ionic_dim_update(rxqcq, 0); 1057 flags |= IONIC_INTR_CRED_UNMASK; 1058 rxcq->bound_intr->rearm_count++; 1059 } 1060 1061 if (rx_work_done || flags) { 1062 flags |= IONIC_INTR_CRED_RESET_COALESCE; 1063 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index, 1064 tx_work_done + rx_work_done, flags); 1065 } 1066 1067 if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q)) 1068 resched = true; 1069 if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q)) 1070 resched = true; 1071 if (resched) 1072 mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE); 1073 1074 return rx_work_done; 1075 } 1076 1077 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, 1078 void *data, size_t len) 1079 { 1080 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1081 struct device *dev = q->dev; 1082 dma_addr_t dma_addr; 1083 1084 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE); 1085 if (dma_mapping_error(dev, dma_addr)) { 1086 net_warn_ratelimited("%s: DMA single map failed on %s!\n", 1087 dev_name(dev), q->name); 1088 stats->dma_map_err++; 1089 return 0; 1090 } 1091 return dma_addr; 1092 } 1093 1094 static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, 1095 const skb_frag_t *frag, 1096 size_t offset, size_t len) 1097 { 1098 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1099 struct device *dev = q->dev; 1100 dma_addr_t dma_addr; 1101 1102 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE); 1103 if (dma_mapping_error(dev, dma_addr)) { 1104 net_warn_ratelimited("%s: DMA frag map failed on %s!\n", 1105 dev_name(dev), q->name); 1106 stats->dma_map_err++; 1107 } 1108 return dma_addr; 1109 } 1110 1111 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb, 1112 struct ionic_desc_info *desc_info) 1113 { 1114 struct ionic_buf_info *buf_info = desc_info->bufs; 1115 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1116 struct device *dev = q->dev; 1117 dma_addr_t dma_addr; 1118 unsigned int nfrags; 1119 skb_frag_t *frag; 1120 int frag_idx; 1121 1122 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb)); 1123 if (dma_mapping_error(dev, dma_addr)) { 1124 stats->dma_map_err++; 1125 return -EIO; 1126 } 1127 buf_info->dma_addr = dma_addr; 1128 buf_info->len = skb_headlen(skb); 1129 buf_info++; 1130 1131 frag = skb_shinfo(skb)->frags; 1132 nfrags = skb_shinfo(skb)->nr_frags; 1133 for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) { 1134 dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); 1135 if (dma_mapping_error(dev, dma_addr)) { 1136 stats->dma_map_err++; 1137 goto dma_fail; 1138 } 1139 buf_info->dma_addr = dma_addr; 1140 buf_info->len = skb_frag_size(frag); 1141 buf_info++; 1142 } 1143 1144 desc_info->nbufs = 1 + nfrags; 1145 1146 return 0; 1147 1148 dma_fail: 1149 /* unwind the frag mappings and the head mapping */ 1150 while (frag_idx > 0) { 1151 frag_idx--; 1152 buf_info--; 1153 dma_unmap_page(dev, buf_info->dma_addr, 1154 buf_info->len, DMA_TO_DEVICE); 1155 } 1156 dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE); 1157 return -EIO; 1158 } 1159 1160 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q, 1161 struct ionic_desc_info *desc_info) 1162 { 1163 struct ionic_buf_info *buf_info = desc_info->bufs; 1164 struct device *dev = q->dev; 1165 unsigned int i; 1166 1167 if (!desc_info->nbufs) 1168 return; 1169 1170 dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr, 1171 buf_info->len, DMA_TO_DEVICE); 1172 buf_info++; 1173 for (i = 1; i < desc_info->nbufs; i++, buf_info++) 1174 dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr, 1175 buf_info->len, DMA_TO_DEVICE); 1176 1177 desc_info->nbufs = 0; 1178 } 1179 1180 static void ionic_tx_clean(struct ionic_queue *q, 1181 struct ionic_desc_info *desc_info, 1182 struct ionic_cq_info *cq_info, 1183 void *cb_arg) 1184 { 1185 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1186 struct ionic_qcq *qcq = q_to_qcq(q); 1187 struct sk_buff *skb = cb_arg; 1188 1189 if (desc_info->xdpf) { 1190 ionic_xdp_tx_desc_clean(q->partner, desc_info); 1191 stats->clean++; 1192 1193 if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index))) 1194 netif_wake_subqueue(q->lif->netdev, q->index); 1195 1196 return; 1197 } 1198 1199 ionic_tx_desc_unmap_bufs(q, desc_info); 1200 1201 if (!skb) 1202 return; 1203 1204 if (unlikely(ionic_txq_hwstamp_enabled(q))) { 1205 if (cq_info) { 1206 struct skb_shared_hwtstamps hwts = {}; 1207 __le64 *cq_desc_hwstamp; 1208 u64 hwstamp; 1209 1210 cq_desc_hwstamp = 1211 cq_info->cq_desc + 1212 qcq->cq.desc_size - 1213 sizeof(struct ionic_txq_comp) - 1214 IONIC_HWSTAMP_CQ_NEGOFFSET; 1215 1216 hwstamp = le64_to_cpu(*cq_desc_hwstamp); 1217 1218 if (hwstamp != IONIC_HWSTAMP_INVALID) { 1219 hwts.hwtstamp = ionic_lif_phc_ktime(q->lif, hwstamp); 1220 1221 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1222 skb_tstamp_tx(skb, &hwts); 1223 1224 stats->hwstamp_valid++; 1225 } else { 1226 stats->hwstamp_invalid++; 1227 } 1228 } 1229 } 1230 1231 desc_info->bytes = skb->len; 1232 stats->clean++; 1233 1234 napi_consume_skb(skb, 1); 1235 } 1236 1237 static bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info, 1238 unsigned int *total_pkts, unsigned int *total_bytes) 1239 { 1240 struct ionic_queue *q = cq->bound_q; 1241 struct ionic_desc_info *desc_info; 1242 struct ionic_txq_comp *comp; 1243 unsigned int bytes = 0; 1244 unsigned int pkts = 0; 1245 u16 index; 1246 1247 comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp); 1248 1249 if (!color_match(comp->color, cq->done_color)) 1250 return false; 1251 1252 /* clean the related q entries, there could be 1253 * several q entries completed for each cq completion 1254 */ 1255 do { 1256 desc_info = &q->info[q->tail_idx]; 1257 desc_info->bytes = 0; 1258 index = q->tail_idx; 1259 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1260 ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg); 1261 if (desc_info->cb_arg) { 1262 pkts++; 1263 bytes += desc_info->bytes; 1264 } 1265 desc_info->cb = NULL; 1266 desc_info->cb_arg = NULL; 1267 } while (index != le16_to_cpu(comp->comp_index)); 1268 1269 (*total_pkts) += pkts; 1270 (*total_bytes) += bytes; 1271 1272 return true; 1273 } 1274 1275 unsigned int ionic_tx_cq_service(struct ionic_cq *cq, unsigned int work_to_do) 1276 { 1277 struct ionic_cq_info *cq_info; 1278 unsigned int work_done = 0; 1279 unsigned int bytes = 0; 1280 unsigned int pkts = 0; 1281 1282 if (work_to_do == 0) 1283 return 0; 1284 1285 cq_info = &cq->info[cq->tail_idx]; 1286 while (ionic_tx_service(cq, cq_info, &pkts, &bytes)) { 1287 if (cq->tail_idx == cq->num_descs - 1) 1288 cq->done_color = !cq->done_color; 1289 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 1290 cq_info = &cq->info[cq->tail_idx]; 1291 1292 if (++work_done >= work_to_do) 1293 break; 1294 } 1295 1296 if (work_done) { 1297 struct ionic_queue *q = cq->bound_q; 1298 1299 if (likely(!ionic_txq_hwstamp_enabled(q))) 1300 netif_txq_completed_wake(q_to_ndq(q->lif->netdev, q), 1301 pkts, bytes, 1302 ionic_q_space_avail(q), 1303 IONIC_TSO_DESCS_NEEDED); 1304 } 1305 1306 return work_done; 1307 } 1308 1309 void ionic_tx_flush(struct ionic_cq *cq) 1310 { 1311 struct ionic_dev *idev = &cq->lif->ionic->idev; 1312 u32 work_done; 1313 1314 work_done = ionic_tx_cq_service(cq, cq->num_descs); 1315 if (work_done) 1316 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, 1317 work_done, IONIC_INTR_CRED_RESET_COALESCE); 1318 } 1319 1320 void ionic_tx_empty(struct ionic_queue *q) 1321 { 1322 struct ionic_desc_info *desc_info; 1323 int bytes = 0; 1324 int pkts = 0; 1325 1326 /* walk the not completed tx entries, if any */ 1327 while (q->head_idx != q->tail_idx) { 1328 desc_info = &q->info[q->tail_idx]; 1329 desc_info->bytes = 0; 1330 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1331 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg); 1332 if (desc_info->cb_arg) { 1333 pkts++; 1334 bytes += desc_info->bytes; 1335 } 1336 desc_info->cb = NULL; 1337 desc_info->cb_arg = NULL; 1338 } 1339 1340 if (likely(!ionic_txq_hwstamp_enabled(q))) { 1341 struct netdev_queue *ndq = q_to_ndq(q->lif->netdev, q); 1342 1343 netdev_tx_completed_queue(ndq, pkts, bytes); 1344 netdev_tx_reset_queue(ndq); 1345 } 1346 } 1347 1348 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb) 1349 { 1350 int err; 1351 1352 err = skb_cow_head(skb, 0); 1353 if (err) 1354 return err; 1355 1356 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 1357 inner_ip_hdr(skb)->check = 0; 1358 inner_tcp_hdr(skb)->check = 1359 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 1360 inner_ip_hdr(skb)->daddr, 1361 0, IPPROTO_TCP, 0); 1362 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 1363 inner_tcp_hdr(skb)->check = 1364 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 1365 &inner_ipv6_hdr(skb)->daddr, 1366 0, IPPROTO_TCP, 0); 1367 } 1368 1369 return 0; 1370 } 1371 1372 static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb) 1373 { 1374 int err; 1375 1376 err = skb_cow_head(skb, 0); 1377 if (err) 1378 return err; 1379 1380 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 1381 ip_hdr(skb)->check = 0; 1382 tcp_hdr(skb)->check = 1383 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 1384 ip_hdr(skb)->daddr, 1385 0, IPPROTO_TCP, 0); 1386 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 1387 tcp_v6_gso_csum_prep(skb); 1388 } 1389 1390 return 0; 1391 } 1392 1393 static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q, 1394 struct ionic_desc_info *desc_info, 1395 struct sk_buff *skb, 1396 dma_addr_t addr, u8 nsge, u16 len, 1397 unsigned int hdrlen, unsigned int mss, 1398 bool outer_csum, 1399 u16 vlan_tci, bool has_vlan, 1400 bool start, bool done) 1401 { 1402 struct ionic_txq_desc *desc = desc_info->desc; 1403 u8 flags = 0; 1404 u64 cmd; 1405 1406 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1407 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1408 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 1409 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 1410 1411 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr); 1412 desc->cmd = cpu_to_le64(cmd); 1413 desc->len = cpu_to_le16(len); 1414 desc->vlan_tci = cpu_to_le16(vlan_tci); 1415 desc->hdr_len = cpu_to_le16(hdrlen); 1416 desc->mss = cpu_to_le16(mss); 1417 1418 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 1419 1420 if (start) { 1421 skb_tx_timestamp(skb); 1422 if (likely(!ionic_txq_hwstamp_enabled(q))) 1423 netdev_tx_sent_queue(q_to_ndq(netdev, q), skb->len); 1424 ionic_txq_post(q, false, ionic_tx_clean, skb); 1425 } else { 1426 ionic_txq_post(q, done, NULL, NULL); 1427 } 1428 } 1429 1430 static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q, 1431 struct sk_buff *skb) 1432 { 1433 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1434 struct ionic_desc_info *desc_info; 1435 struct ionic_buf_info *buf_info; 1436 struct ionic_txq_sg_elem *elem; 1437 struct ionic_txq_desc *desc; 1438 unsigned int chunk_len; 1439 unsigned int frag_rem; 1440 unsigned int tso_rem; 1441 unsigned int seg_rem; 1442 dma_addr_t desc_addr; 1443 dma_addr_t frag_addr; 1444 unsigned int hdrlen; 1445 unsigned int len; 1446 unsigned int mss; 1447 bool start, done; 1448 bool outer_csum; 1449 bool has_vlan; 1450 u16 desc_len; 1451 u8 desc_nsge; 1452 u16 vlan_tci; 1453 bool encap; 1454 int err; 1455 1456 desc_info = &q->info[q->head_idx]; 1457 buf_info = desc_info->bufs; 1458 1459 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1460 return -EIO; 1461 1462 len = skb->len; 1463 mss = skb_shinfo(skb)->gso_size; 1464 outer_csum = (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 1465 SKB_GSO_GRE_CSUM | 1466 SKB_GSO_IPXIP4 | 1467 SKB_GSO_IPXIP6 | 1468 SKB_GSO_UDP_TUNNEL | 1469 SKB_GSO_UDP_TUNNEL_CSUM)); 1470 has_vlan = !!skb_vlan_tag_present(skb); 1471 vlan_tci = skb_vlan_tag_get(skb); 1472 encap = skb->encapsulation; 1473 1474 /* Preload inner-most TCP csum field with IP pseudo hdr 1475 * calculated with IP length set to zero. HW will later 1476 * add in length to each TCP segment resulting from the TSO. 1477 */ 1478 1479 if (encap) 1480 err = ionic_tx_tcp_inner_pseudo_csum(skb); 1481 else 1482 err = ionic_tx_tcp_pseudo_csum(skb); 1483 if (err) { 1484 /* clean up mapping from ionic_tx_map_skb */ 1485 ionic_tx_desc_unmap_bufs(q, desc_info); 1486 return err; 1487 } 1488 1489 if (encap) 1490 hdrlen = skb_inner_tcp_all_headers(skb); 1491 else 1492 hdrlen = skb_tcp_all_headers(skb); 1493 1494 tso_rem = len; 1495 seg_rem = min(tso_rem, hdrlen + mss); 1496 1497 frag_addr = 0; 1498 frag_rem = 0; 1499 1500 start = true; 1501 1502 while (tso_rem > 0) { 1503 desc = NULL; 1504 elem = NULL; 1505 desc_addr = 0; 1506 desc_len = 0; 1507 desc_nsge = 0; 1508 /* use fragments until we have enough to post a single descriptor */ 1509 while (seg_rem > 0) { 1510 /* if the fragment is exhausted then move to the next one */ 1511 if (frag_rem == 0) { 1512 /* grab the next fragment */ 1513 frag_addr = buf_info->dma_addr; 1514 frag_rem = buf_info->len; 1515 buf_info++; 1516 } 1517 chunk_len = min(frag_rem, seg_rem); 1518 if (!desc) { 1519 /* fill main descriptor */ 1520 desc = desc_info->txq_desc; 1521 elem = desc_info->txq_sg_desc->elems; 1522 desc_addr = frag_addr; 1523 desc_len = chunk_len; 1524 } else { 1525 /* fill sg descriptor */ 1526 elem->addr = cpu_to_le64(frag_addr); 1527 elem->len = cpu_to_le16(chunk_len); 1528 elem++; 1529 desc_nsge++; 1530 } 1531 frag_addr += chunk_len; 1532 frag_rem -= chunk_len; 1533 tso_rem -= chunk_len; 1534 seg_rem -= chunk_len; 1535 } 1536 seg_rem = min(tso_rem, mss); 1537 done = (tso_rem == 0); 1538 /* post descriptor */ 1539 ionic_tx_tso_post(netdev, q, desc_info, skb, 1540 desc_addr, desc_nsge, desc_len, 1541 hdrlen, mss, outer_csum, vlan_tci, has_vlan, 1542 start, done); 1543 start = false; 1544 /* Buffer information is stored with the first tso descriptor */ 1545 desc_info = &q->info[q->head_idx]; 1546 desc_info->nbufs = 0; 1547 } 1548 1549 stats->pkts += DIV_ROUND_UP(len - hdrlen, mss); 1550 stats->bytes += len; 1551 stats->tso++; 1552 stats->tso_bytes = len; 1553 1554 return 0; 1555 } 1556 1557 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb, 1558 struct ionic_desc_info *desc_info) 1559 { 1560 struct ionic_txq_desc *desc = desc_info->txq_desc; 1561 struct ionic_buf_info *buf_info = desc_info->bufs; 1562 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1563 bool has_vlan; 1564 u8 flags = 0; 1565 bool encap; 1566 u64 cmd; 1567 1568 has_vlan = !!skb_vlan_tag_present(skb); 1569 encap = skb->encapsulation; 1570 1571 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1572 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1573 1574 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL, 1575 flags, skb_shinfo(skb)->nr_frags, 1576 buf_info->dma_addr); 1577 desc->cmd = cpu_to_le64(cmd); 1578 desc->len = cpu_to_le16(buf_info->len); 1579 if (has_vlan) { 1580 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1581 stats->vlan_inserted++; 1582 } else { 1583 desc->vlan_tci = 0; 1584 } 1585 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb)); 1586 desc->csum_offset = cpu_to_le16(skb->csum_offset); 1587 1588 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 1589 1590 if (skb_csum_is_sctp(skb)) 1591 stats->crc32_csum++; 1592 else 1593 stats->csum++; 1594 } 1595 1596 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb, 1597 struct ionic_desc_info *desc_info) 1598 { 1599 struct ionic_txq_desc *desc = desc_info->txq_desc; 1600 struct ionic_buf_info *buf_info = desc_info->bufs; 1601 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1602 bool has_vlan; 1603 u8 flags = 0; 1604 bool encap; 1605 u64 cmd; 1606 1607 has_vlan = !!skb_vlan_tag_present(skb); 1608 encap = skb->encapsulation; 1609 1610 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 1611 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 1612 1613 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, 1614 flags, skb_shinfo(skb)->nr_frags, 1615 buf_info->dma_addr); 1616 desc->cmd = cpu_to_le64(cmd); 1617 desc->len = cpu_to_le16(buf_info->len); 1618 if (has_vlan) { 1619 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb)); 1620 stats->vlan_inserted++; 1621 } else { 1622 desc->vlan_tci = 0; 1623 } 1624 desc->csum_start = 0; 1625 desc->csum_offset = 0; 1626 1627 ionic_write_cmb_desc(q, desc_info->cmb_desc, desc); 1628 1629 stats->csum_none++; 1630 } 1631 1632 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb, 1633 struct ionic_desc_info *desc_info) 1634 { 1635 struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc; 1636 struct ionic_buf_info *buf_info = &desc_info->bufs[1]; 1637 struct ionic_txq_sg_elem *elem = sg_desc->elems; 1638 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1639 unsigned int i; 1640 1641 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) { 1642 elem->addr = cpu_to_le64(buf_info->dma_addr); 1643 elem->len = cpu_to_le16(buf_info->len); 1644 } 1645 1646 stats->frags += skb_shinfo(skb)->nr_frags; 1647 } 1648 1649 static int ionic_tx(struct net_device *netdev, struct ionic_queue *q, 1650 struct sk_buff *skb) 1651 { 1652 struct ionic_desc_info *desc_info = &q->info[q->head_idx]; 1653 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1654 bool ring_dbell = true; 1655 1656 if (unlikely(ionic_tx_map_skb(q, skb, desc_info))) 1657 return -EIO; 1658 1659 /* set up the initial descriptor */ 1660 if (skb->ip_summed == CHECKSUM_PARTIAL) 1661 ionic_tx_calc_csum(q, skb, desc_info); 1662 else 1663 ionic_tx_calc_no_csum(q, skb, desc_info); 1664 1665 /* add frags */ 1666 ionic_tx_skb_frags(q, skb, desc_info); 1667 1668 skb_tx_timestamp(skb); 1669 stats->pkts++; 1670 stats->bytes += skb->len; 1671 1672 if (likely(!ionic_txq_hwstamp_enabled(q))) { 1673 struct netdev_queue *ndq = q_to_ndq(netdev, q); 1674 1675 if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1))) 1676 netif_tx_stop_queue(ndq); 1677 ring_dbell = __netdev_tx_sent_queue(ndq, skb->len, 1678 netdev_xmit_more()); 1679 } 1680 ionic_txq_post(q, ring_dbell, ionic_tx_clean, skb); 1681 1682 return 0; 1683 } 1684 1685 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb) 1686 { 1687 int nr_frags = skb_shinfo(skb)->nr_frags; 1688 bool too_many_frags = false; 1689 skb_frag_t *frag; 1690 int desc_bufs; 1691 int chunk_len; 1692 int frag_rem; 1693 int tso_rem; 1694 int seg_rem; 1695 bool encap; 1696 int hdrlen; 1697 int ndescs; 1698 int err; 1699 1700 /* Each desc is mss long max, so a descriptor for each gso_seg */ 1701 if (skb_is_gso(skb)) { 1702 ndescs = skb_shinfo(skb)->gso_segs; 1703 if (!nr_frags) 1704 return ndescs; 1705 } else { 1706 ndescs = 1; 1707 if (!nr_frags) 1708 return ndescs; 1709 1710 if (unlikely(nr_frags > q->max_sg_elems)) { 1711 too_many_frags = true; 1712 goto linearize; 1713 } 1714 1715 return ndescs; 1716 } 1717 1718 /* We need to scan the skb to be sure that none of the MTU sized 1719 * packets in the TSO will require more sgs per descriptor than we 1720 * can support. We loop through the frags, add up the lengths for 1721 * a packet, and count the number of sgs used per packet. 1722 */ 1723 tso_rem = skb->len; 1724 frag = skb_shinfo(skb)->frags; 1725 encap = skb->encapsulation; 1726 1727 /* start with just hdr in first part of first descriptor */ 1728 if (encap) 1729 hdrlen = skb_inner_tcp_all_headers(skb); 1730 else 1731 hdrlen = skb_tcp_all_headers(skb); 1732 seg_rem = min_t(int, tso_rem, hdrlen + skb_shinfo(skb)->gso_size); 1733 frag_rem = hdrlen; 1734 1735 while (tso_rem > 0) { 1736 desc_bufs = 0; 1737 while (seg_rem > 0) { 1738 desc_bufs++; 1739 1740 /* We add the +1 because we can take buffers for one 1741 * more than we have SGs: one for the initial desc data 1742 * in addition to the SG segments that might follow. 1743 */ 1744 if (desc_bufs > q->max_sg_elems + 1) { 1745 too_many_frags = true; 1746 goto linearize; 1747 } 1748 1749 if (frag_rem == 0) { 1750 frag_rem = skb_frag_size(frag); 1751 frag++; 1752 } 1753 chunk_len = min(frag_rem, seg_rem); 1754 frag_rem -= chunk_len; 1755 tso_rem -= chunk_len; 1756 seg_rem -= chunk_len; 1757 } 1758 1759 seg_rem = min_t(int, tso_rem, skb_shinfo(skb)->gso_size); 1760 } 1761 1762 linearize: 1763 if (too_many_frags) { 1764 struct ionic_tx_stats *stats = q_to_tx_stats(q); 1765 1766 err = skb_linearize(skb); 1767 if (err) 1768 return err; 1769 stats->linearize++; 1770 } 1771 1772 return ndescs; 1773 } 1774 1775 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb, 1776 struct net_device *netdev) 1777 { 1778 struct ionic_lif *lif = netdev_priv(netdev); 1779 struct ionic_queue *q; 1780 int err, ndescs; 1781 1782 /* Does not stop/start txq, because we post to a separate tx queue 1783 * for timestamping, and if a packet can't be posted immediately to 1784 * the timestamping queue, it is dropped. 1785 */ 1786 1787 q = &lif->hwstamp_txq->q; 1788 ndescs = ionic_tx_descs_needed(q, skb); 1789 if (unlikely(ndescs < 0)) 1790 goto err_out_drop; 1791 1792 if (unlikely(!ionic_q_has_space(q, ndescs))) 1793 goto err_out_drop; 1794 1795 skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP; 1796 if (skb_is_gso(skb)) 1797 err = ionic_tx_tso(netdev, q, skb); 1798 else 1799 err = ionic_tx(netdev, q, skb); 1800 1801 if (err) 1802 goto err_out_drop; 1803 1804 return NETDEV_TX_OK; 1805 1806 err_out_drop: 1807 q->drop++; 1808 dev_kfree_skb(skb); 1809 return NETDEV_TX_OK; 1810 } 1811 1812 netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev) 1813 { 1814 u16 queue_index = skb_get_queue_mapping(skb); 1815 struct ionic_lif *lif = netdev_priv(netdev); 1816 struct ionic_queue *q; 1817 int ndescs; 1818 int err; 1819 1820 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) { 1821 dev_kfree_skb(skb); 1822 return NETDEV_TX_OK; 1823 } 1824 1825 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 1826 if (lif->hwstamp_txq && lif->phc->ts_config_tx_mode) 1827 return ionic_start_hwstamp_xmit(skb, netdev); 1828 1829 if (unlikely(queue_index >= lif->nxqs)) 1830 queue_index = 0; 1831 q = &lif->txqcqs[queue_index]->q; 1832 1833 ndescs = ionic_tx_descs_needed(q, skb); 1834 if (ndescs < 0) 1835 goto err_out_drop; 1836 1837 if (!netif_txq_maybe_stop(q_to_ndq(netdev, q), 1838 ionic_q_space_avail(q), 1839 ndescs, ndescs)) 1840 return NETDEV_TX_BUSY; 1841 1842 if (skb_is_gso(skb)) 1843 err = ionic_tx_tso(netdev, q, skb); 1844 else 1845 err = ionic_tx(netdev, q, skb); 1846 1847 if (err) 1848 goto err_out_drop; 1849 1850 return NETDEV_TX_OK; 1851 1852 err_out_drop: 1853 q->drop++; 1854 dev_kfree_skb(skb); 1855 return NETDEV_TX_OK; 1856 } 1857