1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Atlantic Network Driver 3 * 4 * Copyright (C) 2014-2019 aQuantia Corporation 5 * Copyright (C) 2019-2020 Marvell International Ltd. 6 */ 7 8 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ 9 10 #include "aq_nic.h" 11 #include "aq_hw.h" 12 #include "aq_hw_utils.h" 13 #include "aq_ptp.h" 14 #include "aq_vec.h" 15 #include "aq_main.h" 16 17 #include <net/xdp.h> 18 #include <linux/filter.h> 19 #include <linux/bpf_trace.h> 20 #include <linux/netdevice.h> 21 #include <linux/etherdevice.h> 22 23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff, 24 struct xdp_buff *xdp) 25 { 26 struct skb_shared_info *sinfo; 27 int i; 28 29 if (xdp_buff_has_frags(xdp)) { 30 sinfo = xdp_get_shared_info_from_buff(xdp); 31 32 for (i = 0; i < sinfo->nr_frags; i++) { 33 skb_frag_t *frag = &sinfo->frags[i]; 34 35 page_ref_inc(skb_frag_page(frag)); 36 } 37 } 38 page_ref_inc(buff->rxdata.page); 39 } 40 41 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev) 42 { 43 unsigned int len = PAGE_SIZE << rxpage->order; 44 45 dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE); 46 47 /* Drop the ref for being in the ring. */ 48 __free_pages(rxpage->page, rxpage->order); 49 rxpage->page = NULL; 50 } 51 52 static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring) 53 { 54 struct device *dev = aq_nic_get_dev(rx_ring->aq_nic); 55 unsigned int order = rx_ring->page_order; 56 struct page *page; 57 int ret = -ENOMEM; 58 dma_addr_t daddr; 59 60 page = dev_alloc_pages(order); 61 if (unlikely(!page)) 62 goto err_exit; 63 64 daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order, 65 DMA_FROM_DEVICE); 66 67 if (unlikely(dma_mapping_error(dev, daddr))) 68 goto free_page; 69 70 rxpage->page = page; 71 rxpage->daddr = daddr; 72 rxpage->order = order; 73 rxpage->pg_off = rx_ring->page_offset; 74 75 return 0; 76 77 free_page: 78 __free_pages(page, order); 79 80 err_exit: 81 return ret; 82 } 83 84 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf) 85 { 86 unsigned int order = self->page_order; 87 u16 page_offset = self->page_offset; 88 u16 frame_max = self->frame_max; 89 u16 tail_size = self->tail_size; 90 int ret; 91 92 if (rxbuf->rxdata.page) { 93 /* One means ring is the only user and can reuse */ 94 if (page_ref_count(rxbuf->rxdata.page) > 1) { 95 /* Try reuse buffer */ 96 rxbuf->rxdata.pg_off += frame_max + page_offset + 97 tail_size; 98 if (rxbuf->rxdata.pg_off + frame_max + tail_size <= 99 (PAGE_SIZE << order)) { 100 u64_stats_update_begin(&self->stats.rx.syncp); 101 self->stats.rx.pg_flips++; 102 u64_stats_update_end(&self->stats.rx.syncp); 103 104 } else { 105 /* Buffer exhausted. We have other users and 106 * should release this page and realloc 107 */ 108 aq_free_rxpage(&rxbuf->rxdata, 109 aq_nic_get_dev(self->aq_nic)); 110 u64_stats_update_begin(&self->stats.rx.syncp); 111 self->stats.rx.pg_losts++; 112 u64_stats_update_end(&self->stats.rx.syncp); 113 } 114 } else { 115 rxbuf->rxdata.pg_off = page_offset; 116 u64_stats_update_begin(&self->stats.rx.syncp); 117 self->stats.rx.pg_reuses++; 118 u64_stats_update_end(&self->stats.rx.syncp); 119 } 120 } 121 122 if (!rxbuf->rxdata.page) { 123 ret = aq_alloc_rxpages(&rxbuf->rxdata, self); 124 if (ret) { 125 u64_stats_update_begin(&self->stats.rx.syncp); 126 self->stats.rx.alloc_fails++; 127 u64_stats_update_end(&self->stats.rx.syncp); 128 } 129 return ret; 130 } 131 132 return 0; 133 } 134 135 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, 136 struct aq_nic_s *aq_nic) 137 { 138 int err = 0; 139 140 self->buff_ring = 141 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); 142 143 if (!self->buff_ring) { 144 err = -ENOMEM; 145 goto err_exit; 146 } 147 148 self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), 149 self->size * self->dx_size, 150 &self->dx_ring_pa, GFP_KERNEL); 151 if (!self->dx_ring) { 152 err = -ENOMEM; 153 goto err_exit; 154 } 155 156 err_exit: 157 if (err < 0) { 158 aq_ring_free(self); 159 self = NULL; 160 } 161 162 return self; 163 } 164 165 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, 166 struct aq_nic_s *aq_nic, 167 unsigned int idx, 168 struct aq_nic_cfg_s *aq_nic_cfg) 169 { 170 int err = 0; 171 172 self->aq_nic = aq_nic; 173 self->idx = idx; 174 self->size = aq_nic_cfg->txds; 175 self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; 176 177 self = aq_ring_alloc(self, aq_nic); 178 if (!self) { 179 err = -ENOMEM; 180 goto err_exit; 181 } 182 183 err_exit: 184 if (err < 0) { 185 aq_ring_free(self); 186 self = NULL; 187 } 188 189 return self; 190 } 191 192 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, 193 struct aq_nic_s *aq_nic, 194 unsigned int idx, 195 struct aq_nic_cfg_s *aq_nic_cfg) 196 { 197 int err = 0; 198 199 self->aq_nic = aq_nic; 200 self->idx = idx; 201 self->size = aq_nic_cfg->rxds; 202 self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; 203 self->xdp_prog = aq_nic->xdp_prog; 204 self->frame_max = AQ_CFG_RX_FRAME_MAX; 205 206 /* Only order-2 is allowed if XDP is enabled */ 207 if (READ_ONCE(self->xdp_prog)) { 208 self->page_offset = AQ_XDP_HEADROOM; 209 self->page_order = AQ_CFG_XDP_PAGEORDER; 210 self->tail_size = AQ_XDP_TAILROOM; 211 } else { 212 self->page_offset = 0; 213 self->page_order = fls(self->frame_max / PAGE_SIZE + 214 (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1; 215 if (aq_nic_cfg->rxpageorder > self->page_order) 216 self->page_order = aq_nic_cfg->rxpageorder; 217 self->tail_size = 0; 218 } 219 220 self = aq_ring_alloc(self, aq_nic); 221 if (!self) { 222 err = -ENOMEM; 223 goto err_exit; 224 } 225 226 err_exit: 227 if (err < 0) { 228 aq_ring_free(self); 229 self = NULL; 230 } 231 232 return self; 233 } 234 235 struct aq_ring_s * 236 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic, 237 unsigned int idx, unsigned int size, unsigned int dx_size) 238 { 239 struct device *dev = aq_nic_get_dev(aq_nic); 240 size_t sz = size * dx_size + AQ_CFG_RXDS_DEF; 241 242 memset(self, 0, sizeof(*self)); 243 244 self->aq_nic = aq_nic; 245 self->idx = idx; 246 self->size = size; 247 self->dx_size = dx_size; 248 249 self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa, 250 GFP_KERNEL); 251 if (!self->dx_ring) { 252 aq_ring_free(self); 253 return NULL; 254 } 255 256 return self; 257 } 258 259 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type) 260 { 261 self->hw_head = 0; 262 self->sw_head = 0; 263 self->sw_tail = 0; 264 self->ring_type = ring_type; 265 266 if (self->ring_type == ATL_RING_RX) 267 u64_stats_init(&self->stats.rx.syncp); 268 else 269 u64_stats_init(&self->stats.tx.syncp); 270 271 return 0; 272 } 273 274 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, 275 unsigned int t) 276 { 277 return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); 278 } 279 280 void aq_ring_update_queue_state(struct aq_ring_s *ring) 281 { 282 if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX) 283 aq_ring_queue_stop(ring); 284 else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES) 285 aq_ring_queue_wake(ring); 286 } 287 288 void aq_ring_queue_wake(struct aq_ring_s *ring) 289 { 290 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 291 292 if (__netif_subqueue_stopped(ndev, 293 AQ_NIC_RING2QMAP(ring->aq_nic, 294 ring->idx))) { 295 netif_wake_subqueue(ndev, 296 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 297 u64_stats_update_begin(&ring->stats.tx.syncp); 298 ring->stats.tx.queue_restarts++; 299 u64_stats_update_end(&ring->stats.tx.syncp); 300 } 301 } 302 303 void aq_ring_queue_stop(struct aq_ring_s *ring) 304 { 305 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 306 307 if (!__netif_subqueue_stopped(ndev, 308 AQ_NIC_RING2QMAP(ring->aq_nic, 309 ring->idx))) 310 netif_stop_subqueue(ndev, 311 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 312 } 313 314 bool aq_ring_tx_clean(struct aq_ring_s *self) 315 { 316 struct device *dev = aq_nic_get_dev(self->aq_nic); 317 unsigned int budget; 318 319 for (budget = AQ_CFG_TX_CLEAN_BUDGET; 320 budget && self->sw_head != self->hw_head; budget--) { 321 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 322 323 if (likely(buff->is_mapped)) { 324 if (unlikely(buff->is_sop)) { 325 if (!buff->is_eop && 326 buff->eop_index != 0xffffU && 327 (!aq_ring_dx_in_range(self->sw_head, 328 buff->eop_index, 329 self->hw_head))) 330 break; 331 332 dma_unmap_single(dev, buff->pa, buff->len, 333 DMA_TO_DEVICE); 334 } else { 335 dma_unmap_page(dev, buff->pa, buff->len, 336 DMA_TO_DEVICE); 337 } 338 } 339 340 if (likely(!buff->is_eop)) 341 goto out; 342 343 if (buff->skb) { 344 u64_stats_update_begin(&self->stats.tx.syncp); 345 ++self->stats.tx.packets; 346 self->stats.tx.bytes += buff->skb->len; 347 u64_stats_update_end(&self->stats.tx.syncp); 348 dev_kfree_skb_any(buff->skb); 349 } else if (buff->xdpf) { 350 u64_stats_update_begin(&self->stats.tx.syncp); 351 ++self->stats.tx.packets; 352 self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf); 353 u64_stats_update_end(&self->stats.tx.syncp); 354 xdp_return_frame_rx_napi(buff->xdpf); 355 } 356 357 out: 358 buff->skb = NULL; 359 buff->xdpf = NULL; 360 buff->pa = 0U; 361 buff->eop_index = 0xffffU; 362 self->sw_head = aq_ring_next_dx(self, self->sw_head); 363 } 364 365 return !!budget; 366 } 367 368 static void aq_rx_checksum(struct aq_ring_s *self, 369 struct aq_ring_buff_s *buff, 370 struct sk_buff *skb) 371 { 372 if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM)) 373 return; 374 375 if (unlikely(buff->is_cso_err)) { 376 u64_stats_update_begin(&self->stats.rx.syncp); 377 ++self->stats.rx.errors; 378 u64_stats_update_end(&self->stats.rx.syncp); 379 skb->ip_summed = CHECKSUM_NONE; 380 return; 381 } 382 if (buff->is_ip_cso) { 383 __skb_incr_checksum_unnecessary(skb); 384 } else { 385 skb->ip_summed = CHECKSUM_NONE; 386 } 387 388 if (buff->is_udp_cso || buff->is_tcp_cso) 389 __skb_incr_checksum_unnecessary(skb); 390 } 391 392 int aq_xdp_xmit(struct net_device *dev, int num_frames, 393 struct xdp_frame **frames, u32 flags) 394 { 395 struct aq_nic_s *aq_nic = netdev_priv(dev); 396 unsigned int vec, i, drop = 0; 397 int cpu = smp_processor_id(); 398 struct aq_nic_cfg_s *aq_cfg; 399 struct aq_ring_s *ring; 400 401 aq_cfg = aq_nic_get_cfg(aq_nic); 402 vec = cpu % aq_cfg->vecs; 403 ring = aq_nic->aq_ring_tx[AQ_NIC_CFG_TCVEC2RING(aq_cfg, 0, vec)]; 404 405 for (i = 0; i < num_frames; i++) { 406 struct xdp_frame *xdpf = frames[i]; 407 408 if (aq_nic_xmit_xdpf(aq_nic, ring, xdpf) == NETDEV_TX_BUSY) 409 drop++; 410 } 411 412 return num_frames - drop; 413 } 414 415 static struct sk_buff *aq_xdp_build_skb(struct xdp_buff *xdp, 416 struct net_device *dev, 417 struct aq_ring_buff_s *buff) 418 { 419 struct xdp_frame *xdpf; 420 struct sk_buff *skb; 421 422 xdpf = xdp_convert_buff_to_frame(xdp); 423 if (unlikely(!xdpf)) 424 return NULL; 425 426 skb = xdp_build_skb_from_frame(xdpf, dev); 427 if (!skb) 428 return NULL; 429 430 aq_get_rxpages_xdp(buff, xdp); 431 return skb; 432 } 433 434 static struct sk_buff *aq_xdp_run_prog(struct aq_nic_s *aq_nic, 435 struct xdp_buff *xdp, 436 struct aq_ring_s *rx_ring, 437 struct aq_ring_buff_s *buff) 438 { 439 int result = NETDEV_TX_BUSY; 440 struct aq_ring_s *tx_ring; 441 struct xdp_frame *xdpf; 442 struct bpf_prog *prog; 443 u32 act = XDP_ABORTED; 444 struct sk_buff *skb; 445 446 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 447 ++rx_ring->stats.rx.packets; 448 rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp); 449 u64_stats_update_end(&rx_ring->stats.rx.syncp); 450 451 prog = READ_ONCE(rx_ring->xdp_prog); 452 if (!prog) 453 return aq_xdp_build_skb(xdp, aq_nic->ndev, buff); 454 455 prefetchw(xdp->data_hard_start); /* xdp_frame write */ 456 457 /* single buffer XDP program, but packet is multi buffer, aborted */ 458 if (xdp_buff_has_frags(xdp) && !prog->aux->xdp_has_frags) 459 goto out_aborted; 460 461 act = bpf_prog_run_xdp(prog, xdp); 462 switch (act) { 463 case XDP_PASS: 464 skb = aq_xdp_build_skb(xdp, aq_nic->ndev, buff); 465 if (!skb) 466 goto out_aborted; 467 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 468 ++rx_ring->stats.rx.xdp_pass; 469 u64_stats_update_end(&rx_ring->stats.rx.syncp); 470 return skb; 471 case XDP_TX: 472 xdpf = xdp_convert_buff_to_frame(xdp); 473 if (unlikely(!xdpf)) 474 goto out_aborted; 475 tx_ring = aq_nic->aq_ring_tx[rx_ring->idx]; 476 result = aq_nic_xmit_xdpf(aq_nic, tx_ring, xdpf); 477 if (result == NETDEV_TX_BUSY) 478 goto out_aborted; 479 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 480 ++rx_ring->stats.rx.xdp_tx; 481 u64_stats_update_end(&rx_ring->stats.rx.syncp); 482 aq_get_rxpages_xdp(buff, xdp); 483 break; 484 case XDP_REDIRECT: 485 if (xdp_do_redirect(aq_nic->ndev, xdp, prog) < 0) 486 goto out_aborted; 487 xdp_do_flush(); 488 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 489 ++rx_ring->stats.rx.xdp_redirect; 490 u64_stats_update_end(&rx_ring->stats.rx.syncp); 491 aq_get_rxpages_xdp(buff, xdp); 492 break; 493 default: 494 fallthrough; 495 case XDP_ABORTED: 496 out_aborted: 497 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 498 ++rx_ring->stats.rx.xdp_aborted; 499 u64_stats_update_end(&rx_ring->stats.rx.syncp); 500 trace_xdp_exception(aq_nic->ndev, prog, act); 501 bpf_warn_invalid_xdp_action(aq_nic->ndev, prog, act); 502 break; 503 case XDP_DROP: 504 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 505 ++rx_ring->stats.rx.xdp_drop; 506 u64_stats_update_end(&rx_ring->stats.rx.syncp); 507 break; 508 } 509 510 return ERR_PTR(-result); 511 } 512 513 static bool aq_add_rx_fragment(struct device *dev, 514 struct aq_ring_s *ring, 515 struct aq_ring_buff_s *buff, 516 struct xdp_buff *xdp) 517 { 518 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 519 struct aq_ring_buff_s *buff_ = buff; 520 521 memset(sinfo, 0, sizeof(*sinfo)); 522 do { 523 skb_frag_t *frag; 524 525 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) 526 return true; 527 528 frag = &sinfo->frags[sinfo->nr_frags++]; 529 buff_ = &ring->buff_ring[buff_->next]; 530 dma_sync_single_range_for_cpu(dev, 531 buff_->rxdata.daddr, 532 buff_->rxdata.pg_off, 533 buff_->len, 534 DMA_FROM_DEVICE); 535 sinfo->xdp_frags_size += buff_->len; 536 skb_frag_fill_page_desc(frag, buff_->rxdata.page, 537 buff_->rxdata.pg_off, 538 buff_->len); 539 540 buff_->is_cleaned = 1; 541 542 buff->is_ip_cso &= buff_->is_ip_cso; 543 buff->is_udp_cso &= buff_->is_udp_cso; 544 buff->is_tcp_cso &= buff_->is_tcp_cso; 545 buff->is_cso_err |= buff_->is_cso_err; 546 547 if (page_is_pfmemalloc(buff_->rxdata.page)) 548 xdp_buff_set_frag_pfmemalloc(xdp); 549 550 } while (!buff_->is_eop); 551 552 xdp_buff_set_frags_flag(xdp); 553 554 return false; 555 } 556 557 static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi, 558 int *work_done, int budget) 559 { 560 struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); 561 int err = 0; 562 563 for (; (self->sw_head != self->hw_head) && budget; 564 self->sw_head = aq_ring_next_dx(self, self->sw_head), 565 --budget, ++(*work_done)) { 566 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 567 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self); 568 struct aq_ring_buff_s *buff_ = NULL; 569 struct sk_buff *skb = NULL; 570 unsigned int next_ = 0U; 571 unsigned int i = 0U; 572 u16 hdr_len; 573 574 if (buff->is_cleaned) 575 continue; 576 577 if (!buff->is_eop) { 578 unsigned int frag_cnt = 0U; 579 buff_ = buff; 580 do { 581 bool is_rsc_completed = true; 582 583 if (buff_->next >= self->size) { 584 err = -EIO; 585 goto err_exit; 586 } 587 588 frag_cnt++; 589 next_ = buff_->next, 590 buff_ = &self->buff_ring[next_]; 591 is_rsc_completed = 592 aq_ring_dx_in_range(self->sw_head, 593 next_, 594 self->hw_head); 595 596 if (unlikely(!is_rsc_completed) || 597 frag_cnt > MAX_SKB_FRAGS) { 598 err = 0; 599 goto err_exit; 600 } 601 602 buff->is_error |= buff_->is_error; 603 buff->is_cso_err |= buff_->is_cso_err; 604 605 } while (!buff_->is_eop); 606 607 if (buff->is_error || 608 (buff->is_lro && buff->is_cso_err)) { 609 buff_ = buff; 610 do { 611 if (buff_->next >= self->size) { 612 err = -EIO; 613 goto err_exit; 614 } 615 next_ = buff_->next, 616 buff_ = &self->buff_ring[next_]; 617 618 buff_->is_cleaned = true; 619 } while (!buff_->is_eop); 620 621 u64_stats_update_begin(&self->stats.rx.syncp); 622 ++self->stats.rx.errors; 623 u64_stats_update_end(&self->stats.rx.syncp); 624 continue; 625 } 626 } 627 628 if (buff->is_error) { 629 u64_stats_update_begin(&self->stats.rx.syncp); 630 ++self->stats.rx.errors; 631 u64_stats_update_end(&self->stats.rx.syncp); 632 continue; 633 } 634 635 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 636 buff->rxdata.daddr, 637 buff->rxdata.pg_off, 638 buff->len, DMA_FROM_DEVICE); 639 640 skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE); 641 if (unlikely(!skb)) { 642 u64_stats_update_begin(&self->stats.rx.syncp); 643 self->stats.rx.skb_alloc_fails++; 644 u64_stats_update_end(&self->stats.rx.syncp); 645 err = -ENOMEM; 646 goto err_exit; 647 } 648 if (is_ptp_ring) 649 buff->len -= 650 aq_ptp_extract_ts(self->aq_nic, skb, 651 aq_buf_vaddr(&buff->rxdata), 652 buff->len); 653 654 hdr_len = buff->len; 655 if (hdr_len > AQ_CFG_RX_HDR_SIZE) 656 hdr_len = eth_get_headlen(skb->dev, 657 aq_buf_vaddr(&buff->rxdata), 658 AQ_CFG_RX_HDR_SIZE); 659 660 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata), 661 ALIGN(hdr_len, sizeof(long))); 662 663 if (buff->len - hdr_len > 0) { 664 skb_add_rx_frag(skb, i++, buff->rxdata.page, 665 buff->rxdata.pg_off + hdr_len, 666 buff->len - hdr_len, 667 self->frame_max); 668 page_ref_inc(buff->rxdata.page); 669 } 670 671 if (!buff->is_eop) { 672 buff_ = buff; 673 do { 674 next_ = buff_->next; 675 buff_ = &self->buff_ring[next_]; 676 677 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 678 buff_->rxdata.daddr, 679 buff_->rxdata.pg_off, 680 buff_->len, 681 DMA_FROM_DEVICE); 682 skb_add_rx_frag(skb, i++, 683 buff_->rxdata.page, 684 buff_->rxdata.pg_off, 685 buff_->len, 686 self->frame_max); 687 page_ref_inc(buff_->rxdata.page); 688 buff_->is_cleaned = 1; 689 690 buff->is_ip_cso &= buff_->is_ip_cso; 691 buff->is_udp_cso &= buff_->is_udp_cso; 692 buff->is_tcp_cso &= buff_->is_tcp_cso; 693 buff->is_cso_err |= buff_->is_cso_err; 694 695 } while (!buff_->is_eop); 696 } 697 698 if (buff->is_vlan) 699 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 700 buff->vlan_rx_tag); 701 702 skb->protocol = eth_type_trans(skb, ndev); 703 704 aq_rx_checksum(self, buff, skb); 705 706 skb_set_hash(skb, buff->rss_hash, 707 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 708 PKT_HASH_TYPE_NONE); 709 /* Send all PTP traffic to 0 queue */ 710 skb_record_rx_queue(skb, 711 is_ptp_ring ? 0 712 : AQ_NIC_RING2QMAP(self->aq_nic, 713 self->idx)); 714 715 u64_stats_update_begin(&self->stats.rx.syncp); 716 ++self->stats.rx.packets; 717 self->stats.rx.bytes += skb->len; 718 u64_stats_update_end(&self->stats.rx.syncp); 719 720 napi_gro_receive(napi, skb); 721 } 722 723 err_exit: 724 return err; 725 } 726 727 static int __aq_ring_xdp_clean(struct aq_ring_s *rx_ring, 728 struct napi_struct *napi, int *work_done, 729 int budget) 730 { 731 int frame_sz = rx_ring->page_offset + rx_ring->frame_max + 732 rx_ring->tail_size; 733 struct aq_nic_s *aq_nic = rx_ring->aq_nic; 734 bool is_rsc_completed = true; 735 struct device *dev; 736 int err = 0; 737 738 dev = aq_nic_get_dev(aq_nic); 739 for (; (rx_ring->sw_head != rx_ring->hw_head) && budget; 740 rx_ring->sw_head = aq_ring_next_dx(rx_ring, rx_ring->sw_head), 741 --budget, ++(*work_done)) { 742 struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head]; 743 bool is_ptp_ring = aq_ptp_ring(rx_ring->aq_nic, rx_ring); 744 struct aq_ring_buff_s *buff_ = NULL; 745 struct sk_buff *skb = NULL; 746 unsigned int next_ = 0U; 747 struct xdp_buff xdp; 748 void *hard_start; 749 750 if (buff->is_cleaned) 751 continue; 752 753 if (!buff->is_eop) { 754 buff_ = buff; 755 do { 756 if (buff_->next >= rx_ring->size) { 757 err = -EIO; 758 goto err_exit; 759 } 760 next_ = buff_->next; 761 buff_ = &rx_ring->buff_ring[next_]; 762 is_rsc_completed = 763 aq_ring_dx_in_range(rx_ring->sw_head, 764 next_, 765 rx_ring->hw_head); 766 767 if (unlikely(!is_rsc_completed)) 768 break; 769 770 buff->is_error |= buff_->is_error; 771 buff->is_cso_err |= buff_->is_cso_err; 772 } while (!buff_->is_eop); 773 774 if (!is_rsc_completed) { 775 err = 0; 776 goto err_exit; 777 } 778 if (buff->is_error || 779 (buff->is_lro && buff->is_cso_err)) { 780 buff_ = buff; 781 do { 782 if (buff_->next >= rx_ring->size) { 783 err = -EIO; 784 goto err_exit; 785 } 786 next_ = buff_->next; 787 buff_ = &rx_ring->buff_ring[next_]; 788 789 buff_->is_cleaned = true; 790 } while (!buff_->is_eop); 791 792 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 793 ++rx_ring->stats.rx.errors; 794 u64_stats_update_end(&rx_ring->stats.rx.syncp); 795 continue; 796 } 797 } 798 799 if (buff->is_error) { 800 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 801 ++rx_ring->stats.rx.errors; 802 u64_stats_update_end(&rx_ring->stats.rx.syncp); 803 continue; 804 } 805 806 dma_sync_single_range_for_cpu(dev, 807 buff->rxdata.daddr, 808 buff->rxdata.pg_off, 809 buff->len, DMA_FROM_DEVICE); 810 hard_start = page_address(buff->rxdata.page) + 811 buff->rxdata.pg_off - rx_ring->page_offset; 812 813 if (is_ptp_ring) 814 buff->len -= 815 aq_ptp_extract_ts(rx_ring->aq_nic, skb, 816 aq_buf_vaddr(&buff->rxdata), 817 buff->len); 818 819 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); 820 xdp_prepare_buff(&xdp, hard_start, rx_ring->page_offset, 821 buff->len, false); 822 if (!buff->is_eop) { 823 if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) { 824 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 825 ++rx_ring->stats.rx.packets; 826 rx_ring->stats.rx.bytes += xdp_get_buff_len(&xdp); 827 ++rx_ring->stats.rx.xdp_aborted; 828 u64_stats_update_end(&rx_ring->stats.rx.syncp); 829 continue; 830 } 831 } 832 833 skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff); 834 if (IS_ERR(skb) || !skb) 835 continue; 836 837 if (buff->is_vlan) 838 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 839 buff->vlan_rx_tag); 840 841 aq_rx_checksum(rx_ring, buff, skb); 842 843 skb_set_hash(skb, buff->rss_hash, 844 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 845 PKT_HASH_TYPE_NONE); 846 /* Send all PTP traffic to 0 queue */ 847 skb_record_rx_queue(skb, 848 is_ptp_ring ? 0 849 : AQ_NIC_RING2QMAP(rx_ring->aq_nic, 850 rx_ring->idx)); 851 852 napi_gro_receive(napi, skb); 853 } 854 855 err_exit: 856 return err; 857 } 858 859 int aq_ring_rx_clean(struct aq_ring_s *self, 860 struct napi_struct *napi, 861 int *work_done, 862 int budget) 863 { 864 if (static_branch_unlikely(&aq_xdp_locking_key)) 865 return __aq_ring_xdp_clean(self, napi, work_done, budget); 866 else 867 return __aq_ring_rx_clean(self, napi, work_done, budget); 868 } 869 870 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic) 871 { 872 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK) 873 while (self->sw_head != self->hw_head) { 874 u64 ns; 875 876 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw, 877 self->dx_ring + 878 (self->sw_head * self->dx_size), 879 self->dx_size, &ns); 880 aq_ptp_tx_hwtstamp(aq_nic, ns); 881 882 self->sw_head = aq_ring_next_dx(self, self->sw_head); 883 } 884 #endif 885 } 886 887 int aq_ring_rx_fill(struct aq_ring_s *self) 888 { 889 struct aq_ring_buff_s *buff = NULL; 890 int err = 0; 891 int i = 0; 892 893 if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES, 894 self->size / 2)) 895 return err; 896 897 for (i = aq_ring_avail_dx(self); i--; 898 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { 899 buff = &self->buff_ring[self->sw_tail]; 900 901 buff->flags = 0U; 902 buff->len = self->frame_max; 903 904 err = aq_get_rxpages(self, buff); 905 if (err) 906 goto err_exit; 907 908 buff->pa = aq_buf_daddr(&buff->rxdata); 909 buff = NULL; 910 } 911 912 err_exit: 913 return err; 914 } 915 916 void aq_ring_rx_deinit(struct aq_ring_s *self) 917 { 918 if (!self) 919 return; 920 921 for (; self->sw_head != self->sw_tail; 922 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 923 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 924 925 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic)); 926 } 927 } 928 929 void aq_ring_free(struct aq_ring_s *self) 930 { 931 if (!self) 932 return; 933 934 kfree(self->buff_ring); 935 936 if (self->dx_ring) 937 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 938 self->size * self->dx_size, self->dx_ring, 939 self->dx_ring_pa); 940 } 941 942 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) 943 { 944 unsigned int count; 945 unsigned int start; 946 947 if (self->ring_type == ATL_RING_RX) { 948 /* This data should mimic aq_ethtool_queue_rx_stat_names structure */ 949 do { 950 count = 0; 951 start = u64_stats_fetch_begin(&self->stats.rx.syncp); 952 data[count] = self->stats.rx.packets; 953 data[++count] = self->stats.rx.jumbo_packets; 954 data[++count] = self->stats.rx.lro_packets; 955 data[++count] = self->stats.rx.errors; 956 data[++count] = self->stats.rx.alloc_fails; 957 data[++count] = self->stats.rx.skb_alloc_fails; 958 data[++count] = self->stats.rx.polls; 959 data[++count] = self->stats.rx.pg_flips; 960 data[++count] = self->stats.rx.pg_reuses; 961 data[++count] = self->stats.rx.pg_losts; 962 data[++count] = self->stats.rx.xdp_aborted; 963 data[++count] = self->stats.rx.xdp_drop; 964 data[++count] = self->stats.rx.xdp_pass; 965 data[++count] = self->stats.rx.xdp_tx; 966 data[++count] = self->stats.rx.xdp_invalid; 967 data[++count] = self->stats.rx.xdp_redirect; 968 } while (u64_stats_fetch_retry(&self->stats.rx.syncp, start)); 969 } else { 970 /* This data should mimic aq_ethtool_queue_tx_stat_names structure */ 971 do { 972 count = 0; 973 start = u64_stats_fetch_begin(&self->stats.tx.syncp); 974 data[count] = self->stats.tx.packets; 975 data[++count] = self->stats.tx.queue_restarts; 976 } while (u64_stats_fetch_retry(&self->stats.tx.syncp, start)); 977 } 978 979 return ++count; 980 } 981