1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Atlantic Network Driver 3 * 4 * Copyright (C) 2014-2019 aQuantia Corporation 5 * Copyright (C) 2019-2020 Marvell International Ltd. 6 */ 7 8 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ 9 10 #include "aq_nic.h" 11 #include "aq_hw.h" 12 #include "aq_hw_utils.h" 13 #include "aq_ptp.h" 14 #include "aq_vec.h" 15 #include "aq_main.h" 16 17 #include <net/xdp.h> 18 #include <linux/filter.h> 19 #include <linux/bpf_trace.h> 20 #include <linux/netdevice.h> 21 #include <linux/etherdevice.h> 22 23 static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff, 24 struct xdp_buff *xdp) 25 { 26 struct skb_shared_info *sinfo; 27 int i; 28 29 if (xdp_buff_has_frags(xdp)) { 30 sinfo = xdp_get_shared_info_from_buff(xdp); 31 32 for (i = 0; i < sinfo->nr_frags; i++) { 33 skb_frag_t *frag = &sinfo->frags[i]; 34 35 page_ref_inc(skb_frag_page(frag)); 36 } 37 } 38 page_ref_inc(buff->rxdata.page); 39 } 40 41 static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev) 42 { 43 unsigned int len = PAGE_SIZE << rxpage->order; 44 45 dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE); 46 47 /* Drop the ref for being in the ring. */ 48 __free_pages(rxpage->page, rxpage->order); 49 rxpage->page = NULL; 50 } 51 52 static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring) 53 { 54 struct device *dev = aq_nic_get_dev(rx_ring->aq_nic); 55 unsigned int order = rx_ring->page_order; 56 struct page *page; 57 int ret = -ENOMEM; 58 dma_addr_t daddr; 59 60 page = dev_alloc_pages(order); 61 if (unlikely(!page)) 62 goto err_exit; 63 64 daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order, 65 DMA_FROM_DEVICE); 66 67 if (unlikely(dma_mapping_error(dev, daddr))) 68 goto free_page; 69 70 rxpage->page = page; 71 rxpage->daddr = daddr; 72 rxpage->order = order; 73 rxpage->pg_off = rx_ring->page_offset; 74 75 return 0; 76 77 free_page: 78 __free_pages(page, order); 79 80 err_exit: 81 return ret; 82 } 83 84 static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf) 85 { 86 unsigned int order = self->page_order; 87 u16 page_offset = self->page_offset; 88 u16 frame_max = self->frame_max; 89 u16 tail_size = self->tail_size; 90 int ret; 91 92 if (rxbuf->rxdata.page) { 93 /* One means ring is the only user and can reuse */ 94 if (page_ref_count(rxbuf->rxdata.page) > 1) { 95 /* Try reuse buffer */ 96 rxbuf->rxdata.pg_off += frame_max + page_offset + 97 tail_size; 98 if (rxbuf->rxdata.pg_off + frame_max + tail_size <= 99 (PAGE_SIZE << order)) { 100 u64_stats_update_begin(&self->stats.rx.syncp); 101 self->stats.rx.pg_flips++; 102 u64_stats_update_end(&self->stats.rx.syncp); 103 104 } else { 105 /* Buffer exhausted. We have other users and 106 * should release this page and realloc 107 */ 108 aq_free_rxpage(&rxbuf->rxdata, 109 aq_nic_get_dev(self->aq_nic)); 110 u64_stats_update_begin(&self->stats.rx.syncp); 111 self->stats.rx.pg_losts++; 112 u64_stats_update_end(&self->stats.rx.syncp); 113 } 114 } else { 115 rxbuf->rxdata.pg_off = page_offset; 116 u64_stats_update_begin(&self->stats.rx.syncp); 117 self->stats.rx.pg_reuses++; 118 u64_stats_update_end(&self->stats.rx.syncp); 119 } 120 } 121 122 if (!rxbuf->rxdata.page) { 123 ret = aq_alloc_rxpages(&rxbuf->rxdata, self); 124 if (ret) { 125 u64_stats_update_begin(&self->stats.rx.syncp); 126 self->stats.rx.alloc_fails++; 127 u64_stats_update_end(&self->stats.rx.syncp); 128 } 129 return ret; 130 } 131 132 return 0; 133 } 134 135 static int aq_ring_alloc(struct aq_ring_s *self, 136 struct aq_nic_s *aq_nic) 137 { 138 int err = 0; 139 140 self->buff_ring = 141 kzalloc_objs(struct aq_ring_buff_s, self->size); 142 143 if (!self->buff_ring) { 144 err = -ENOMEM; 145 goto err_exit; 146 } 147 148 self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), 149 self->size * self->dx_size, 150 &self->dx_ring_pa, GFP_KERNEL); 151 if (!self->dx_ring) { 152 err = -ENOMEM; 153 goto err_exit; 154 } 155 156 err_exit: 157 if (err < 0) { 158 aq_ring_free(self); 159 } 160 161 return err; 162 } 163 164 int aq_ring_tx_alloc(struct aq_ring_s *self, 165 struct aq_nic_s *aq_nic, 166 unsigned int idx, 167 struct aq_nic_cfg_s *aq_nic_cfg) 168 { 169 self->aq_nic = aq_nic; 170 self->idx = idx; 171 self->size = aq_nic_cfg->txds; 172 self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; 173 174 return aq_ring_alloc(self, aq_nic); 175 } 176 177 int aq_ring_rx_alloc(struct aq_ring_s *self, 178 struct aq_nic_s *aq_nic, 179 unsigned int idx, 180 struct aq_nic_cfg_s *aq_nic_cfg) 181 { 182 self->aq_nic = aq_nic; 183 self->idx = idx; 184 self->size = aq_nic_cfg->rxds; 185 self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; 186 self->xdp_prog = aq_nic->xdp_prog; 187 self->frame_max = AQ_CFG_RX_FRAME_MAX; 188 189 /* Only order-2 is allowed if XDP is enabled */ 190 if (READ_ONCE(self->xdp_prog)) { 191 self->page_offset = AQ_XDP_HEADROOM; 192 self->page_order = AQ_CFG_XDP_PAGEORDER; 193 self->tail_size = AQ_XDP_TAILROOM; 194 } else { 195 self->page_offset = 0; 196 self->page_order = fls(self->frame_max / PAGE_SIZE + 197 (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1; 198 if (aq_nic_cfg->rxpageorder > self->page_order) 199 self->page_order = aq_nic_cfg->rxpageorder; 200 self->tail_size = 0; 201 } 202 203 return aq_ring_alloc(self, aq_nic); 204 } 205 206 int 207 aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic, 208 unsigned int idx, unsigned int size, unsigned int dx_size) 209 { 210 struct device *dev = aq_nic_get_dev(aq_nic); 211 size_t sz = size * dx_size + AQ_CFG_RXDS_DEF; 212 213 memset(self, 0, sizeof(*self)); 214 215 self->aq_nic = aq_nic; 216 self->idx = idx; 217 self->size = size; 218 self->dx_size = dx_size; 219 220 self->dx_ring = dma_alloc_coherent(dev, sz, &self->dx_ring_pa, 221 GFP_KERNEL); 222 if (!self->dx_ring) { 223 aq_ring_free(self); 224 return -ENOMEM; 225 } 226 227 return 0; 228 } 229 230 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type) 231 { 232 self->hw_head = 0; 233 self->sw_head = 0; 234 self->sw_tail = 0; 235 self->ring_type = ring_type; 236 237 if (self->ring_type == ATL_RING_RX) 238 u64_stats_init(&self->stats.rx.syncp); 239 else 240 u64_stats_init(&self->stats.tx.syncp); 241 242 return 0; 243 } 244 245 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, 246 unsigned int t) 247 { 248 return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); 249 } 250 251 void aq_ring_update_queue_state(struct aq_ring_s *ring) 252 { 253 if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX) 254 aq_ring_queue_stop(ring); 255 else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES) 256 aq_ring_queue_wake(ring); 257 } 258 259 void aq_ring_queue_wake(struct aq_ring_s *ring) 260 { 261 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 262 263 if (__netif_subqueue_stopped(ndev, 264 AQ_NIC_RING2QMAP(ring->aq_nic, 265 ring->idx))) { 266 netif_wake_subqueue(ndev, 267 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 268 u64_stats_update_begin(&ring->stats.tx.syncp); 269 ring->stats.tx.queue_restarts++; 270 u64_stats_update_end(&ring->stats.tx.syncp); 271 } 272 } 273 274 void aq_ring_queue_stop(struct aq_ring_s *ring) 275 { 276 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 277 278 if (!__netif_subqueue_stopped(ndev, 279 AQ_NIC_RING2QMAP(ring->aq_nic, 280 ring->idx))) 281 netif_stop_subqueue(ndev, 282 AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); 283 } 284 285 bool aq_ring_tx_clean(struct aq_ring_s *self) 286 { 287 struct device *dev = aq_nic_get_dev(self->aq_nic); 288 unsigned int budget; 289 290 for (budget = AQ_CFG_TX_CLEAN_BUDGET; 291 budget && self->sw_head != self->hw_head; budget--) { 292 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 293 294 if (likely(buff->is_mapped)) { 295 if (unlikely(buff->is_sop)) { 296 if (!buff->is_eop && 297 buff->eop_index != 0xffffU && 298 (!aq_ring_dx_in_range(self->sw_head, 299 buff->eop_index, 300 self->hw_head))) 301 break; 302 303 dma_unmap_single(dev, buff->pa, buff->len, 304 DMA_TO_DEVICE); 305 } else { 306 dma_unmap_page(dev, buff->pa, buff->len, 307 DMA_TO_DEVICE); 308 } 309 } 310 311 if (likely(!buff->is_eop)) 312 goto out; 313 314 if (unlikely(buff->request_ts) && 315 self->aq_nic->aq_hw_ops->hw_ring_tx_ptp_get_ts) { 316 u64 ts = self->aq_nic->aq_hw_ops->hw_ring_tx_ptp_get_ts(self); 317 318 if (!ts) { 319 if (time_after(jiffies, 320 self->ptp_ts_deadline)) { 321 /* Timeout: drain skb_ring head to 322 * keep in sync with buff_ring 323 */ 324 aq_ptp_tx_skb_drop_head(self->aq_nic); 325 buff->request_ts = 0; 326 dev_kfree_skb_any(buff->skb); 327 buff->skb = NULL; 328 goto out; 329 } else { 330 buff->is_mapped = 0; 331 buff->pa = 0U; 332 break; 333 } 334 } 335 336 aq_ptp_tx_hwtstamp(self->aq_nic, ts); 337 } 338 if (buff->skb) { 339 u64_stats_update_begin(&self->stats.tx.syncp); 340 ++self->stats.tx.packets; 341 self->stats.tx.bytes += buff->skb->len; 342 u64_stats_update_end(&self->stats.tx.syncp); 343 dev_kfree_skb_any(buff->skb); 344 } else if (buff->xdpf) { 345 u64_stats_update_begin(&self->stats.tx.syncp); 346 ++self->stats.tx.packets; 347 self->stats.tx.bytes += xdp_get_frame_len(buff->xdpf); 348 u64_stats_update_end(&self->stats.tx.syncp); 349 xdp_return_frame_rx_napi(buff->xdpf); 350 } 351 352 out: 353 buff->skb = NULL; 354 buff->xdpf = NULL; 355 buff->pa = 0U; 356 buff->eop_index = 0xffffU; 357 self->sw_head = aq_ring_next_dx(self, self->sw_head); 358 } 359 360 return !!budget; 361 } 362 363 static void aq_rx_checksum(struct aq_ring_s *self, 364 struct aq_ring_buff_s *buff, 365 struct sk_buff *skb) 366 { 367 if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM)) 368 return; 369 370 if (unlikely(buff->is_cso_err)) { 371 u64_stats_update_begin(&self->stats.rx.syncp); 372 ++self->stats.rx.errors; 373 u64_stats_update_end(&self->stats.rx.syncp); 374 skb->ip_summed = CHECKSUM_NONE; 375 return; 376 } 377 if (buff->is_ip_cso) { 378 __skb_incr_checksum_unnecessary(skb); 379 } else { 380 skb->ip_summed = CHECKSUM_NONE; 381 } 382 383 if (buff->is_udp_cso || buff->is_tcp_cso) 384 __skb_incr_checksum_unnecessary(skb); 385 } 386 387 int aq_xdp_xmit(struct net_device *dev, int num_frames, 388 struct xdp_frame **frames, u32 flags) 389 { 390 struct aq_nic_s *aq_nic = netdev_priv(dev); 391 unsigned int vec, i, drop = 0; 392 int cpu = smp_processor_id(); 393 struct aq_nic_cfg_s *aq_cfg; 394 struct aq_ring_s *ring; 395 396 aq_cfg = aq_nic_get_cfg(aq_nic); 397 vec = cpu % aq_cfg->vecs; 398 ring = aq_nic->aq_ring_tx[AQ_NIC_CFG_TCVEC2RING(aq_cfg, 0, vec)]; 399 400 for (i = 0; i < num_frames; i++) { 401 struct xdp_frame *xdpf = frames[i]; 402 403 if (aq_nic_xmit_xdpf(aq_nic, ring, xdpf) == NETDEV_TX_BUSY) 404 drop++; 405 } 406 407 return num_frames - drop; 408 } 409 410 static struct sk_buff *aq_xdp_build_skb(struct xdp_buff *xdp, 411 struct net_device *dev, 412 struct aq_ring_buff_s *buff) 413 { 414 struct xdp_frame *xdpf; 415 struct sk_buff *skb; 416 417 xdpf = xdp_convert_buff_to_frame(xdp); 418 if (unlikely(!xdpf)) 419 return NULL; 420 421 skb = xdp_build_skb_from_frame(xdpf, dev); 422 if (!skb) 423 return NULL; 424 425 aq_get_rxpages_xdp(buff, xdp); 426 return skb; 427 } 428 429 static struct sk_buff *aq_xdp_run_prog(struct aq_nic_s *aq_nic, 430 struct xdp_buff *xdp, 431 struct aq_ring_s *rx_ring, 432 struct aq_ring_buff_s *buff) 433 { 434 int result = NETDEV_TX_BUSY; 435 struct aq_ring_s *tx_ring; 436 struct xdp_frame *xdpf; 437 struct bpf_prog *prog; 438 u32 act = XDP_ABORTED; 439 struct sk_buff *skb; 440 441 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 442 ++rx_ring->stats.rx.packets; 443 rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp); 444 u64_stats_update_end(&rx_ring->stats.rx.syncp); 445 446 prog = READ_ONCE(rx_ring->xdp_prog); 447 if (!prog) 448 return aq_xdp_build_skb(xdp, aq_nic->ndev, buff); 449 450 prefetchw(xdp->data_hard_start); /* xdp_frame write */ 451 452 /* single buffer XDP program, but packet is multi buffer, aborted */ 453 if (xdp_buff_has_frags(xdp) && !prog->aux->xdp_has_frags) 454 goto out_aborted; 455 456 act = bpf_prog_run_xdp(prog, xdp); 457 switch (act) { 458 case XDP_PASS: 459 skb = aq_xdp_build_skb(xdp, aq_nic->ndev, buff); 460 if (!skb) 461 goto out_aborted; 462 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 463 ++rx_ring->stats.rx.xdp_pass; 464 u64_stats_update_end(&rx_ring->stats.rx.syncp); 465 return skb; 466 case XDP_TX: 467 xdpf = xdp_convert_buff_to_frame(xdp); 468 if (unlikely(!xdpf)) 469 goto out_aborted; 470 tx_ring = aq_nic->aq_ring_tx[rx_ring->idx]; 471 result = aq_nic_xmit_xdpf(aq_nic, tx_ring, xdpf); 472 if (result == NETDEV_TX_BUSY) 473 goto out_aborted; 474 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 475 ++rx_ring->stats.rx.xdp_tx; 476 u64_stats_update_end(&rx_ring->stats.rx.syncp); 477 aq_get_rxpages_xdp(buff, xdp); 478 break; 479 case XDP_REDIRECT: 480 if (xdp_do_redirect(aq_nic->ndev, xdp, prog) < 0) 481 goto out_aborted; 482 xdp_do_flush(); 483 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 484 ++rx_ring->stats.rx.xdp_redirect; 485 u64_stats_update_end(&rx_ring->stats.rx.syncp); 486 aq_get_rxpages_xdp(buff, xdp); 487 break; 488 default: 489 fallthrough; 490 case XDP_ABORTED: 491 out_aborted: 492 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 493 ++rx_ring->stats.rx.xdp_aborted; 494 u64_stats_update_end(&rx_ring->stats.rx.syncp); 495 trace_xdp_exception(aq_nic->ndev, prog, act); 496 bpf_warn_invalid_xdp_action(aq_nic->ndev, prog, act); 497 break; 498 case XDP_DROP: 499 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 500 ++rx_ring->stats.rx.xdp_drop; 501 u64_stats_update_end(&rx_ring->stats.rx.syncp); 502 break; 503 } 504 505 return ERR_PTR(-result); 506 } 507 508 static bool aq_add_rx_fragment(struct device *dev, 509 struct aq_ring_s *ring, 510 struct aq_ring_buff_s *buff, 511 struct xdp_buff *xdp) 512 { 513 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 514 struct aq_ring_buff_s *buff_ = buff; 515 516 memset(sinfo, 0, sizeof(*sinfo)); 517 do { 518 skb_frag_t *frag; 519 520 if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) 521 return true; 522 523 frag = &sinfo->frags[sinfo->nr_frags++]; 524 buff_ = &ring->buff_ring[buff_->next]; 525 dma_sync_single_range_for_cpu(dev, 526 buff_->rxdata.daddr, 527 buff_->rxdata.pg_off, 528 buff_->len, 529 DMA_FROM_DEVICE); 530 sinfo->xdp_frags_size += buff_->len; 531 skb_frag_fill_page_desc(frag, buff_->rxdata.page, 532 buff_->rxdata.pg_off, 533 buff_->len); 534 535 buff_->is_cleaned = 1; 536 537 buff->is_ip_cso &= buff_->is_ip_cso; 538 buff->is_udp_cso &= buff_->is_udp_cso; 539 buff->is_tcp_cso &= buff_->is_tcp_cso; 540 buff->is_cso_err |= buff_->is_cso_err; 541 542 if (page_is_pfmemalloc(buff_->rxdata.page)) 543 xdp_buff_set_frag_pfmemalloc(xdp); 544 545 } while (!buff_->is_eop); 546 547 xdp_buff_set_frags_flag(xdp); 548 549 return false; 550 } 551 552 static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi, 553 int *work_done, int budget) 554 { 555 struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); 556 int err = 0; 557 558 for (; (self->sw_head != self->hw_head) && budget; 559 self->sw_head = aq_ring_next_dx(self, self->sw_head), 560 --budget, ++(*work_done)) { 561 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 562 bool is_ptp_ring = aq_ptp_ring(self->aq_nic, self); 563 struct aq_ring_buff_s *buff_ = NULL; 564 struct sk_buff *skb = NULL; 565 unsigned int next_ = 0U; 566 unsigned int i = 0U; 567 u16 hdr_len; 568 569 if (buff->is_cleaned) 570 continue; 571 572 if (!buff->is_eop) { 573 unsigned int frag_cnt = 0U; 574 575 /* There will be an extra fragment */ 576 if (buff->len > AQ_CFG_RX_HDR_SIZE) 577 frag_cnt++; 578 579 buff_ = buff; 580 do { 581 bool is_rsc_completed = true; 582 583 if (buff_->next >= self->size) { 584 err = -EIO; 585 goto err_exit; 586 } 587 588 frag_cnt++; 589 next_ = buff_->next; 590 buff_ = &self->buff_ring[next_]; 591 is_rsc_completed = 592 aq_ring_dx_in_range(self->sw_head, 593 next_, 594 self->hw_head); 595 596 if (unlikely(!is_rsc_completed) || 597 frag_cnt > MAX_SKB_FRAGS) { 598 err = 0; 599 goto err_exit; 600 } 601 602 buff->is_error |= buff_->is_error; 603 buff->is_cso_err |= buff_->is_cso_err; 604 605 } while (!buff_->is_eop); 606 607 if (buff->is_error || 608 (buff->is_lro && buff->is_cso_err)) { 609 buff_ = buff; 610 do { 611 if (buff_->next >= self->size) { 612 err = -EIO; 613 goto err_exit; 614 } 615 next_ = buff_->next; 616 buff_ = &self->buff_ring[next_]; 617 618 buff_->is_cleaned = true; 619 } while (!buff_->is_eop); 620 621 u64_stats_update_begin(&self->stats.rx.syncp); 622 ++self->stats.rx.errors; 623 u64_stats_update_end(&self->stats.rx.syncp); 624 continue; 625 } 626 } 627 628 if (buff->is_error) { 629 u64_stats_update_begin(&self->stats.rx.syncp); 630 ++self->stats.rx.errors; 631 u64_stats_update_end(&self->stats.rx.syncp); 632 continue; 633 } 634 635 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 636 buff->rxdata.daddr, 637 buff->rxdata.pg_off, 638 buff->len, DMA_FROM_DEVICE); 639 640 skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE); 641 if (unlikely(!skb)) { 642 u64_stats_update_begin(&self->stats.rx.syncp); 643 self->stats.rx.skb_alloc_fails++; 644 u64_stats_update_end(&self->stats.rx.syncp); 645 err = -ENOMEM; 646 goto err_exit; 647 } 648 if (is_ptp_ring) 649 buff->len -= 650 aq_ptp_extract_ts(self->aq_nic, skb_hwtstamps(skb), 651 aq_buf_vaddr(&buff->rxdata), 652 buff->len); 653 654 hdr_len = buff->len; 655 if (hdr_len > AQ_CFG_RX_HDR_SIZE) 656 hdr_len = eth_get_headlen(skb->dev, 657 aq_buf_vaddr(&buff->rxdata), 658 AQ_CFG_RX_HDR_SIZE); 659 660 memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata), 661 ALIGN(hdr_len, sizeof(long))); 662 663 if (buff->len - hdr_len > 0) { 664 skb_add_rx_frag(skb, i++, buff->rxdata.page, 665 buff->rxdata.pg_off + hdr_len, 666 buff->len - hdr_len, 667 self->frame_max); 668 page_ref_inc(buff->rxdata.page); 669 } 670 671 if (!buff->is_eop) { 672 buff_ = buff; 673 do { 674 next_ = buff_->next; 675 buff_ = &self->buff_ring[next_]; 676 677 dma_sync_single_range_for_cpu(aq_nic_get_dev(self->aq_nic), 678 buff_->rxdata.daddr, 679 buff_->rxdata.pg_off, 680 buff_->len, 681 DMA_FROM_DEVICE); 682 skb_add_rx_frag(skb, i++, 683 buff_->rxdata.page, 684 buff_->rxdata.pg_off, 685 buff_->len, 686 self->frame_max); 687 page_ref_inc(buff_->rxdata.page); 688 buff_->is_cleaned = 1; 689 690 buff->is_ip_cso &= buff_->is_ip_cso; 691 buff->is_udp_cso &= buff_->is_udp_cso; 692 buff->is_tcp_cso &= buff_->is_tcp_cso; 693 buff->is_cso_err |= buff_->is_cso_err; 694 695 } while (!buff_->is_eop); 696 } 697 698 if (buff->is_vlan) 699 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 700 buff->vlan_rx_tag); 701 702 skb->protocol = eth_type_trans(skb, ndev); 703 704 aq_rx_checksum(self, buff, skb); 705 706 skb_set_hash(skb, buff->rss_hash, 707 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 708 PKT_HASH_TYPE_NONE); 709 /* Send all PTP traffic to 0 queue */ 710 skb_record_rx_queue(skb, 711 is_ptp_ring ? 0 712 : AQ_NIC_RING2QMAP(self->aq_nic, 713 self->idx)); 714 715 u64_stats_update_begin(&self->stats.rx.syncp); 716 ++self->stats.rx.packets; 717 self->stats.rx.bytes += skb->len; 718 u64_stats_update_end(&self->stats.rx.syncp); 719 720 napi_gro_receive(napi, skb); 721 } 722 723 err_exit: 724 return err; 725 } 726 727 static int __aq_ring_xdp_clean(struct aq_ring_s *rx_ring, 728 struct napi_struct *napi, int *work_done, 729 int budget) 730 { 731 int frame_sz = rx_ring->page_offset + rx_ring->frame_max + 732 rx_ring->tail_size; 733 struct aq_nic_s *aq_nic = rx_ring->aq_nic; 734 bool is_rsc_completed = true; 735 struct device *dev; 736 int err = 0; 737 738 dev = aq_nic_get_dev(aq_nic); 739 for (; (rx_ring->sw_head != rx_ring->hw_head) && budget; 740 rx_ring->sw_head = aq_ring_next_dx(rx_ring, rx_ring->sw_head), 741 --budget, ++(*work_done)) { 742 struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head]; 743 bool is_ptp_ring = aq_ptp_ring(rx_ring->aq_nic, rx_ring); 744 struct aq_ring_buff_s *buff_ = NULL; 745 u16 ptp_hwtstamp_len = 0; 746 struct skb_shared_hwtstamps shhwtstamps; 747 struct sk_buff *skb = NULL; 748 unsigned int next_ = 0U; 749 struct xdp_buff xdp; 750 void *hard_start; 751 752 if (buff->is_cleaned) 753 continue; 754 755 if (!buff->is_eop) { 756 buff_ = buff; 757 do { 758 if (buff_->next >= rx_ring->size) { 759 err = -EIO; 760 goto err_exit; 761 } 762 next_ = buff_->next; 763 buff_ = &rx_ring->buff_ring[next_]; 764 is_rsc_completed = 765 aq_ring_dx_in_range(rx_ring->sw_head, 766 next_, 767 rx_ring->hw_head); 768 769 if (unlikely(!is_rsc_completed)) 770 break; 771 772 buff->is_error |= buff_->is_error; 773 buff->is_cso_err |= buff_->is_cso_err; 774 } while (!buff_->is_eop); 775 776 if (!is_rsc_completed) { 777 err = 0; 778 goto err_exit; 779 } 780 if (buff->is_error || 781 (buff->is_lro && buff->is_cso_err)) { 782 buff_ = buff; 783 do { 784 if (buff_->next >= rx_ring->size) { 785 err = -EIO; 786 goto err_exit; 787 } 788 next_ = buff_->next; 789 buff_ = &rx_ring->buff_ring[next_]; 790 791 buff_->is_cleaned = true; 792 } while (!buff_->is_eop); 793 794 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 795 ++rx_ring->stats.rx.errors; 796 u64_stats_update_end(&rx_ring->stats.rx.syncp); 797 continue; 798 } 799 } 800 801 if (buff->is_error) { 802 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 803 ++rx_ring->stats.rx.errors; 804 u64_stats_update_end(&rx_ring->stats.rx.syncp); 805 continue; 806 } 807 808 dma_sync_single_range_for_cpu(dev, 809 buff->rxdata.daddr, 810 buff->rxdata.pg_off, 811 buff->len, DMA_FROM_DEVICE); 812 hard_start = page_address(buff->rxdata.page) + 813 buff->rxdata.pg_off - rx_ring->page_offset; 814 815 if (is_ptp_ring) { 816 ptp_hwtstamp_len = aq_ptp_extract_ts(rx_ring->aq_nic, &shhwtstamps, 817 aq_buf_vaddr(&buff->rxdata), 818 buff->len); 819 buff->len -= ptp_hwtstamp_len; 820 } 821 822 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); 823 xdp_prepare_buff(&xdp, hard_start, rx_ring->page_offset, 824 buff->len, false); 825 if (!buff->is_eop) { 826 if (aq_add_rx_fragment(dev, rx_ring, buff, &xdp)) { 827 u64_stats_update_begin(&rx_ring->stats.rx.syncp); 828 ++rx_ring->stats.rx.packets; 829 rx_ring->stats.rx.bytes += xdp_get_buff_len(&xdp); 830 ++rx_ring->stats.rx.xdp_aborted; 831 u64_stats_update_end(&rx_ring->stats.rx.syncp); 832 continue; 833 } 834 } 835 836 skb = aq_xdp_run_prog(aq_nic, &xdp, rx_ring, buff); 837 if (IS_ERR(skb) || !skb) 838 continue; 839 840 if (ptp_hwtstamp_len > 0) 841 *skb_hwtstamps(skb) = shhwtstamps; 842 843 if (buff->is_vlan) 844 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 845 buff->vlan_rx_tag); 846 847 aq_rx_checksum(rx_ring, buff, skb); 848 849 skb_set_hash(skb, buff->rss_hash, 850 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 851 PKT_HASH_TYPE_NONE); 852 /* Send all PTP traffic to 0 queue */ 853 skb_record_rx_queue(skb, 854 is_ptp_ring ? 0 855 : AQ_NIC_RING2QMAP(rx_ring->aq_nic, 856 rx_ring->idx)); 857 858 napi_gro_receive(napi, skb); 859 } 860 861 err_exit: 862 return err; 863 } 864 865 int aq_ring_rx_clean(struct aq_ring_s *self, 866 struct napi_struct *napi, 867 int *work_done, 868 int budget) 869 { 870 if (static_branch_unlikely(&aq_xdp_locking_key)) 871 return __aq_ring_xdp_clean(self, napi, work_done, budget); 872 else 873 return __aq_ring_rx_clean(self, napi, work_done, budget); 874 } 875 876 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic) 877 { 878 #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK) 879 while (self->sw_head != self->hw_head) { 880 u64 ns; 881 882 aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw, 883 self->dx_ring + 884 (self->sw_head * self->dx_size), 885 self->dx_size, &ns); 886 aq_ptp_tx_hwtstamp(aq_nic, ns); 887 888 self->sw_head = aq_ring_next_dx(self, self->sw_head); 889 } 890 #endif 891 } 892 893 int aq_ring_rx_fill(struct aq_ring_s *self) 894 { 895 struct aq_ring_buff_s *buff = NULL; 896 int err = 0; 897 int i = 0; 898 899 if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES, 900 self->size / 2)) 901 return err; 902 903 for (i = aq_ring_avail_dx(self); i--; 904 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { 905 buff = &self->buff_ring[self->sw_tail]; 906 907 buff->flags = 0U; 908 buff->len = self->frame_max; 909 910 err = aq_get_rxpages(self, buff); 911 if (err) 912 goto err_exit; 913 914 buff->pa = aq_buf_daddr(&buff->rxdata); 915 buff = NULL; 916 } 917 918 err_exit: 919 return err; 920 } 921 922 void aq_ring_rx_deinit(struct aq_ring_s *self) 923 { 924 if (!self) 925 return; 926 927 for (; self->sw_head != self->sw_tail; 928 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 929 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 930 931 aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic)); 932 } 933 } 934 935 void aq_ring_free(struct aq_ring_s *self) 936 { 937 if (!self) 938 return; 939 940 kfree(self->buff_ring); 941 self->buff_ring = NULL; 942 943 if (self->dx_ring) { 944 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 945 self->size * self->dx_size, self->dx_ring, 946 self->dx_ring_pa); 947 self->dx_ring = NULL; 948 } 949 } 950 951 void aq_ring_hwts_rx_free(struct aq_ring_s *self) 952 { 953 if (!self) 954 return; 955 956 if (self->dx_ring) { 957 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 958 self->size * self->dx_size + AQ_CFG_RXDS_DEF, 959 self->dx_ring, self->dx_ring_pa); 960 self->dx_ring = NULL; 961 } 962 } 963 964 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) 965 { 966 unsigned int count; 967 unsigned int start; 968 969 if (self->ring_type == ATL_RING_RX) { 970 /* This data should mimic aq_ethtool_queue_rx_stat_names structure */ 971 do { 972 count = 0; 973 start = u64_stats_fetch_begin(&self->stats.rx.syncp); 974 data[count] = self->stats.rx.packets; 975 data[++count] = self->stats.rx.jumbo_packets; 976 data[++count] = self->stats.rx.lro_packets; 977 data[++count] = self->stats.rx.errors; 978 data[++count] = self->stats.rx.alloc_fails; 979 data[++count] = self->stats.rx.skb_alloc_fails; 980 data[++count] = self->stats.rx.polls; 981 data[++count] = self->stats.rx.pg_flips; 982 data[++count] = self->stats.rx.pg_reuses; 983 data[++count] = self->stats.rx.pg_losts; 984 data[++count] = self->stats.rx.xdp_aborted; 985 data[++count] = self->stats.rx.xdp_drop; 986 data[++count] = self->stats.rx.xdp_pass; 987 data[++count] = self->stats.rx.xdp_tx; 988 data[++count] = self->stats.rx.xdp_invalid; 989 data[++count] = self->stats.rx.xdp_redirect; 990 } while (u64_stats_fetch_retry(&self->stats.rx.syncp, start)); 991 } else { 992 /* This data should mimic aq_ethtool_queue_tx_stat_names structure */ 993 do { 994 count = 0; 995 start = u64_stats_fetch_begin(&self->stats.tx.syncp); 996 data[count] = self->stats.tx.packets; 997 data[++count] = self->stats.tx.queue_restarts; 998 } while (u64_stats_fetch_retry(&self->stats.tx.syncp, start)); 999 } 1000 1001 return ++count; 1002 } 1003