1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/bpf.h> 6 #include <linux/bpf_trace.h> 7 #include <linux/iopoll.h> 8 #include <linux/pci.h> 9 #include <net/netdev_queues.h> 10 #include <net/page_pool/helpers.h> 11 #include <net/tcp.h> 12 #include <net/xdp.h> 13 14 #include "fbnic.h" 15 #include "fbnic_csr.h" 16 #include "fbnic_netdev.h" 17 #include "fbnic_txrx.h" 18 19 enum { 20 FBNIC_XDP_PASS = 0, 21 FBNIC_XDP_CONSUME, 22 FBNIC_XDP_TX, 23 FBNIC_XDP_LEN_ERR, 24 }; 25 26 enum { 27 FBNIC_XMIT_CB_TS = 0x01, 28 }; 29 30 struct fbnic_xmit_cb { 31 u32 bytecount; 32 u16 gso_segs; 33 u8 desc_count; 34 u8 flags; 35 int hw_head; 36 }; 37 38 #define FBNIC_XMIT_CB(__skb) ((struct fbnic_xmit_cb *)((__skb)->cb)) 39 40 #define FBNIC_XMIT_NOUNMAP ((void *)1) 41 42 u32 __iomem *fbnic_ring_csr_base(const struct fbnic_ring *ring) 43 { 44 unsigned long csr_base = (unsigned long)ring->doorbell; 45 46 csr_base &= ~(FBNIC_QUEUE_STRIDE * sizeof(u32) - 1); 47 48 return (u32 __iomem *)csr_base; 49 } 50 51 static u32 fbnic_ring_rd32(struct fbnic_ring *ring, unsigned int csr) 52 { 53 u32 __iomem *csr_base = fbnic_ring_csr_base(ring); 54 55 return readl(csr_base + csr); 56 } 57 58 static void fbnic_ring_wr32(struct fbnic_ring *ring, unsigned int csr, u32 val) 59 { 60 u32 __iomem *csr_base = fbnic_ring_csr_base(ring); 61 62 writel(val, csr_base + csr); 63 } 64 65 /** 66 * fbnic_ts40_to_ns() - convert descriptor timestamp to PHC time 67 * @fbn: netdev priv of the FB NIC 68 * @ts40: timestamp read from a descriptor 69 * 70 * Return: u64 value of PHC time in nanoseconds 71 * 72 * Convert truncated 40 bit device timestamp as read from a descriptor 73 * to the full PHC time in nanoseconds. 74 */ 75 static __maybe_unused u64 fbnic_ts40_to_ns(struct fbnic_net *fbn, u64 ts40) 76 { 77 unsigned int s; 78 u64 time_ns; 79 s64 offset; 80 u8 ts_top; 81 u32 high; 82 83 do { 84 s = u64_stats_fetch_begin(&fbn->time_seq); 85 offset = READ_ONCE(fbn->time_offset); 86 } while (u64_stats_fetch_retry(&fbn->time_seq, s)); 87 88 high = READ_ONCE(fbn->time_high); 89 90 /* Bits 63..40 from periodic clock reads, 39..0 from ts40 */ 91 time_ns = (u64)(high >> 8) << 40 | ts40; 92 93 /* Compare bits 32-39 between periodic reads and ts40, 94 * see if HW clock may have wrapped since last read. We are sure 95 * that periodic reads are always at least ~1 minute behind, so 96 * this logic works perfectly fine. 97 */ 98 ts_top = ts40 >> 32; 99 if (ts_top < (u8)high && (u8)high - ts_top > U8_MAX / 2) 100 time_ns += 1ULL << 40; 101 102 return time_ns + offset; 103 } 104 105 static unsigned int fbnic_desc_unused(struct fbnic_ring *ring) 106 { 107 return (ring->head - ring->tail - 1) & ring->size_mask; 108 } 109 110 static unsigned int fbnic_desc_used(struct fbnic_ring *ring) 111 { 112 return (ring->tail - ring->head) & ring->size_mask; 113 } 114 115 static struct netdev_queue *txring_txq(const struct net_device *dev, 116 const struct fbnic_ring *ring) 117 { 118 return netdev_get_tx_queue(dev, ring->q_idx); 119 } 120 121 static int fbnic_maybe_stop_tx(const struct net_device *dev, 122 struct fbnic_ring *ring, 123 const unsigned int size) 124 { 125 struct netdev_queue *txq = txring_txq(dev, ring); 126 int res; 127 128 res = netif_txq_maybe_stop(txq, fbnic_desc_unused(ring), size, 129 FBNIC_TX_DESC_WAKEUP); 130 if (!res) { 131 u64_stats_update_begin(&ring->stats.syncp); 132 ring->stats.twq.stop++; 133 u64_stats_update_end(&ring->stats.syncp); 134 } 135 136 return !res; 137 } 138 139 static bool fbnic_tx_sent_queue(struct sk_buff *skb, struct fbnic_ring *ring) 140 { 141 struct netdev_queue *dev_queue = txring_txq(skb->dev, ring); 142 unsigned int bytecount = FBNIC_XMIT_CB(skb)->bytecount; 143 bool xmit_more = netdev_xmit_more(); 144 145 /* TBD: Request completion more often if xmit_more becomes large */ 146 147 return __netdev_tx_sent_queue(dev_queue, bytecount, xmit_more); 148 } 149 150 static void fbnic_unmap_single_twd(struct device *dev, __le64 *twd) 151 { 152 u64 raw_twd = le64_to_cpu(*twd); 153 unsigned int len; 154 dma_addr_t dma; 155 156 dma = FIELD_GET(FBNIC_TWD_ADDR_MASK, raw_twd); 157 len = FIELD_GET(FBNIC_TWD_LEN_MASK, raw_twd); 158 159 dma_unmap_single(dev, dma, len, DMA_TO_DEVICE); 160 } 161 162 static void fbnic_unmap_page_twd(struct device *dev, __le64 *twd) 163 { 164 u64 raw_twd = le64_to_cpu(*twd); 165 unsigned int len; 166 dma_addr_t dma; 167 168 dma = FIELD_GET(FBNIC_TWD_ADDR_MASK, raw_twd); 169 len = FIELD_GET(FBNIC_TWD_LEN_MASK, raw_twd); 170 171 dma_unmap_page(dev, dma, len, DMA_TO_DEVICE); 172 } 173 174 #define FBNIC_TWD_TYPE(_type) \ 175 cpu_to_le64(FIELD_PREP(FBNIC_TWD_TYPE_MASK, FBNIC_TWD_TYPE_##_type)) 176 177 static bool fbnic_tx_tstamp(struct sk_buff *skb) 178 { 179 struct fbnic_net *fbn; 180 181 if (!unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 182 return false; 183 184 fbn = netdev_priv(skb->dev); 185 if (fbn->hwtstamp_config.tx_type == HWTSTAMP_TX_OFF) 186 return false; 187 188 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 189 FBNIC_XMIT_CB(skb)->flags |= FBNIC_XMIT_CB_TS; 190 FBNIC_XMIT_CB(skb)->hw_head = -1; 191 192 return true; 193 } 194 195 static bool 196 fbnic_tx_lso(struct fbnic_ring *ring, struct sk_buff *skb, 197 __le64 *meta, unsigned int *l2len, unsigned int *i3len) 198 { 199 unsigned int l3_type, l4_type, l4len, hdrlen; 200 struct skb_shared_info *shinfo; 201 unsigned char *l4hdr; 202 __be16 payload_len; 203 204 if (unlikely(skb_cow_head(skb, 0))) 205 return true; 206 207 shinfo = skb_shinfo(skb); 208 209 if (shinfo->gso_type & SKB_GSO_PARTIAL) { 210 l3_type = FBNIC_TWD_L3_TYPE_OTHER; 211 } else if (!skb->encapsulation) { 212 if (ip_hdr(skb)->version == 4) 213 l3_type = FBNIC_TWD_L3_TYPE_IPV4; 214 else 215 l3_type = FBNIC_TWD_L3_TYPE_IPV6; 216 } else { 217 unsigned int o3len; 218 219 o3len = skb_inner_network_header(skb) - skb_network_header(skb); 220 *i3len -= o3len; 221 *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L3_OHLEN_MASK, 222 o3len / 2)); 223 l3_type = FBNIC_TWD_L3_TYPE_V6V6; 224 } 225 226 l4hdr = skb_checksum_start(skb); 227 payload_len = cpu_to_be16(skb->len - (l4hdr - skb->data)); 228 229 if (shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) { 230 struct tcphdr *tcph = (struct tcphdr *)l4hdr; 231 232 l4_type = FBNIC_TWD_L4_TYPE_TCP; 233 l4len = __tcp_hdrlen((struct tcphdr *)l4hdr); 234 csum_replace_by_diff(&tcph->check, (__force __wsum)payload_len); 235 } else { 236 struct udphdr *udph = (struct udphdr *)l4hdr; 237 238 l4_type = FBNIC_TWD_L4_TYPE_UDP; 239 l4len = sizeof(struct udphdr); 240 csum_replace_by_diff(&udph->check, (__force __wsum)payload_len); 241 } 242 243 hdrlen = (l4hdr - skb->data) + l4len; 244 *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L3_TYPE_MASK, l3_type) | 245 FIELD_PREP(FBNIC_TWD_L4_TYPE_MASK, l4_type) | 246 FIELD_PREP(FBNIC_TWD_L4_HLEN_MASK, l4len / 4) | 247 FIELD_PREP(FBNIC_TWD_MSS_MASK, shinfo->gso_size) | 248 FBNIC_TWD_FLAG_REQ_LSO); 249 250 FBNIC_XMIT_CB(skb)->bytecount += (shinfo->gso_segs - 1) * hdrlen; 251 FBNIC_XMIT_CB(skb)->gso_segs = shinfo->gso_segs; 252 253 u64_stats_update_begin(&ring->stats.syncp); 254 ring->stats.twq.lso += shinfo->gso_segs; 255 u64_stats_update_end(&ring->stats.syncp); 256 257 return false; 258 } 259 260 static bool 261 fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) 262 { 263 unsigned int l2len, i3len; 264 265 if (fbnic_tx_tstamp(skb)) 266 *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_TS); 267 268 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) 269 return false; 270 271 l2len = skb_mac_header_len(skb); 272 i3len = skb_checksum_start(skb) - skb_network_header(skb); 273 274 *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_CSUM_OFFSET_MASK, 275 skb->csum_offset / 2)); 276 277 if (skb_is_gso(skb)) { 278 if (fbnic_tx_lso(ring, skb, meta, &l2len, &i3len)) 279 return true; 280 } else { 281 *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_CSO); 282 u64_stats_update_begin(&ring->stats.syncp); 283 ring->stats.twq.csum_partial++; 284 u64_stats_update_end(&ring->stats.syncp); 285 } 286 287 *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L2_HLEN_MASK, l2len / 2) | 288 FIELD_PREP(FBNIC_TWD_L3_IHLEN_MASK, i3len / 2)); 289 return false; 290 } 291 292 static void 293 fbnic_rx_csum(u64 rcd, struct sk_buff *skb, struct fbnic_ring *rcq, 294 u64 *csum_cmpl, u64 *csum_none) 295 { 296 skb_checksum_none_assert(skb); 297 298 if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) { 299 (*csum_none)++; 300 return; 301 } 302 303 if (FIELD_GET(FBNIC_RCD_META_L4_CSUM_UNNECESSARY, rcd)) { 304 skb->ip_summed = CHECKSUM_UNNECESSARY; 305 } else { 306 u16 csum = FIELD_GET(FBNIC_RCD_META_L2_CSUM_MASK, rcd); 307 308 skb->ip_summed = CHECKSUM_COMPLETE; 309 skb->csum = (__force __wsum)csum; 310 (*csum_cmpl)++; 311 } 312 } 313 314 static void fbnic_tx_doorbell(struct fbnic_ring *ring, __le64 *meta) 315 { 316 *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_COMPLETION); 317 ring->deferred_meta = -1; 318 319 /* Force DMA writes to flush before writing to tail */ 320 dma_wmb(); 321 322 writel(ring->tail, ring->doorbell); 323 } 324 325 /* Packets handed to us with xmit_more set are left in the ring without a 326 * doorbell, and without a completion request, in the expectation that the 327 * packet ending the burst will ring for all of them. If that packet gets 328 * dropped instead we have to ring here, otherwise the descriptors sit in 329 * the ring until the next transmit, which may never come. 330 */ 331 static void fbnic_tx_flush_doorbell(struct fbnic_ring *ring) 332 { 333 if (ring->deferred_meta >= 0) 334 fbnic_tx_doorbell(ring, &ring->desc[ring->deferred_meta]); 335 } 336 337 static bool 338 fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) 339 { 340 struct device *dev = skb->dev->dev.parent; 341 unsigned int tail = ring->tail, first; 342 unsigned int size, data_len; 343 skb_frag_t *frag; 344 bool is_net_iov; 345 dma_addr_t dma; 346 __le64 *twd; 347 348 ring->tx_buf[tail] = skb; 349 350 tail++; 351 tail &= ring->size_mask; 352 first = tail; 353 354 size = skb_headlen(skb); 355 data_len = skb->data_len; 356 357 if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) 358 goto dma_error; 359 360 is_net_iov = false; 361 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); 362 363 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 364 twd = &ring->desc[tail]; 365 366 if (dma_mapping_error(dev, dma)) 367 goto dma_error; 368 369 *twd = cpu_to_le64(FIELD_PREP(FBNIC_TWD_ADDR_MASK, dma) | 370 FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | 371 FIELD_PREP(FBNIC_TWD_TYPE_MASK, 372 FBNIC_TWD_TYPE_AL)); 373 if (is_net_iov) 374 ring->tx_buf[tail] = FBNIC_XMIT_NOUNMAP; 375 376 tail++; 377 tail &= ring->size_mask; 378 379 if (!data_len) 380 break; 381 382 size = skb_frag_size(frag); 383 data_len -= size; 384 385 if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) 386 goto dma_error; 387 388 is_net_iov = skb_frag_is_net_iov(frag); 389 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); 390 } 391 392 *twd |= FBNIC_TWD_TYPE(LAST_AL); 393 394 FBNIC_XMIT_CB(skb)->desc_count = ((twd - meta) + 1) & ring->size_mask; 395 396 ring->tail = tail; 397 398 /* Record SW timestamp */ 399 skb_tx_timestamp(skb); 400 401 /* Verify there is room for another packet */ 402 fbnic_maybe_stop_tx(skb->dev, ring, FBNIC_MAX_SKB_DESC); 403 404 if (fbnic_tx_sent_queue(skb, ring)) 405 fbnic_tx_doorbell(ring, meta); 406 else 407 ring->deferred_meta = meta - ring->desc; 408 409 return false; 410 dma_error: 411 if (net_ratelimit()) 412 netdev_err(skb->dev, "TX DMA map failed\n"); 413 414 while (tail != first) { 415 tail--; 416 tail &= ring->size_mask; 417 twd = &ring->desc[tail]; 418 if (tail == first) 419 fbnic_unmap_single_twd(dev, twd); 420 else if (ring->tx_buf[tail] == FBNIC_XMIT_NOUNMAP) 421 ring->tx_buf[tail] = NULL; 422 else 423 fbnic_unmap_page_twd(dev, twd); 424 } 425 426 return true; 427 } 428 429 #define FBNIC_MIN_FRAME_LEN 60 430 431 static netdev_tx_t 432 fbnic_xmit_frame_ring(struct sk_buff *skb, struct fbnic_ring *ring) 433 { 434 __le64 *meta = &ring->desc[ring->tail]; 435 u16 desc_needed; 436 437 if (skb_put_padto(skb, FBNIC_MIN_FRAME_LEN)) 438 goto err_count; 439 440 /* Need: 1 descriptor per page, 441 * + 1 desc for skb_head, 442 * + 2 desc for metadata and timestamp metadata 443 * + 7 desc gap to keep tail from touching head 444 * otherwise try next time 445 */ 446 desc_needed = skb_shinfo(skb)->nr_frags + 10; 447 if (fbnic_maybe_stop_tx(skb->dev, ring, desc_needed)) { 448 fbnic_tx_flush_doorbell(ring); 449 return NETDEV_TX_BUSY; 450 } 451 452 *meta = cpu_to_le64(FBNIC_TWD_FLAG_DEST_MAC); 453 454 /* Write all members within DWORD to condense this into 2 4B writes */ 455 FBNIC_XMIT_CB(skb)->bytecount = skb->len; 456 FBNIC_XMIT_CB(skb)->gso_segs = 1; 457 FBNIC_XMIT_CB(skb)->desc_count = 0; 458 FBNIC_XMIT_CB(skb)->flags = 0; 459 460 if (fbnic_tx_offloads(ring, skb, meta)) 461 goto err_free; 462 463 if (fbnic_tx_map(ring, skb, meta)) 464 goto err_free; 465 466 return NETDEV_TX_OK; 467 468 err_free: 469 dev_kfree_skb_any(skb); 470 err_count: 471 fbnic_tx_flush_doorbell(ring); 472 473 u64_stats_update_begin(&ring->stats.syncp); 474 ring->stats.dropped++; 475 u64_stats_update_end(&ring->stats.syncp); 476 return NETDEV_TX_OK; 477 } 478 479 netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev) 480 { 481 struct fbnic_net *fbn = netdev_priv(dev); 482 unsigned int q_map = skb->queue_mapping; 483 484 return fbnic_xmit_frame_ring(skb, fbn->tx[q_map]); 485 } 486 487 static netdev_features_t 488 fbnic_features_check_encap_gso(struct sk_buff *skb, struct net_device *dev, 489 netdev_features_t features, unsigned int l3len) 490 { 491 netdev_features_t skb_gso_features; 492 struct ipv6hdr *ip6_hdr; 493 unsigned char l4_hdr; 494 unsigned int start; 495 __be16 frag_off; 496 497 /* Require MANGLEID for GSO_PARTIAL of IPv4. 498 * In theory we could support TSO with single, innermost v4 header 499 * by pretending everything before it is L2, but that needs to be 500 * parsed case by case.. so leaving it for when the need arises. 501 */ 502 if (!(features & NETIF_F_TSO_MANGLEID)) 503 features &= ~NETIF_F_TSO; 504 505 skb_gso_features = skb_shinfo(skb)->gso_type; 506 skb_gso_features <<= NETIF_F_GSO_SHIFT; 507 508 /* We'd only clear the native GSO features, so don't bother validating 509 * if the match can only be on those supported thru GSO_PARTIAL. 510 */ 511 if (!(skb_gso_features & FBNIC_TUN_GSO_FEATURES)) 512 return features; 513 514 /* We can only do IPv6-in-IPv6, not v4-in-v6. It'd be nice 515 * to fall back to partial for this, or any failure below. 516 * This is just an optimization, UDPv4 will be caught later on. 517 */ 518 if (skb_gso_features & NETIF_F_TSO) 519 return features & ~FBNIC_TUN_GSO_FEATURES; 520 521 /* Inner headers multiple of 2 */ 522 if ((skb_inner_network_header(skb) - skb_network_header(skb)) % 2) 523 return features & ~FBNIC_TUN_GSO_FEATURES; 524 525 /* Encapsulated GSO packet, make 100% sure it's IPv6-in-IPv6. */ 526 ip6_hdr = ipv6_hdr(skb); 527 if (ip6_hdr->version != 6) 528 return features & ~FBNIC_TUN_GSO_FEATURES; 529 530 l4_hdr = ip6_hdr->nexthdr; 531 start = (unsigned char *)ip6_hdr - skb->data + sizeof(struct ipv6hdr); 532 start = ipv6_skip_exthdr(skb, start, &l4_hdr, &frag_off); 533 if (frag_off || l4_hdr != IPPROTO_IPV6 || 534 skb->data + start != skb_inner_network_header(skb)) 535 return features & ~FBNIC_TUN_GSO_FEATURES; 536 537 return features; 538 } 539 540 netdev_features_t 541 fbnic_features_check(struct sk_buff *skb, struct net_device *dev, 542 netdev_features_t features) 543 { 544 unsigned int l2len, l3len; 545 546 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) 547 return features; 548 549 l2len = skb_mac_header_len(skb); 550 l3len = skb_checksum_start(skb) - skb_network_header(skb); 551 552 /* Check header lengths are multiple of 2. 553 * In case of 6in6 we support longer headers (IHLEN + OHLEN) 554 * but keep things simple for now, 512B is plenty. 555 */ 556 if ((l2len | l3len | skb->csum_offset) % 2 || 557 !FIELD_FIT(FBNIC_TWD_L2_HLEN_MASK, l2len / 2) || 558 !FIELD_FIT(FBNIC_TWD_L3_IHLEN_MASK, l3len / 2) || 559 !FIELD_FIT(FBNIC_TWD_CSUM_OFFSET_MASK, skb->csum_offset / 2)) 560 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 561 562 if (likely(!skb->encapsulation) || !skb_is_gso(skb)) 563 return features; 564 565 return fbnic_features_check_encap_gso(skb, dev, features, l3len); 566 } 567 568 static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, 569 struct fbnic_ring *ring, bool discard, 570 unsigned int hw_head) 571 { 572 u64 total_bytes = 0, total_packets = 0, ts_lost = 0; 573 unsigned int head = ring->head; 574 struct netdev_queue *txq; 575 unsigned int clean_desc; 576 577 clean_desc = (hw_head - head) & ring->size_mask; 578 579 while (clean_desc) { 580 struct sk_buff *skb = ring->tx_buf[head]; 581 unsigned int desc_cnt; 582 583 desc_cnt = FBNIC_XMIT_CB(skb)->desc_count; 584 if (desc_cnt > clean_desc) 585 break; 586 587 if (unlikely(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)) { 588 FBNIC_XMIT_CB(skb)->hw_head = hw_head; 589 if (likely(!discard)) 590 break; 591 ts_lost++; 592 } 593 594 ring->tx_buf[head] = NULL; 595 596 clean_desc -= desc_cnt; 597 598 while (!(ring->desc[head] & FBNIC_TWD_TYPE(AL))) { 599 head++; 600 head &= ring->size_mask; 601 desc_cnt--; 602 } 603 604 fbnic_unmap_single_twd(nv->dev, &ring->desc[head]); 605 head++; 606 head &= ring->size_mask; 607 desc_cnt--; 608 609 while (desc_cnt--) { 610 if (ring->tx_buf[head] != FBNIC_XMIT_NOUNMAP) 611 fbnic_unmap_page_twd(nv->dev, 612 &ring->desc[head]); 613 else 614 ring->tx_buf[head] = NULL; 615 head++; 616 head &= ring->size_mask; 617 } 618 619 total_bytes += FBNIC_XMIT_CB(skb)->bytecount; 620 total_packets += FBNIC_XMIT_CB(skb)->gso_segs; 621 622 napi_consume_skb(skb, napi_budget); 623 } 624 625 if (!total_bytes) 626 return; 627 628 ring->head = head; 629 630 txq = txring_txq(nv->napi.dev, ring); 631 632 if (unlikely(discard)) { 633 u64_stats_update_begin(&ring->stats.syncp); 634 ring->stats.dropped += total_packets; 635 ring->stats.twq.ts_lost += ts_lost; 636 u64_stats_update_end(&ring->stats.syncp); 637 638 netdev_tx_completed_queue(txq, total_packets, total_bytes); 639 return; 640 } 641 642 u64_stats_update_begin(&ring->stats.syncp); 643 ring->stats.bytes += total_bytes; 644 ring->stats.packets += total_packets; 645 u64_stats_update_end(&ring->stats.syncp); 646 647 if (!netif_txq_completed_wake(txq, total_packets, total_bytes, 648 fbnic_desc_unused(ring), 649 FBNIC_TX_DESC_WAKEUP)) { 650 u64_stats_update_begin(&ring->stats.syncp); 651 ring->stats.twq.wake++; 652 u64_stats_update_end(&ring->stats.syncp); 653 } 654 } 655 656 static void fbnic_clean_twq1(struct fbnic_napi_vector *nv, bool pp_allow_direct, 657 struct fbnic_ring *ring, bool discard, 658 unsigned int hw_head) 659 { 660 u64 total_bytes = 0, total_packets = 0; 661 unsigned int head = ring->head; 662 663 while (hw_head != head) { 664 struct page *page; 665 u64 twd; 666 667 if (unlikely(!(ring->desc[head] & FBNIC_TWD_TYPE(AL)))) 668 goto next_desc; 669 670 twd = le64_to_cpu(ring->desc[head]); 671 page = ring->tx_buf[head]; 672 673 /* TYPE_AL is 2, TYPE_LAST_AL is 3. So this trick gives 674 * us one increment per packet, with no branches. 675 */ 676 total_packets += FIELD_GET(FBNIC_TWD_TYPE_MASK, twd) - 677 FBNIC_TWD_TYPE_AL; 678 total_bytes += FIELD_GET(FBNIC_TWD_LEN_MASK, twd); 679 680 page_pool_put_page(pp_page_to_nmdesc(page)->pp, page, -1, 681 pp_allow_direct); 682 next_desc: 683 head++; 684 head &= ring->size_mask; 685 } 686 687 if (!total_bytes) 688 return; 689 690 ring->head = head; 691 692 if (discard) { 693 u64_stats_update_begin(&ring->stats.syncp); 694 ring->stats.dropped += total_packets; 695 u64_stats_update_end(&ring->stats.syncp); 696 return; 697 } 698 699 u64_stats_update_begin(&ring->stats.syncp); 700 ring->stats.bytes += total_bytes; 701 ring->stats.packets += total_packets; 702 u64_stats_update_end(&ring->stats.syncp); 703 } 704 705 static void fbnic_clean_tsq(struct fbnic_napi_vector *nv, 706 struct fbnic_ring *ring, 707 u64 tcd, int *ts_head, int *head0) 708 { 709 struct skb_shared_hwtstamps hwtstamp; 710 struct fbnic_net *fbn; 711 struct sk_buff *skb; 712 int head; 713 u64 ns; 714 715 head = (*ts_head < 0) ? ring->head : *ts_head; 716 717 do { 718 unsigned int desc_cnt; 719 720 if (head == ring->tail) { 721 if (unlikely(net_ratelimit())) 722 netdev_err(nv->napi.dev, 723 "Tx timestamp without matching packet\n"); 724 return; 725 } 726 727 skb = ring->tx_buf[head]; 728 desc_cnt = FBNIC_XMIT_CB(skb)->desc_count; 729 730 head += desc_cnt; 731 head &= ring->size_mask; 732 } while (!(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)); 733 734 fbn = netdev_priv(nv->napi.dev); 735 ns = fbnic_ts40_to_ns(fbn, FIELD_GET(FBNIC_TCD_TYPE1_TS_MASK, tcd)); 736 737 memset(&hwtstamp, 0, sizeof(hwtstamp)); 738 hwtstamp.hwtstamp = ns_to_ktime(ns); 739 740 *ts_head = head; 741 742 FBNIC_XMIT_CB(skb)->flags &= ~FBNIC_XMIT_CB_TS; 743 if (*head0 < 0) { 744 head = FBNIC_XMIT_CB(skb)->hw_head; 745 if (head >= 0) 746 *head0 = head; 747 } 748 749 skb_tstamp_tx(skb, &hwtstamp); 750 u64_stats_update_begin(&ring->stats.syncp); 751 ring->stats.twq.ts_packets++; 752 u64_stats_update_end(&ring->stats.syncp); 753 } 754 755 static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx, 756 netmem_ref netmem) 757 { 758 struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; 759 760 page_pool_fragment_netmem(netmem, FBNIC_PAGECNT_BIAS_MAX); 761 rx_buf->pagecnt_bias = FBNIC_PAGECNT_BIAS_MAX; 762 rx_buf->netmem = netmem; 763 } 764 765 static struct page * 766 fbnic_page_pool_get_head(struct fbnic_q_triad *qt, unsigned int idx) 767 { 768 struct fbnic_rx_buf *rx_buf = &qt->sub0.rx_buf[idx]; 769 770 rx_buf->pagecnt_bias--; 771 772 /* sub0 is always fed system pages, from the NAPI-level page_pool */ 773 return netmem_to_page(rx_buf->netmem); 774 } 775 776 static netmem_ref 777 fbnic_page_pool_get_data(struct fbnic_q_triad *qt, unsigned int idx) 778 { 779 struct fbnic_rx_buf *rx_buf = &qt->sub1.rx_buf[idx]; 780 781 rx_buf->pagecnt_bias--; 782 783 return rx_buf->netmem; 784 } 785 786 static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx, 787 int budget) 788 { 789 struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; 790 netmem_ref netmem = rx_buf->netmem; 791 792 if (!page_pool_unref_netmem(netmem, rx_buf->pagecnt_bias)) 793 page_pool_put_unrefed_netmem(ring->page_pool, netmem, -1, 794 !!budget); 795 796 rx_buf->netmem = 0; 797 } 798 799 static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget, 800 struct fbnic_q_triad *qt, s32 ts_head, s32 head0, 801 s32 head1) 802 { 803 if (head0 >= 0) 804 fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0); 805 else if (ts_head >= 0) 806 fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, ts_head); 807 808 if (head1 >= 0) { 809 qt->cmpl.deferred_head = -1; 810 if (napi_budget) 811 fbnic_clean_twq1(nv, true, &qt->sub1, false, head1); 812 else 813 qt->cmpl.deferred_head = head1; 814 } 815 } 816 817 static void 818 fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, 819 int napi_budget) 820 { 821 struct fbnic_ring *cmpl = &qt->cmpl; 822 s32 head1 = cmpl->deferred_head; 823 s32 head0 = -1, ts_head = -1; 824 __le64 *raw_tcd, done; 825 u32 head = cmpl->head; 826 827 done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(FBNIC_TCD_DONE); 828 raw_tcd = &cmpl->desc[head & cmpl->size_mask]; 829 830 /* Walk the completion queue collecting the heads reported by NIC */ 831 while ((*raw_tcd & cpu_to_le64(FBNIC_TCD_DONE)) == done) { 832 u64 tcd; 833 834 dma_rmb(); 835 836 tcd = le64_to_cpu(*raw_tcd); 837 838 switch (FIELD_GET(FBNIC_TCD_TYPE_MASK, tcd)) { 839 case FBNIC_TCD_TYPE_0: 840 if (tcd & FBNIC_TCD_TWQ1) 841 head1 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD1_MASK, 842 tcd); 843 else 844 head0 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD0_MASK, 845 tcd); 846 /* Currently all err status bits are related to 847 * timestamps and as those have yet to be added 848 * they are skipped for now. 849 */ 850 break; 851 case FBNIC_TCD_TYPE_1: 852 if (WARN_ON_ONCE(tcd & FBNIC_TCD_TWQ1)) 853 break; 854 855 fbnic_clean_tsq(nv, &qt->sub0, tcd, &ts_head, &head0); 856 break; 857 default: 858 break; 859 } 860 861 raw_tcd++; 862 head++; 863 if (!(head & cmpl->size_mask)) { 864 done ^= cpu_to_le64(FBNIC_TCD_DONE); 865 raw_tcd = &cmpl->desc[0]; 866 } 867 } 868 869 /* Record the current head/tail of the queue */ 870 if (cmpl->head != head) { 871 cmpl->head = head; 872 writel(head & cmpl->size_mask, cmpl->doorbell); 873 } 874 875 /* Unmap and free processed buffers */ 876 fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0, head1); 877 } 878 879 static void fbnic_clean_bdq(struct fbnic_ring *ring, unsigned int hw_head, 880 int napi_budget) 881 { 882 unsigned int head = ring->head; 883 884 if (head == hw_head) 885 return; 886 887 do { 888 fbnic_page_pool_drain(ring, head, napi_budget); 889 890 head++; 891 head &= ring->size_mask; 892 } while (head != hw_head); 893 894 ring->head = head; 895 } 896 897 static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, netmem_ref netmem) 898 { 899 __le64 *bdq_desc = &bdq->desc[id * FBNIC_BD_FRAG_COUNT]; 900 dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem); 901 u64 bd, i = FBNIC_BD_FRAG_COUNT; 902 903 bd = (FBNIC_BD_PAGE_ADDR_MASK & dma) | 904 FIELD_PREP(FBNIC_BD_PAGE_ID_MASK, id); 905 906 /* In the case that a page size is larger than 4K we will map a 907 * single page to multiple fragments. The fragments will be 908 * FBNIC_BD_FRAG_COUNT in size and the lower n bits will be use 909 * to indicate the individual fragment IDs. 910 */ 911 do { 912 *bdq_desc = cpu_to_le64(bd); 913 bd += FIELD_PREP(FBNIC_BD_DESC_ADDR_MASK, 1) | 914 FIELD_PREP(FBNIC_BD_DESC_ID_MASK, 1); 915 bdq_desc++; 916 } while (--i); 917 } 918 919 static void fbnic_fill_bdq(struct fbnic_ring *bdq) 920 { 921 unsigned int count = fbnic_desc_unused(bdq); 922 unsigned int i = bdq->tail; 923 924 if (!count) 925 return; 926 927 do { 928 netmem_ref netmem; 929 930 netmem = page_pool_dev_alloc_netmems(bdq->page_pool); 931 if (!netmem) { 932 u64_stats_update_begin(&bdq->stats.syncp); 933 bdq->stats.bdq.alloc_failed++; 934 u64_stats_update_end(&bdq->stats.syncp); 935 936 break; 937 } 938 939 fbnic_page_pool_init(bdq, i, netmem); 940 fbnic_bd_prep(bdq, i, netmem); 941 942 i++; 943 i &= bdq->size_mask; 944 945 count--; 946 } while (count); 947 948 if (bdq->tail != i) { 949 bdq->tail = i; 950 951 /* Force DMA writes to flush before writing to tail */ 952 dma_wmb(); 953 954 writel(i * FBNIC_BD_FRAG_COUNT, bdq->doorbell); 955 } 956 } 957 958 static unsigned int fbnic_hdr_pg_start(unsigned int pg_off) 959 { 960 /* The headroom of the first header may be larger than FBNIC_RX_HROOM 961 * due to alignment. So account for that by just making the page 962 * offset 0 if we are starting at the first header. 963 */ 964 if (ALIGN(FBNIC_RX_HROOM, 128) > FBNIC_RX_HROOM && 965 pg_off == ALIGN(FBNIC_RX_HROOM, 128)) 966 return 0; 967 968 return pg_off - FBNIC_RX_HROOM; 969 } 970 971 static unsigned int fbnic_hdr_pg_end(unsigned int pg_off, unsigned int len) 972 { 973 /* Determine the end of the buffer by finding the start of the next 974 * and then subtracting the headroom from that frame. 975 */ 976 pg_off += len + FBNIC_RX_TROOM + FBNIC_RX_HROOM; 977 978 return ALIGN(pg_off, 128) - FBNIC_RX_HROOM; 979 } 980 981 static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, 982 struct fbnic_pkt_buff *pkt, 983 struct fbnic_q_triad *qt) 984 { 985 unsigned int hdr_pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); 986 unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); 987 struct page *page = fbnic_page_pool_get_head(qt, hdr_pg_idx); 988 unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); 989 unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom; 990 unsigned char *hdr_start; 991 992 /* data_hard_start should always be NULL when this is called */ 993 WARN_ON_ONCE(pkt->buff.data_hard_start); 994 995 /* Short-cut the end calculation if we know page is fully consumed */ 996 hdr_pg_end = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? 997 FBNIC_BD_FRAG_SIZE : fbnic_hdr_pg_end(hdr_pg_off, len); 998 hdr_pg_start = fbnic_hdr_pg_start(hdr_pg_off); 999 1000 headroom = hdr_pg_off - hdr_pg_start + FBNIC_RX_PAD; 1001 frame_sz = hdr_pg_end - hdr_pg_start; 1002 xdp_init_buff(&pkt->buff, frame_sz, &qt->xdp_rxq); 1003 hdr_pg_start += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * 1004 FBNIC_BD_FRAG_SIZE; 1005 1006 /* Sync DMA buffer */ 1007 dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), 1008 hdr_pg_start, frame_sz, 1009 DMA_BIDIRECTIONAL); 1010 1011 /* Build frame around buffer */ 1012 hdr_start = page_address(page) + hdr_pg_start; 1013 net_prefetch(pkt->buff.data); 1014 xdp_prepare_buff(&pkt->buff, hdr_start, headroom, 1015 len - FBNIC_RX_PAD, true); 1016 1017 pkt->hwtstamp = 0; 1018 pkt->add_frag_failed = false; 1019 } 1020 1021 static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, 1022 struct fbnic_pkt_buff *pkt, 1023 struct fbnic_q_triad *qt) 1024 { 1025 unsigned int pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); 1026 unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); 1027 unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); 1028 netmem_ref netmem = fbnic_page_pool_get_data(qt, pg_idx); 1029 unsigned int truesize; 1030 bool added; 1031 1032 truesize = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? 1033 FBNIC_BD_FRAG_SIZE - pg_off : ALIGN(len, 128); 1034 1035 pg_off += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * 1036 FBNIC_BD_FRAG_SIZE; 1037 1038 /* Sync DMA buffer */ 1039 page_pool_dma_sync_netmem_for_cpu(qt->sub1.page_pool, netmem, 1040 pg_off, truesize); 1041 1042 added = xdp_buff_add_frag(&pkt->buff, netmem, pg_off, len, truesize); 1043 if (unlikely(!added)) { 1044 pkt->add_frag_failed = true; 1045 netdev_err_once(nv->napi.dev, 1046 "Failed to add fragment to xdp_buff\n"); 1047 } 1048 } 1049 1050 static void fbnic_put_pkt_buff(struct fbnic_q_triad *qt, 1051 struct fbnic_pkt_buff *pkt, int budget) 1052 { 1053 struct page *page; 1054 1055 if (!pkt->buff.data_hard_start) 1056 return; 1057 1058 if (xdp_buff_has_frags(&pkt->buff)) { 1059 struct skb_shared_info *shinfo; 1060 netmem_ref netmem; 1061 int nr_frags; 1062 1063 shinfo = xdp_get_shared_info_from_buff(&pkt->buff); 1064 nr_frags = shinfo->nr_frags; 1065 1066 while (nr_frags--) { 1067 netmem = skb_frag_netmem(&shinfo->frags[nr_frags]); 1068 page_pool_put_full_netmem(qt->sub1.page_pool, netmem, 1069 !!budget); 1070 } 1071 } 1072 1073 page = virt_to_page(pkt->buff.data_hard_start); 1074 page_pool_put_full_page(qt->sub0.page_pool, page, !!budget); 1075 } 1076 1077 static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv, 1078 struct fbnic_pkt_buff *pkt) 1079 { 1080 struct sk_buff *skb; 1081 1082 skb = xdp_build_skb_from_buff(&pkt->buff); 1083 if (!skb) 1084 return NULL; 1085 1086 /* Add timestamp if present */ 1087 if (pkt->hwtstamp) 1088 skb_hwtstamps(skb)->hwtstamp = pkt->hwtstamp; 1089 1090 return skb; 1091 } 1092 1093 static long fbnic_pkt_tx(struct fbnic_napi_vector *nv, 1094 struct fbnic_pkt_buff *pkt) 1095 { 1096 struct fbnic_ring *ring = &nv->qt[0].sub1; 1097 int size, offset, nsegs = 1, data_len = 0; 1098 unsigned int tail = ring->tail; 1099 struct skb_shared_info *shinfo; 1100 skb_frag_t *frag = NULL; 1101 struct page *page; 1102 dma_addr_t dma; 1103 __le64 *twd; 1104 1105 if (unlikely(xdp_buff_has_frags(&pkt->buff))) { 1106 shinfo = xdp_get_shared_info_from_buff(&pkt->buff); 1107 nsegs += shinfo->nr_frags; 1108 data_len = shinfo->xdp_frags_size; 1109 frag = &shinfo->frags[0]; 1110 } 1111 1112 if (fbnic_desc_unused(ring) < nsegs) { 1113 u64_stats_update_begin(&ring->stats.syncp); 1114 ring->stats.dropped++; 1115 u64_stats_update_end(&ring->stats.syncp); 1116 return -FBNIC_XDP_CONSUME; 1117 } 1118 1119 page = virt_to_page(pkt->buff.data_hard_start); 1120 offset = offset_in_page(pkt->buff.data); 1121 dma = page_pool_get_dma_addr(page); 1122 1123 size = pkt->buff.data_end - pkt->buff.data; 1124 1125 while (nsegs--) { 1126 dma_sync_single_range_for_device(nv->dev, dma, offset, size, 1127 DMA_BIDIRECTIONAL); 1128 dma += offset; 1129 1130 ring->tx_buf[tail] = page; 1131 1132 twd = &ring->desc[tail]; 1133 *twd = cpu_to_le64(FIELD_PREP(FBNIC_TWD_ADDR_MASK, dma) | 1134 FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | 1135 FIELD_PREP(FBNIC_TWD_TYPE_MASK, 1136 FBNIC_TWD_TYPE_AL)); 1137 1138 tail++; 1139 tail &= ring->size_mask; 1140 1141 if (!data_len) 1142 break; 1143 1144 offset = skb_frag_off(frag); 1145 page = skb_frag_page(frag); 1146 dma = page_pool_get_dma_addr(page); 1147 1148 size = skb_frag_size(frag); 1149 data_len -= size; 1150 frag++; 1151 } 1152 1153 *twd |= FBNIC_TWD_TYPE(LAST_AL); 1154 1155 ring->tail = tail; 1156 1157 return -FBNIC_XDP_TX; 1158 } 1159 1160 static void fbnic_pkt_commit_tail(struct fbnic_napi_vector *nv, 1161 unsigned int pkt_tail) 1162 { 1163 struct fbnic_ring *ring = &nv->qt[0].sub1; 1164 1165 /* Force DMA writes to flush before writing to tail */ 1166 dma_wmb(); 1167 1168 writel(pkt_tail, ring->doorbell); 1169 } 1170 1171 static struct sk_buff *fbnic_run_xdp(struct fbnic_napi_vector *nv, 1172 struct fbnic_pkt_buff *pkt) 1173 { 1174 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 1175 struct bpf_prog *xdp_prog; 1176 int act; 1177 1178 xdp_prog = READ_ONCE(fbn->xdp_prog); 1179 if (!xdp_prog) 1180 goto xdp_pass; 1181 1182 /* Should never happen, config paths enforce HDS threshold > MTU */ 1183 if (xdp_buff_has_frags(&pkt->buff) && !xdp_prog->aux->xdp_has_frags) 1184 return ERR_PTR(-FBNIC_XDP_LEN_ERR); 1185 1186 act = bpf_prog_run_xdp(xdp_prog, &pkt->buff); 1187 switch (act) { 1188 case XDP_PASS: 1189 xdp_pass: 1190 return fbnic_build_skb(nv, pkt); 1191 case XDP_TX: 1192 return ERR_PTR(fbnic_pkt_tx(nv, pkt)); 1193 default: 1194 bpf_warn_invalid_xdp_action(nv->napi.dev, xdp_prog, act); 1195 fallthrough; 1196 case XDP_ABORTED: 1197 trace_xdp_exception(nv->napi.dev, xdp_prog, act); 1198 fallthrough; 1199 case XDP_DROP: 1200 break; 1201 } 1202 1203 return ERR_PTR(-FBNIC_XDP_CONSUME); 1204 } 1205 1206 static enum pkt_hash_types fbnic_skb_hash_type(u64 rcd) 1207 { 1208 return (FBNIC_RCD_META_L4_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L4 : 1209 (FBNIC_RCD_META_L3_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L3 : 1210 PKT_HASH_TYPE_L2; 1211 } 1212 1213 static void fbnic_rx_tstamp(struct fbnic_napi_vector *nv, u64 rcd, 1214 struct fbnic_pkt_buff *pkt) 1215 { 1216 struct fbnic_net *fbn; 1217 u64 ns, ts; 1218 1219 if (!FIELD_GET(FBNIC_RCD_OPT_META_TS, rcd)) 1220 return; 1221 1222 fbn = netdev_priv(nv->napi.dev); 1223 ts = FIELD_GET(FBNIC_RCD_OPT_META_TS_MASK, rcd); 1224 ns = fbnic_ts40_to_ns(fbn, ts); 1225 1226 /* Add timestamp to shared info */ 1227 pkt->hwtstamp = ns_to_ktime(ns); 1228 } 1229 1230 static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv, 1231 u64 rcd, struct sk_buff *skb, 1232 struct fbnic_q_triad *qt, 1233 u64 *csum_cmpl, u64 *csum_none) 1234 { 1235 struct net_device *netdev = nv->napi.dev; 1236 struct fbnic_ring *rcq = &qt->cmpl; 1237 1238 fbnic_rx_csum(rcd, skb, rcq, csum_cmpl, csum_none); 1239 1240 if (netdev->features & NETIF_F_RXHASH) 1241 skb_set_hash(skb, 1242 FIELD_GET(FBNIC_RCD_META_RSS_HASH_MASK, rcd), 1243 fbnic_skb_hash_type(rcd)); 1244 1245 skb_record_rx_queue(skb, rcq->q_idx); 1246 } 1247 1248 static bool fbnic_rcd_metadata_err(u64 rcd) 1249 { 1250 return !!(FBNIC_RCD_META_UNCORRECTABLE_ERR_MASK & rcd); 1251 } 1252 1253 static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, 1254 struct fbnic_q_triad *qt, int budget) 1255 { 1256 unsigned int packets = 0, bytes = 0, dropped = 0, alloc_failed = 0; 1257 u64 csum_complete = 0, csum_none = 0, length_errors = 0; 1258 s32 head0 = -1, head1 = -1, pkt_tail = -1; 1259 struct fbnic_ring *rcq = &qt->cmpl; 1260 struct fbnic_pkt_buff *pkt; 1261 __le64 *raw_rcd, done; 1262 u32 head = rcq->head; 1263 1264 done = (head & (rcq->size_mask + 1)) ? cpu_to_le64(FBNIC_RCD_DONE) : 0; 1265 raw_rcd = &rcq->desc[head & rcq->size_mask]; 1266 pkt = rcq->pkt; 1267 1268 /* Walk the completion queue collecting the heads reported by NIC */ 1269 while (likely(packets < budget)) { 1270 struct sk_buff *skb = ERR_PTR(-EINVAL); 1271 u32 pkt_bytes; 1272 u64 rcd; 1273 1274 if ((*raw_rcd & cpu_to_le64(FBNIC_RCD_DONE)) == done) 1275 break; 1276 1277 dma_rmb(); 1278 1279 rcd = le64_to_cpu(*raw_rcd); 1280 1281 switch (FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd)) { 1282 case FBNIC_RCD_TYPE_HDR_AL: 1283 head0 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); 1284 fbnic_pkt_prepare(nv, rcd, pkt, qt); 1285 1286 break; 1287 case FBNIC_RCD_TYPE_PAY_AL: 1288 head1 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); 1289 fbnic_add_rx_frag(nv, rcd, pkt, qt); 1290 1291 break; 1292 case FBNIC_RCD_TYPE_OPT_META: 1293 /* Only type 0 is currently supported */ 1294 if (FIELD_GET(FBNIC_RCD_OPT_META_TYPE_MASK, rcd)) 1295 break; 1296 1297 fbnic_rx_tstamp(nv, rcd, pkt); 1298 1299 /* We currently ignore the action table index */ 1300 break; 1301 case FBNIC_RCD_TYPE_META: 1302 if (likely(!fbnic_rcd_metadata_err(rcd) && 1303 !pkt->add_frag_failed)) { 1304 pkt_bytes = xdp_get_buff_len(&pkt->buff); 1305 skb = fbnic_run_xdp(nv, pkt); 1306 } 1307 1308 /* Populate skb and invalidate XDP */ 1309 if (!IS_ERR_OR_NULL(skb)) { 1310 fbnic_populate_skb_fields(nv, rcd, skb, qt, 1311 &csum_complete, 1312 &csum_none); 1313 napi_gro_receive(&nv->napi, skb); 1314 } else if (skb == ERR_PTR(-FBNIC_XDP_TX)) { 1315 pkt_tail = nv->qt[0].sub1.tail; 1316 } else if (PTR_ERR(skb) == -FBNIC_XDP_CONSUME) { 1317 fbnic_put_pkt_buff(qt, pkt, 1); 1318 } else { 1319 if (!skb) 1320 alloc_failed++; 1321 1322 if (skb == ERR_PTR(-FBNIC_XDP_LEN_ERR)) 1323 length_errors++; 1324 else 1325 dropped++; 1326 1327 fbnic_put_pkt_buff(qt, pkt, 1); 1328 goto next_dont_count; 1329 } 1330 1331 packets++; 1332 bytes += pkt_bytes; 1333 next_dont_count: 1334 pkt->buff.data_hard_start = NULL; 1335 1336 break; 1337 } 1338 1339 raw_rcd++; 1340 head++; 1341 if (!(head & rcq->size_mask)) { 1342 done ^= cpu_to_le64(FBNIC_RCD_DONE); 1343 raw_rcd = &rcq->desc[0]; 1344 } 1345 } 1346 1347 u64_stats_update_begin(&rcq->stats.syncp); 1348 rcq->stats.packets += packets; 1349 rcq->stats.bytes += bytes; 1350 rcq->stats.dropped += dropped; 1351 rcq->stats.rx.alloc_failed += alloc_failed; 1352 rcq->stats.rx.csum_complete += csum_complete; 1353 rcq->stats.rx.csum_none += csum_none; 1354 rcq->stats.rx.length_errors += length_errors; 1355 u64_stats_update_end(&rcq->stats.syncp); 1356 1357 if (pkt_tail >= 0) 1358 fbnic_pkt_commit_tail(nv, pkt_tail); 1359 1360 /* Unmap and free processed buffers */ 1361 if (head0 >= 0) 1362 fbnic_clean_bdq(&qt->sub0, head0, budget); 1363 fbnic_fill_bdq(&qt->sub0); 1364 1365 if (head1 >= 0) 1366 fbnic_clean_bdq(&qt->sub1, head1, budget); 1367 fbnic_fill_bdq(&qt->sub1); 1368 1369 /* Record the current head/tail of the queue */ 1370 if (rcq->head != head) { 1371 rcq->head = head; 1372 writel(head & rcq->size_mask, rcq->doorbell); 1373 } 1374 1375 return packets; 1376 } 1377 1378 static void fbnic_nv_irq_disable(struct fbnic_napi_vector *nv) 1379 { 1380 struct fbnic_dev *fbd = nv->fbd; 1381 u32 v_idx = nv->v_idx; 1382 1383 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(v_idx / 32), 1 << (v_idx % 32)); 1384 } 1385 1386 static void fbnic_nv_irq_rearm(struct fbnic_napi_vector *nv) 1387 { 1388 struct fbnic_dev *fbd = nv->fbd; 1389 u32 v_idx = nv->v_idx; 1390 1391 fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(v_idx), 1392 FBNIC_INTR_CQ_REARM_INTR_UNMASK); 1393 } 1394 1395 static int fbnic_poll(struct napi_struct *napi, int budget) 1396 { 1397 struct fbnic_napi_vector *nv = container_of(napi, 1398 struct fbnic_napi_vector, 1399 napi); 1400 int i, j, work_done = 0; 1401 1402 for (i = 0; i < nv->txt_count; i++) 1403 fbnic_clean_tcq(nv, &nv->qt[i], budget); 1404 1405 for (j = 0; j < nv->rxt_count; j++, i++) 1406 work_done += fbnic_clean_rcq(nv, &nv->qt[i], budget); 1407 1408 if (work_done >= budget) 1409 return budget; 1410 1411 if (likely(napi_complete_done(napi, work_done))) 1412 fbnic_nv_irq_rearm(nv); 1413 1414 return work_done; 1415 } 1416 1417 irqreturn_t fbnic_msix_clean_rings(int __always_unused irq, void *data) 1418 { 1419 struct fbnic_napi_vector *nv = *(void **)data; 1420 1421 napi_schedule_irqoff(&nv->napi); 1422 1423 return IRQ_HANDLED; 1424 } 1425 1426 void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn, 1427 struct fbnic_ring *rxr) 1428 { 1429 struct fbnic_queue_stats *stats = &rxr->stats; 1430 1431 /* Capture stats from queues before dissasociating them */ 1432 fbn->rx_stats.bytes += stats->bytes; 1433 fbn->rx_stats.packets += stats->packets; 1434 fbn->rx_stats.dropped += stats->dropped; 1435 fbn->rx_stats.rx.alloc_failed += stats->rx.alloc_failed; 1436 fbn->rx_stats.rx.csum_complete += stats->rx.csum_complete; 1437 fbn->rx_stats.rx.csum_none += stats->rx.csum_none; 1438 fbn->rx_stats.rx.length_errors += stats->rx.length_errors; 1439 /* Remember to add new stats here */ 1440 BUILD_BUG_ON(sizeof(fbn->rx_stats.rx) / 8 != 4); 1441 } 1442 1443 void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn, 1444 struct fbnic_ring *bdq) 1445 { 1446 struct fbnic_queue_stats *stats = &bdq->stats; 1447 1448 /* Capture stats from queues before dissasociating them */ 1449 fbn->bdq_stats.bdq.alloc_failed += stats->bdq.alloc_failed; 1450 /* Remember to add new stats here */ 1451 BUILD_BUG_ON(sizeof(fbn->rx_stats.bdq) / 8 != 1); 1452 } 1453 1454 void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn, 1455 struct fbnic_ring *txr) 1456 { 1457 struct fbnic_queue_stats *stats = &txr->stats; 1458 1459 /* Capture stats from queues before dissasociating them */ 1460 fbn->tx_stats.bytes += stats->bytes; 1461 fbn->tx_stats.packets += stats->packets; 1462 fbn->tx_stats.dropped += stats->dropped; 1463 fbn->tx_stats.twq.csum_partial += stats->twq.csum_partial; 1464 fbn->tx_stats.twq.lso += stats->twq.lso; 1465 fbn->tx_stats.twq.ts_lost += stats->twq.ts_lost; 1466 fbn->tx_stats.twq.ts_packets += stats->twq.ts_packets; 1467 fbn->tx_stats.twq.stop += stats->twq.stop; 1468 fbn->tx_stats.twq.wake += stats->twq.wake; 1469 /* Remember to add new stats here */ 1470 BUILD_BUG_ON(sizeof(fbn->tx_stats.twq) / 8 != 6); 1471 } 1472 1473 void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn, 1474 struct fbnic_ring *xdpr) 1475 { 1476 struct fbnic_queue_stats *stats = &xdpr->stats; 1477 1478 if (!(xdpr->flags & FBNIC_RING_F_STATS)) 1479 return; 1480 1481 /* Capture stats from queues before dissasociating them */ 1482 fbn->tx_stats.dropped += stats->dropped; 1483 fbn->tx_stats.bytes += stats->bytes; 1484 fbn->tx_stats.packets += stats->packets; 1485 } 1486 1487 static void fbnic_remove_tx_ring(struct fbnic_net *fbn, 1488 struct fbnic_ring *txr) 1489 { 1490 if (!(txr->flags & FBNIC_RING_F_STATS)) 1491 return; 1492 1493 fbnic_aggregate_ring_tx_counters(fbn, txr); 1494 1495 /* Remove pointer to the Tx ring */ 1496 WARN_ON(fbn->tx[txr->q_idx] && fbn->tx[txr->q_idx] != txr); 1497 fbn->tx[txr->q_idx] = NULL; 1498 } 1499 1500 static void fbnic_remove_xdp_ring(struct fbnic_net *fbn, 1501 struct fbnic_ring *xdpr) 1502 { 1503 if (!(xdpr->flags & FBNIC_RING_F_STATS)) 1504 return; 1505 1506 fbnic_aggregate_ring_xdp_counters(fbn, xdpr); 1507 1508 /* Remove pointer to the Tx ring */ 1509 WARN_ON(fbn->tx[xdpr->q_idx] && fbn->tx[xdpr->q_idx] != xdpr); 1510 fbn->tx[xdpr->q_idx] = NULL; 1511 } 1512 1513 static void fbnic_remove_rx_ring(struct fbnic_net *fbn, 1514 struct fbnic_ring *rxr) 1515 { 1516 if (!(rxr->flags & FBNIC_RING_F_STATS)) 1517 return; 1518 1519 fbnic_aggregate_ring_rx_counters(fbn, rxr); 1520 1521 /* Remove pointer to the Rx ring */ 1522 WARN_ON(fbn->rx[rxr->q_idx] && fbn->rx[rxr->q_idx] != rxr); 1523 fbn->rx[rxr->q_idx] = NULL; 1524 } 1525 1526 static void fbnic_remove_bdq_ring(struct fbnic_net *fbn, 1527 struct fbnic_ring *bdq) 1528 { 1529 if (!(bdq->flags & FBNIC_RING_F_STATS)) 1530 return; 1531 1532 fbnic_aggregate_ring_bdq_counters(fbn, bdq); 1533 } 1534 1535 static void fbnic_free_qt_page_pools(struct fbnic_q_triad *qt) 1536 { 1537 page_pool_destroy(qt->sub0.page_pool); 1538 page_pool_destroy(qt->sub1.page_pool); 1539 } 1540 1541 static void fbnic_free_napi_vector(struct fbnic_net *fbn, 1542 struct fbnic_napi_vector *nv) 1543 { 1544 struct fbnic_dev *fbd = nv->fbd; 1545 int i, j; 1546 1547 for (i = 0; i < nv->txt_count; i++) { 1548 fbnic_remove_tx_ring(fbn, &nv->qt[i].sub0); 1549 fbnic_remove_xdp_ring(fbn, &nv->qt[i].sub1); 1550 fbnic_remove_tx_ring(fbn, &nv->qt[i].cmpl); 1551 } 1552 1553 for (j = 0; j < nv->rxt_count; j++, i++) { 1554 fbnic_remove_bdq_ring(fbn, &nv->qt[i].sub0); 1555 fbnic_remove_bdq_ring(fbn, &nv->qt[i].sub1); 1556 fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl); 1557 } 1558 1559 fbnic_napi_free_irq(fbd, nv); 1560 netif_napi_del_locked(&nv->napi); 1561 fbn->napi[fbnic_napi_idx(nv)] = NULL; 1562 kfree(nv); 1563 } 1564 1565 void fbnic_free_napi_vectors(struct fbnic_net *fbn) 1566 { 1567 int i; 1568 1569 for (i = 0; i < fbn->num_napi; i++) 1570 if (fbn->napi[i]) 1571 fbnic_free_napi_vector(fbn, fbn->napi[i]); 1572 } 1573 1574 static int 1575 fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, 1576 unsigned int rxq_idx) 1577 { 1578 struct page_pool_params pp_params = { 1579 .order = 0, 1580 .flags = PP_FLAG_DMA_MAP | 1581 PP_FLAG_DMA_SYNC_DEV, 1582 .pool_size = fbn->hpq_size + fbn->ppq_size, 1583 .nid = NUMA_NO_NODE, 1584 .dev = fbn->netdev->dev.parent, 1585 .dma_dir = DMA_BIDIRECTIONAL, 1586 .offset = 0, 1587 .max_len = PAGE_SIZE, 1588 .netdev = fbn->netdev, 1589 .queue_idx = rxq_idx, 1590 }; 1591 struct page_pool *pp; 1592 1593 /* Page pool cannot exceed a size of 32768. This doesn't limit the 1594 * pages on the ring but the number we can have cached waiting on 1595 * the next use. 1596 * 1597 * TBD: Can this be reduced further? Would a multiple of 1598 * NAPI_POLL_WEIGHT possibly make more sense? The question is how 1599 * may pages do we need to hold in reserve to get the best return 1600 * without hogging too much system memory. 1601 */ 1602 if (pp_params.pool_size > 32768) 1603 pp_params.pool_size = 32768; 1604 1605 pp = page_pool_create(&pp_params); 1606 if (IS_ERR(pp)) 1607 return PTR_ERR(pp); 1608 1609 qt->sub0.page_pool = pp; 1610 if (netif_rxq_has_unreadable_mp(fbn->netdev, rxq_idx)) { 1611 pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; 1612 pp_params.dma_dir = DMA_FROM_DEVICE; 1613 1614 pp = page_pool_create(&pp_params); 1615 if (IS_ERR(pp)) 1616 goto err_destroy_sub0; 1617 } else { 1618 page_pool_get(pp); 1619 } 1620 qt->sub1.page_pool = pp; 1621 1622 return 0; 1623 1624 err_destroy_sub0: 1625 page_pool_destroy(pp); 1626 return PTR_ERR(pp); 1627 } 1628 1629 static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, 1630 int q_idx, u8 flags) 1631 { 1632 u64_stats_init(&ring->stats.syncp); 1633 ring->doorbell = doorbell; 1634 ring->q_idx = q_idx; 1635 ring->flags = flags; 1636 ring->deferred_head = -1; 1637 } 1638 1639 static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, 1640 unsigned int v_count, unsigned int v_idx, 1641 unsigned int txq_count, unsigned int txq_idx, 1642 unsigned int rxq_count, unsigned int rxq_idx) 1643 { 1644 int txt_count = txq_count, rxt_count = rxq_count; 1645 u32 __iomem *uc_addr = fbd->uc_addr0; 1646 int xdp_count = 0, qt_count, err; 1647 struct fbnic_napi_vector *nv; 1648 struct fbnic_q_triad *qt; 1649 u32 __iomem *db; 1650 1651 /* We need to reserve at least one Tx Queue Triad for an XDP ring */ 1652 if (rxq_count) { 1653 xdp_count = 1; 1654 if (!txt_count) 1655 txt_count = 1; 1656 } 1657 1658 qt_count = txt_count + rxq_count; 1659 if (!qt_count) 1660 return -EINVAL; 1661 1662 /* If MMIO has already failed there are no rings to initialize */ 1663 if (!uc_addr) 1664 return -EIO; 1665 1666 /* Allocate NAPI vector and queue triads */ 1667 nv = kzalloc_flex(*nv, qt, qt_count); 1668 if (!nv) 1669 return -ENOMEM; 1670 1671 /* Record queue triad counts */ 1672 nv->txt_count = txt_count; 1673 nv->rxt_count = rxt_count; 1674 1675 /* Provide pointer back to fbnic and MSI-X vectors */ 1676 nv->fbd = fbd; 1677 nv->v_idx = v_idx; 1678 1679 /* Tie napi to netdev */ 1680 fbn->napi[fbnic_napi_idx(nv)] = nv; 1681 netif_napi_add_config_locked(fbn->netdev, &nv->napi, fbnic_poll, 1682 fbnic_napi_idx(nv)); 1683 1684 /* Record IRQ to NAPI struct */ 1685 netif_napi_set_irq_locked(&nv->napi, 1686 pci_irq_vector(to_pci_dev(fbd->dev), 1687 nv->v_idx)); 1688 1689 /* Tie nv back to PCIe dev */ 1690 nv->dev = fbd->dev; 1691 1692 /* Request the IRQ for napi vector */ 1693 err = fbnic_napi_request_irq(fbd, nv); 1694 if (err) 1695 goto napi_del; 1696 1697 /* Initialize queue triads */ 1698 qt = nv->qt; 1699 1700 while (txt_count) { 1701 u8 flags = FBNIC_RING_F_CTX | FBNIC_RING_F_STATS; 1702 1703 /* Configure Tx queue */ 1704 db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ0_TAIL]; 1705 1706 /* Assign Tx queue to netdev if applicable */ 1707 if (txq_count > 0) { 1708 1709 fbnic_ring_init(&qt->sub0, db, txq_idx, flags); 1710 fbn->tx[txq_idx] = &qt->sub0; 1711 txq_count--; 1712 } else { 1713 fbnic_ring_init(&qt->sub0, db, 0, 1714 FBNIC_RING_F_DISABLED); 1715 } 1716 1717 /* Configure XDP queue */ 1718 db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ1_TAIL]; 1719 1720 /* Assign XDP queue to netdev if applicable 1721 * 1722 * The setup for this is in itself a bit different. 1723 * 1. We only need one XDP Tx queue per NAPI vector. 1724 * 2. We associate it to the first Rx queue index. 1725 * 3. The hardware side is associated based on the Tx Queue. 1726 * 4. The netdev queue is offset by FBNIC_MAX_TXQs. 1727 */ 1728 if (xdp_count > 0) { 1729 unsigned int xdp_idx = FBNIC_MAX_TXQS + rxq_idx; 1730 1731 fbnic_ring_init(&qt->sub1, db, xdp_idx, flags); 1732 fbn->tx[xdp_idx] = &qt->sub1; 1733 xdp_count--; 1734 } else { 1735 fbnic_ring_init(&qt->sub1, db, 0, 1736 FBNIC_RING_F_DISABLED); 1737 } 1738 1739 /* Configure Tx completion queue */ 1740 db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TCQ_HEAD]; 1741 fbnic_ring_init(&qt->cmpl, db, 0, 0); 1742 1743 /* Update Tx queue index */ 1744 txt_count--; 1745 txq_idx += v_count; 1746 1747 /* Move to next queue triad */ 1748 qt++; 1749 } 1750 1751 while (rxt_count) { 1752 /* Configure header queue */ 1753 db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_HPQ_TAIL]; 1754 fbnic_ring_init(&qt->sub0, db, 0, 1755 FBNIC_RING_F_CTX | FBNIC_RING_F_STATS); 1756 1757 /* Configure payload queue */ 1758 db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_PPQ_TAIL]; 1759 fbnic_ring_init(&qt->sub1, db, 0, 1760 FBNIC_RING_F_CTX | FBNIC_RING_F_STATS); 1761 1762 /* Configure Rx completion queue */ 1763 db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_RCQ_HEAD]; 1764 fbnic_ring_init(&qt->cmpl, db, rxq_idx, FBNIC_RING_F_STATS); 1765 fbn->rx[rxq_idx] = &qt->cmpl; 1766 1767 /* Update Rx queue index */ 1768 rxt_count--; 1769 rxq_idx += v_count; 1770 1771 /* Move to next queue triad */ 1772 qt++; 1773 } 1774 1775 return 0; 1776 1777 napi_del: 1778 netif_napi_del_locked(&nv->napi); 1779 fbn->napi[fbnic_napi_idx(nv)] = NULL; 1780 kfree(nv); 1781 return err; 1782 } 1783 1784 int fbnic_alloc_napi_vectors(struct fbnic_net *fbn) 1785 { 1786 unsigned int txq_idx = 0, rxq_idx = 0, v_idx = FBNIC_NON_NAPI_VECTORS; 1787 unsigned int num_tx = fbn->num_tx_queues; 1788 unsigned int num_rx = fbn->num_rx_queues; 1789 unsigned int num_napi = fbn->num_napi; 1790 struct fbnic_dev *fbd = fbn->fbd; 1791 int err; 1792 1793 /* Allocate 1 Tx queue per napi vector */ 1794 if (num_napi < FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { 1795 while (num_tx) { 1796 err = fbnic_alloc_napi_vector(fbd, fbn, 1797 num_napi, v_idx, 1798 1, txq_idx, 0, 0); 1799 if (err) 1800 goto free_vectors; 1801 1802 /* Update counts and index */ 1803 num_tx--; 1804 txq_idx++; 1805 1806 v_idx++; 1807 } 1808 } 1809 1810 /* Allocate Tx/Rx queue pairs per vector, or allocate remaining Rx */ 1811 while (num_rx | num_tx) { 1812 int tqpv = DIV_ROUND_UP(num_tx, num_napi - txq_idx); 1813 int rqpv = DIV_ROUND_UP(num_rx, num_napi - rxq_idx); 1814 1815 err = fbnic_alloc_napi_vector(fbd, fbn, num_napi, v_idx, 1816 tqpv, txq_idx, rqpv, rxq_idx); 1817 if (err) 1818 goto free_vectors; 1819 1820 /* Update counts and index */ 1821 num_tx -= tqpv; 1822 txq_idx++; 1823 1824 num_rx -= rqpv; 1825 rxq_idx++; 1826 1827 v_idx++; 1828 } 1829 1830 return 0; 1831 1832 free_vectors: 1833 fbnic_free_napi_vectors(fbn); 1834 1835 return err; 1836 } 1837 1838 static void fbnic_free_ring_resources(struct device *dev, 1839 struct fbnic_ring *ring) 1840 { 1841 kvfree(ring->buffer); 1842 ring->buffer = NULL; 1843 1844 /* If size is not set there are no descriptors present */ 1845 if (!ring->size) 1846 return; 1847 1848 dma_free_coherent(dev, ring->size, ring->desc, ring->dma); 1849 ring->size_mask = 0; 1850 ring->size = 0; 1851 } 1852 1853 static int fbnic_alloc_tx_ring_desc(struct fbnic_net *fbn, 1854 struct fbnic_ring *txr) 1855 { 1856 struct device *dev = fbn->netdev->dev.parent; 1857 size_t size; 1858 1859 /* Round size up to nearest 4K */ 1860 size = ALIGN(array_size(sizeof(*txr->desc), fbn->txq_size), 4096); 1861 1862 txr->desc = dma_alloc_coherent(dev, size, &txr->dma, 1863 GFP_KERNEL | __GFP_NOWARN); 1864 if (!txr->desc) 1865 return -ENOMEM; 1866 1867 /* txq_size should be a power of 2, so mask is just that -1 */ 1868 txr->size_mask = fbn->txq_size - 1; 1869 txr->size = size; 1870 1871 return 0; 1872 } 1873 1874 static int fbnic_alloc_tx_ring_buffer(struct fbnic_ring *txr) 1875 { 1876 size_t size = array_size(sizeof(*txr->tx_buf), txr->size_mask + 1); 1877 1878 txr->tx_buf = kvzalloc(size, GFP_KERNEL | __GFP_NOWARN); 1879 1880 return txr->tx_buf ? 0 : -ENOMEM; 1881 } 1882 1883 static int fbnic_alloc_tx_ring_resources(struct fbnic_net *fbn, 1884 struct fbnic_ring *txr) 1885 { 1886 struct device *dev = fbn->netdev->dev.parent; 1887 int err; 1888 1889 if (txr->flags & FBNIC_RING_F_DISABLED) 1890 return 0; 1891 1892 err = fbnic_alloc_tx_ring_desc(fbn, txr); 1893 if (err) 1894 return err; 1895 1896 if (!(txr->flags & FBNIC_RING_F_CTX)) 1897 return 0; 1898 1899 err = fbnic_alloc_tx_ring_buffer(txr); 1900 if (err) 1901 goto free_desc; 1902 1903 return 0; 1904 1905 free_desc: 1906 fbnic_free_ring_resources(dev, txr); 1907 return err; 1908 } 1909 1910 static int fbnic_alloc_rx_ring_desc(struct fbnic_net *fbn, 1911 struct fbnic_ring *rxr) 1912 { 1913 struct device *dev = fbn->netdev->dev.parent; 1914 size_t desc_size = sizeof(*rxr->desc); 1915 u32 rxq_size; 1916 size_t size; 1917 1918 switch (rxr->doorbell - fbnic_ring_csr_base(rxr)) { 1919 case FBNIC_QUEUE_BDQ_HPQ_TAIL: 1920 rxq_size = fbn->hpq_size / FBNIC_BD_FRAG_COUNT; 1921 desc_size *= FBNIC_BD_FRAG_COUNT; 1922 break; 1923 case FBNIC_QUEUE_BDQ_PPQ_TAIL: 1924 rxq_size = fbn->ppq_size / FBNIC_BD_FRAG_COUNT; 1925 desc_size *= FBNIC_BD_FRAG_COUNT; 1926 break; 1927 case FBNIC_QUEUE_RCQ_HEAD: 1928 rxq_size = fbn->rcq_size; 1929 break; 1930 default: 1931 return -EINVAL; 1932 } 1933 1934 /* Round size up to nearest 4K */ 1935 size = ALIGN(array_size(desc_size, rxq_size), 4096); 1936 1937 rxr->desc = dma_alloc_coherent(dev, size, &rxr->dma, 1938 GFP_KERNEL | __GFP_NOWARN); 1939 if (!rxr->desc) 1940 return -ENOMEM; 1941 1942 /* rxq_size should be a power of 2, so mask is just that -1 */ 1943 rxr->size_mask = rxq_size - 1; 1944 rxr->size = size; 1945 1946 return 0; 1947 } 1948 1949 static int fbnic_alloc_rx_ring_buffer(struct fbnic_ring *rxr) 1950 { 1951 size_t size = array_size(sizeof(*rxr->rx_buf), rxr->size_mask + 1); 1952 1953 if (rxr->flags & FBNIC_RING_F_CTX) 1954 size = sizeof(*rxr->rx_buf) * (rxr->size_mask + 1); 1955 else 1956 size = sizeof(*rxr->pkt); 1957 1958 rxr->rx_buf = kvzalloc(size, GFP_KERNEL | __GFP_NOWARN); 1959 1960 return rxr->rx_buf ? 0 : -ENOMEM; 1961 } 1962 1963 static int fbnic_alloc_rx_ring_resources(struct fbnic_net *fbn, 1964 struct fbnic_ring *rxr) 1965 { 1966 struct device *dev = fbn->netdev->dev.parent; 1967 int err; 1968 1969 err = fbnic_alloc_rx_ring_desc(fbn, rxr); 1970 if (err) 1971 return err; 1972 1973 err = fbnic_alloc_rx_ring_buffer(rxr); 1974 if (err) 1975 goto free_desc; 1976 1977 return 0; 1978 1979 free_desc: 1980 fbnic_free_ring_resources(dev, rxr); 1981 return err; 1982 } 1983 1984 static void fbnic_free_qt_resources(struct fbnic_net *fbn, 1985 struct fbnic_q_triad *qt) 1986 { 1987 struct device *dev = fbn->netdev->dev.parent; 1988 1989 fbnic_free_ring_resources(dev, &qt->cmpl); 1990 fbnic_free_ring_resources(dev, &qt->sub1); 1991 fbnic_free_ring_resources(dev, &qt->sub0); 1992 1993 if (xdp_rxq_info_is_reg(&qt->xdp_rxq)) { 1994 xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); 1995 xdp_rxq_info_unreg(&qt->xdp_rxq); 1996 fbnic_free_qt_page_pools(qt); 1997 } 1998 } 1999 2000 static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn, 2001 struct fbnic_q_triad *qt) 2002 { 2003 struct device *dev = fbn->netdev->dev.parent; 2004 int err; 2005 2006 err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub0); 2007 if (err) 2008 return err; 2009 2010 err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub1); 2011 if (err) 2012 goto free_sub0; 2013 2014 err = fbnic_alloc_tx_ring_resources(fbn, &qt->cmpl); 2015 if (err) 2016 goto free_sub1; 2017 2018 return 0; 2019 2020 free_sub1: 2021 fbnic_free_ring_resources(dev, &qt->sub1); 2022 free_sub0: 2023 fbnic_free_ring_resources(dev, &qt->sub0); 2024 return err; 2025 } 2026 2027 static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn, 2028 struct fbnic_napi_vector *nv, 2029 struct fbnic_q_triad *qt) 2030 { 2031 struct device *dev = fbn->netdev->dev.parent; 2032 int err; 2033 2034 err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx); 2035 if (err) 2036 return err; 2037 2038 err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, qt->sub0.q_idx, 2039 nv->napi.napi_id); 2040 if (err) 2041 goto free_page_pools; 2042 2043 err = xdp_rxq_info_reg_mem_model(&qt->xdp_rxq, MEM_TYPE_PAGE_POOL, 2044 qt->sub0.page_pool); 2045 if (err) 2046 goto unreg_rxq; 2047 2048 err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0); 2049 if (err) 2050 goto unreg_mm; 2051 2052 err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub1); 2053 if (err) 2054 goto free_sub0; 2055 2056 err = fbnic_alloc_rx_ring_resources(fbn, &qt->cmpl); 2057 if (err) 2058 goto free_sub1; 2059 2060 return 0; 2061 2062 free_sub1: 2063 fbnic_free_ring_resources(dev, &qt->sub1); 2064 free_sub0: 2065 fbnic_free_ring_resources(dev, &qt->sub0); 2066 unreg_mm: 2067 xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); 2068 unreg_rxq: 2069 xdp_rxq_info_unreg(&qt->xdp_rxq); 2070 free_page_pools: 2071 fbnic_free_qt_page_pools(qt); 2072 return err; 2073 } 2074 2075 static void fbnic_free_nv_resources(struct fbnic_net *fbn, 2076 struct fbnic_napi_vector *nv) 2077 { 2078 int i; 2079 2080 for (i = 0; i < nv->txt_count + nv->rxt_count; i++) 2081 fbnic_free_qt_resources(fbn, &nv->qt[i]); 2082 } 2083 2084 static int fbnic_alloc_nv_resources(struct fbnic_net *fbn, 2085 struct fbnic_napi_vector *nv) 2086 { 2087 int i, j, err; 2088 2089 /* Allocate Tx Resources */ 2090 for (i = 0; i < nv->txt_count; i++) { 2091 err = fbnic_alloc_tx_qt_resources(fbn, &nv->qt[i]); 2092 if (err) 2093 goto free_qt_resources; 2094 } 2095 2096 /* Allocate Rx Resources */ 2097 for (j = 0; j < nv->rxt_count; j++, i++) { 2098 err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i]); 2099 if (err) 2100 goto free_qt_resources; 2101 } 2102 2103 return 0; 2104 2105 free_qt_resources: 2106 while (i--) 2107 fbnic_free_qt_resources(fbn, &nv->qt[i]); 2108 return err; 2109 } 2110 2111 void fbnic_free_resources(struct fbnic_net *fbn) 2112 { 2113 int i; 2114 2115 for (i = 0; i < fbn->num_napi; i++) 2116 fbnic_free_nv_resources(fbn, fbn->napi[i]); 2117 } 2118 2119 int fbnic_alloc_resources(struct fbnic_net *fbn) 2120 { 2121 int i, err = -ENODEV; 2122 2123 for (i = 0; i < fbn->num_napi; i++) { 2124 err = fbnic_alloc_nv_resources(fbn, fbn->napi[i]); 2125 if (err) 2126 goto free_resources; 2127 } 2128 2129 return 0; 2130 2131 free_resources: 2132 while (i--) 2133 fbnic_free_nv_resources(fbn, fbn->napi[i]); 2134 2135 return err; 2136 } 2137 2138 static void fbnic_set_netif_napi(struct fbnic_napi_vector *nv) 2139 { 2140 int i, j; 2141 2142 /* Associate Tx queue with NAPI */ 2143 for (i = 0; i < nv->txt_count; i++) { 2144 struct fbnic_q_triad *qt = &nv->qt[i]; 2145 2146 netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, 2147 NETDEV_QUEUE_TYPE_TX, &nv->napi); 2148 } 2149 2150 /* Associate Rx queue with NAPI */ 2151 for (j = 0; j < nv->rxt_count; j++, i++) { 2152 struct fbnic_q_triad *qt = &nv->qt[i]; 2153 2154 netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, 2155 NETDEV_QUEUE_TYPE_RX, &nv->napi); 2156 } 2157 } 2158 2159 static void fbnic_reset_netif_napi(struct fbnic_napi_vector *nv) 2160 { 2161 int i, j; 2162 2163 /* Disassociate Tx queue from NAPI */ 2164 for (i = 0; i < nv->txt_count; i++) { 2165 struct fbnic_q_triad *qt = &nv->qt[i]; 2166 2167 netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, 2168 NETDEV_QUEUE_TYPE_TX, NULL); 2169 } 2170 2171 /* Disassociate Rx queue from NAPI */ 2172 for (j = 0; j < nv->rxt_count; j++, i++) { 2173 struct fbnic_q_triad *qt = &nv->qt[i]; 2174 2175 netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, 2176 NETDEV_QUEUE_TYPE_RX, NULL); 2177 } 2178 } 2179 2180 int fbnic_set_netif_queues(struct fbnic_net *fbn) 2181 { 2182 int i, err; 2183 2184 err = netif_set_real_num_queues(fbn->netdev, fbn->num_tx_queues, 2185 fbn->num_rx_queues); 2186 if (err) 2187 return err; 2188 2189 for (i = 0; i < fbn->num_napi; i++) 2190 fbnic_set_netif_napi(fbn->napi[i]); 2191 2192 return 0; 2193 } 2194 2195 void fbnic_reset_netif_queues(struct fbnic_net *fbn) 2196 { 2197 int i; 2198 2199 for (i = 0; i < fbn->num_napi; i++) 2200 fbnic_reset_netif_napi(fbn->napi[i]); 2201 } 2202 2203 static void fbnic_disable_twq0(struct fbnic_ring *txr) 2204 { 2205 u32 twq_ctl = fbnic_ring_rd32(txr, FBNIC_QUEUE_TWQ0_CTL); 2206 2207 twq_ctl &= ~FBNIC_QUEUE_TWQ_CTL_ENABLE; 2208 2209 fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ0_CTL, twq_ctl); 2210 } 2211 2212 static void fbnic_disable_twq1(struct fbnic_ring *txr) 2213 { 2214 u32 twq_ctl = fbnic_ring_rd32(txr, FBNIC_QUEUE_TWQ1_CTL); 2215 2216 twq_ctl &= ~FBNIC_QUEUE_TWQ_CTL_ENABLE; 2217 2218 fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ1_CTL, twq_ctl); 2219 } 2220 2221 static void fbnic_disable_tcq(struct fbnic_ring *txr) 2222 { 2223 fbnic_ring_wr32(txr, FBNIC_QUEUE_TCQ_CTL, 0); 2224 fbnic_ring_wr32(txr, FBNIC_QUEUE_TIM_MASK, FBNIC_QUEUE_TIM_MASK_MASK); 2225 } 2226 2227 static void fbnic_disable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) 2228 { 2229 u32 bdq_ctl = fbnic_ring_rd32(hpq, FBNIC_QUEUE_BDQ_CTL); 2230 2231 bdq_ctl &= ~FBNIC_QUEUE_BDQ_CTL_ENABLE; 2232 2233 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, bdq_ctl); 2234 } 2235 2236 static void fbnic_disable_rcq(struct fbnic_ring *rxr) 2237 { 2238 fbnic_ring_wr32(rxr, FBNIC_QUEUE_RCQ_CTL, 0); 2239 fbnic_ring_wr32(rxr, FBNIC_QUEUE_RIM_MASK, FBNIC_QUEUE_RIM_MASK_MASK); 2240 } 2241 2242 void fbnic_napi_disable(struct fbnic_net *fbn) 2243 { 2244 int i; 2245 2246 for (i = 0; i < fbn->num_napi; i++) { 2247 napi_disable_locked(&fbn->napi[i]->napi); 2248 2249 fbnic_nv_irq_disable(fbn->napi[i]); 2250 } 2251 } 2252 2253 static void __fbnic_nv_disable(struct fbnic_napi_vector *nv) 2254 { 2255 int i, t; 2256 2257 /* Disable Tx queue triads */ 2258 for (t = 0; t < nv->txt_count; t++) { 2259 struct fbnic_q_triad *qt = &nv->qt[t]; 2260 2261 fbnic_disable_twq0(&qt->sub0); 2262 fbnic_disable_twq1(&qt->sub1); 2263 fbnic_disable_tcq(&qt->cmpl); 2264 } 2265 2266 /* Disable Rx queue triads */ 2267 for (i = 0; i < nv->rxt_count; i++, t++) { 2268 struct fbnic_q_triad *qt = &nv->qt[t]; 2269 2270 fbnic_disable_bdq(&qt->sub0, &qt->sub1); 2271 fbnic_disable_rcq(&qt->cmpl); 2272 } 2273 } 2274 2275 static void 2276 fbnic_nv_disable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) 2277 { 2278 __fbnic_nv_disable(nv); 2279 fbnic_wrfl(fbn->fbd); 2280 } 2281 2282 void fbnic_dbg_down(struct fbnic_net *fbn) 2283 { 2284 int i; 2285 2286 for (i = 0; i < fbn->num_napi; i++) 2287 fbnic_dbg_nv_exit(fbn->napi[i]); 2288 } 2289 2290 void fbnic_dbg_up(struct fbnic_net *fbn) 2291 { 2292 int i; 2293 2294 for (i = 0; i < fbn->num_napi; i++) 2295 fbnic_dbg_nv_init(fbn->napi[i]); 2296 } 2297 2298 void fbnic_disable(struct fbnic_net *fbn) 2299 { 2300 struct fbnic_dev *fbd = fbn->fbd; 2301 int i; 2302 2303 for (i = 0; i < fbn->num_napi; i++) 2304 __fbnic_nv_disable(fbn->napi[i]); 2305 2306 fbnic_wrfl(fbd); 2307 } 2308 2309 static void fbnic_tx_flush(struct fbnic_dev *fbd) 2310 { 2311 netdev_warn(fbd->netdev, "triggering Tx flush\n"); 2312 2313 fbnic_rmw32(fbd, FBNIC_TMI_DROP_CTRL, FBNIC_TMI_DROP_CTRL_EN, 2314 FBNIC_TMI_DROP_CTRL_EN); 2315 } 2316 2317 static void fbnic_tx_flush_off(struct fbnic_dev *fbd) 2318 { 2319 fbnic_rmw32(fbd, FBNIC_TMI_DROP_CTRL, FBNIC_TMI_DROP_CTRL_EN, 0); 2320 } 2321 2322 struct fbnic_idle_regs { 2323 u32 reg_base; 2324 u8 reg_cnt; 2325 }; 2326 2327 static bool fbnic_all_idle(struct fbnic_dev *fbd, 2328 const struct fbnic_idle_regs *regs, 2329 unsigned int nregs) 2330 { 2331 unsigned int i, j; 2332 2333 for (i = 0; i < nregs; i++) { 2334 for (j = 0; j < regs[i].reg_cnt; j++) { 2335 if (fbnic_rd32(fbd, regs[i].reg_base + j) != ~0U) 2336 return false; 2337 } 2338 } 2339 return true; 2340 } 2341 2342 static void fbnic_idle_dump(struct fbnic_dev *fbd, 2343 const struct fbnic_idle_regs *regs, 2344 unsigned int nregs, const char *dir, int err) 2345 { 2346 unsigned int i, j; 2347 2348 netdev_err(fbd->netdev, "error waiting for %s idle %d\n", dir, err); 2349 for (i = 0; i < nregs; i++) 2350 for (j = 0; j < regs[i].reg_cnt; j++) 2351 netdev_err(fbd->netdev, "0x%04x: %08x\n", 2352 regs[i].reg_base + j, 2353 fbnic_rd32(fbd, regs[i].reg_base + j)); 2354 } 2355 2356 int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail) 2357 { 2358 static const struct fbnic_idle_regs tx[] = { 2359 { FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TWQ_IDLE_CNT, }, 2360 { FBNIC_QM_TQS_IDLE(0), FBNIC_QM_TQS_IDLE_CNT, }, 2361 { FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TDE_IDLE_CNT, }, 2362 { FBNIC_QM_TCQ_IDLE(0), FBNIC_QM_TCQ_IDLE_CNT, }, 2363 }, rx[] = { 2364 { FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_HPQ_IDLE_CNT, }, 2365 { FBNIC_QM_PPQ_IDLE(0), FBNIC_QM_PPQ_IDLE_CNT, }, 2366 { FBNIC_QM_RCQ_IDLE(0), FBNIC_QM_RCQ_IDLE_CNT, }, 2367 }; 2368 bool idle; 2369 int err; 2370 2371 err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2, 500000, 2372 false, fbd, tx, ARRAY_SIZE(tx)); 2373 if (err == -ETIMEDOUT) { 2374 fbnic_tx_flush(fbd); 2375 err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2376 2, 500000, false, 2377 fbd, tx, ARRAY_SIZE(tx)); 2378 fbnic_tx_flush_off(fbd); 2379 } 2380 if (err) { 2381 fbnic_idle_dump(fbd, tx, ARRAY_SIZE(tx), "Tx", err); 2382 if (may_fail) 2383 return err; 2384 } 2385 2386 err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2, 500000, 2387 false, fbd, rx, ARRAY_SIZE(rx)); 2388 if (err) 2389 fbnic_idle_dump(fbd, rx, ARRAY_SIZE(rx), "Rx", err); 2390 return err; 2391 } 2392 2393 static int 2394 fbnic_wait_queue_idle(struct fbnic_net *fbn, bool rx, unsigned int idx) 2395 { 2396 static const unsigned int tx_regs[] = { 2397 FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TQS_IDLE(0), 2398 FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TCQ_IDLE(0), 2399 }, rx_regs[] = { 2400 FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_PPQ_IDLE(0), 2401 FBNIC_QM_RCQ_IDLE(0), 2402 }; 2403 struct fbnic_dev *fbd = fbn->fbd; 2404 unsigned int val, mask, off; 2405 const unsigned int *regs; 2406 unsigned int reg_cnt; 2407 int i, err; 2408 2409 regs = rx ? rx_regs : tx_regs; 2410 reg_cnt = rx ? ARRAY_SIZE(rx_regs) : ARRAY_SIZE(tx_regs); 2411 2412 off = idx / 32; 2413 mask = BIT(idx % 32); 2414 2415 for (i = 0; i < reg_cnt; i++) { 2416 err = read_poll_timeout_atomic(fbnic_rd32, val, val & mask, 2417 2, 500000, false, 2418 fbd, regs[i] + off); 2419 if (err) { 2420 netdev_err(fbd->netdev, 2421 "wait for queue %s%d idle failed 0x%04x(%d): %08x (mask: %08x)\n", 2422 rx ? "Rx" : "Tx", idx, regs[i] + off, i, 2423 val, mask); 2424 return err; 2425 } 2426 } 2427 2428 return 0; 2429 } 2430 2431 static void fbnic_nv_flush(struct fbnic_napi_vector *nv) 2432 { 2433 int j, t; 2434 2435 /* Flush any processed Tx Queue Triads and drop the rest */ 2436 for (t = 0; t < nv->txt_count; t++) { 2437 struct fbnic_q_triad *qt = &nv->qt[t]; 2438 struct netdev_queue *tx_queue; 2439 2440 /* Clean the work queues of unprocessed work */ 2441 fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); 2442 fbnic_clean_twq1(nv, false, &qt->sub1, true, 2443 qt->sub1.tail); 2444 2445 /* Reset completion queue descriptor ring */ 2446 memset(qt->cmpl.desc, 0, qt->cmpl.size); 2447 2448 /* Nothing else to do if Tx queue is disabled */ 2449 if (qt->sub0.flags & FBNIC_RING_F_DISABLED) 2450 continue; 2451 2452 /* Reset BQL associated with Tx queue */ 2453 tx_queue = netdev_get_tx_queue(nv->napi.dev, 2454 qt->sub0.q_idx); 2455 netdev_tx_reset_queue(tx_queue); 2456 } 2457 2458 /* Flush any processed Rx Queue Triads and drop the rest */ 2459 for (j = 0; j < nv->rxt_count; j++, t++) { 2460 struct fbnic_q_triad *qt = &nv->qt[t]; 2461 2462 /* Clean the work queues of unprocessed work */ 2463 fbnic_clean_bdq(&qt->sub0, qt->sub0.tail, 0); 2464 fbnic_clean_bdq(&qt->sub1, qt->sub1.tail, 0); 2465 2466 /* Reset completion queue descriptor ring */ 2467 memset(qt->cmpl.desc, 0, qt->cmpl.size); 2468 2469 fbnic_put_pkt_buff(qt, qt->cmpl.pkt, 0); 2470 memset(qt->cmpl.pkt, 0, sizeof(struct fbnic_pkt_buff)); 2471 } 2472 } 2473 2474 void fbnic_flush(struct fbnic_net *fbn) 2475 { 2476 int i; 2477 2478 for (i = 0; i < fbn->num_napi; i++) 2479 fbnic_nv_flush(fbn->napi[i]); 2480 } 2481 2482 static void fbnic_nv_fill(struct fbnic_napi_vector *nv) 2483 { 2484 int j, t; 2485 2486 /* Configure NAPI mapping and populate pages 2487 * in the BDQ rings to use for Rx 2488 */ 2489 for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { 2490 struct fbnic_q_triad *qt = &nv->qt[t]; 2491 2492 /* Populate the header and payload BDQs */ 2493 fbnic_fill_bdq(&qt->sub0); 2494 fbnic_fill_bdq(&qt->sub1); 2495 } 2496 } 2497 2498 void fbnic_fill(struct fbnic_net *fbn) 2499 { 2500 int i; 2501 2502 for (i = 0; i < fbn->num_napi; i++) 2503 fbnic_nv_fill(fbn->napi[i]); 2504 } 2505 2506 static void fbnic_enable_twq0(struct fbnic_ring *twq) 2507 { 2508 u32 log_size = fls(twq->size_mask); 2509 2510 if (!twq->size_mask) 2511 return; 2512 2513 /* Reset head/tail */ 2514 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); 2515 twq->tail = 0; 2516 twq->head = 0; 2517 twq->deferred_meta = -1; 2518 2519 /* Store descriptor ring address and size */ 2520 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAL, lower_32_bits(twq->dma)); 2521 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAH, upper_32_bits(twq->dma)); 2522 2523 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2524 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_SIZE, log_size & 0xf); 2525 2526 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); 2527 } 2528 2529 static void fbnic_enable_twq1(struct fbnic_ring *twq) 2530 { 2531 u32 log_size = fls(twq->size_mask); 2532 2533 if (!twq->size_mask) 2534 return; 2535 2536 /* Reset head/tail */ 2537 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); 2538 twq->tail = 0; 2539 twq->head = 0; 2540 2541 /* Store descriptor ring address and size */ 2542 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_BAL, lower_32_bits(twq->dma)); 2543 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_BAH, upper_32_bits(twq->dma)); 2544 2545 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2546 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_SIZE, log_size & 0xf); 2547 2548 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); 2549 } 2550 2551 static void fbnic_enable_tcq(struct fbnic_napi_vector *nv, 2552 struct fbnic_ring *tcq) 2553 { 2554 u32 log_size = fls(tcq->size_mask); 2555 2556 if (!tcq->size_mask) 2557 return; 2558 2559 /* Reset head/tail */ 2560 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_RESET); 2561 tcq->tail = 0; 2562 tcq->head = 0; 2563 2564 /* Store descriptor ring address and size */ 2565 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_BAL, lower_32_bits(tcq->dma)); 2566 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_BAH, upper_32_bits(tcq->dma)); 2567 2568 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2569 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_SIZE, log_size & 0xf); 2570 2571 /* Store interrupt information for the completion queue */ 2572 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_CTL, nv->v_idx); 2573 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_THRESHOLD, tcq->size_mask / 2); 2574 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_MASK, 0); 2575 2576 /* Enable queue */ 2577 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_ENABLE); 2578 } 2579 2580 static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) 2581 { 2582 u32 bdq_ctl = FBNIC_QUEUE_BDQ_CTL_ENABLE; 2583 u32 log_size; 2584 2585 /* Reset head/tail */ 2586 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, FBNIC_QUEUE_BDQ_CTL_RESET); 2587 ppq->tail = 0; 2588 ppq->head = 0; 2589 hpq->tail = 0; 2590 hpq->head = 0; 2591 2592 log_size = fls(hpq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); 2593 2594 /* Store descriptor ring address and size */ 2595 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAL, lower_32_bits(hpq->dma)); 2596 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAH, upper_32_bits(hpq->dma)); 2597 2598 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2599 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_SIZE, log_size & 0xf); 2600 2601 if (!ppq->size_mask) 2602 goto write_ctl; 2603 2604 log_size = fls(ppq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); 2605 2606 /* Add enabling of PPQ to BDQ control */ 2607 bdq_ctl |= FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE; 2608 2609 /* Store descriptor ring address and size */ 2610 fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_BAL, lower_32_bits(ppq->dma)); 2611 fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_BAH, upper_32_bits(ppq->dma)); 2612 fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_SIZE, log_size & 0xf); 2613 2614 write_ctl: 2615 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, bdq_ctl); 2616 } 2617 2618 static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, 2619 struct fbnic_ring *rcq, bool tx_pause, 2620 bool hdr_split) 2621 { 2622 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 2623 u32 drop_mode, rcq_ctl; 2624 2625 if (!tx_pause && fbn->num_rx_queues > 1) 2626 drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE; 2627 else 2628 drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_NEVER; 2629 2630 /* Specify packet layout */ 2631 rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK, drop_mode) | 2632 FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK, FBNIC_RX_HROOM) | 2633 FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM) | 2634 FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT, hdr_split); 2635 2636 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL0, rcq_ctl); 2637 } 2638 2639 void fbnic_config_drop_mode(struct fbnic_net *fbn, bool txp) 2640 { 2641 bool hds; 2642 int i, t; 2643 2644 hds = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; 2645 2646 for (i = 0; i < fbn->num_napi; i++) { 2647 struct fbnic_napi_vector *nv = fbn->napi[i]; 2648 2649 for (t = 0; t < nv->rxt_count; t++) { 2650 struct fbnic_q_triad *qt = &nv->qt[nv->txt_count + t]; 2651 2652 fbnic_config_drop_mode_rcq(nv, &qt->cmpl, txp, hds); 2653 } 2654 } 2655 } 2656 2657 static void fbnic_config_rim_threshold(struct fbnic_ring *rcq, u16 nv_idx, u32 rx_desc) 2658 { 2659 u32 threshold; 2660 2661 /* Set the threhsold to half the ring size if rx_frames 2662 * is not configured 2663 */ 2664 threshold = rx_desc ? : rcq->size_mask / 2; 2665 2666 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_CTL, nv_idx); 2667 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_THRESHOLD, threshold); 2668 } 2669 2670 void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm) 2671 { 2672 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 2673 struct fbnic_dev *fbd = nv->fbd; 2674 u32 val = arm; 2675 2676 val |= FIELD_PREP(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT, fbn->rx_usecs) | 2677 FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT_UPD_EN; 2678 val |= FIELD_PREP(FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT, fbn->tx_usecs) | 2679 FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT_UPD_EN; 2680 2681 fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(nv->v_idx), val); 2682 } 2683 2684 void fbnic_config_rx_frames(struct fbnic_napi_vector *nv) 2685 { 2686 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 2687 int i; 2688 2689 for (i = nv->txt_count; i < nv->rxt_count + nv->txt_count; i++) { 2690 struct fbnic_q_triad *qt = &nv->qt[i]; 2691 2692 fbnic_config_rim_threshold(&qt->cmpl, nv->v_idx, 2693 fbn->rx_max_frames * 2694 FBNIC_MIN_RXD_PER_FRAME); 2695 } 2696 } 2697 2698 static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, 2699 struct fbnic_ring *rcq) 2700 { 2701 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 2702 u32 log_size = fls(rcq->size_mask); 2703 u32 rcq_ctl = 0; 2704 bool hdr_split; 2705 u32 hds_thresh; 2706 2707 /* Force lower bound on MAX_HEADER_BYTES. Below this, all frames should 2708 * be split at L4. It would also result in the frames being split at 2709 * L2/L3 depending on the frame size. 2710 */ 2711 hdr_split = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; 2712 fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause, hdr_split); 2713 2714 hds_thresh = max(fbn->hds_thresh, FBNIC_HDR_BYTES_MIN); 2715 rcq_ctl |= FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | 2716 FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, hds_thresh) | 2717 FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, 2718 FBNIC_RX_PAYLD_OFFSET) | 2719 FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_PG_CL_MASK, 2720 FBNIC_RX_PAYLD_PG_CL); 2721 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL1, rcq_ctl); 2722 2723 /* Reset head/tail */ 2724 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_RESET); 2725 rcq->head = 0; 2726 rcq->tail = 0; 2727 2728 /* Store descriptor ring address and size */ 2729 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_BAL, lower_32_bits(rcq->dma)); 2730 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_BAH, upper_32_bits(rcq->dma)); 2731 2732 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2733 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_SIZE, log_size & 0xf); 2734 2735 /* Store interrupt information for the completion queue */ 2736 fbnic_config_rim_threshold(rcq, nv->v_idx, fbn->rx_max_frames * 2737 FBNIC_MIN_RXD_PER_FRAME); 2738 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_MASK, 0); 2739 2740 /* Enable queue */ 2741 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_ENABLE); 2742 } 2743 2744 static void __fbnic_nv_enable(struct fbnic_napi_vector *nv) 2745 { 2746 int j, t; 2747 2748 /* Setup Tx Queue Triads */ 2749 for (t = 0; t < nv->txt_count; t++) { 2750 struct fbnic_q_triad *qt = &nv->qt[t]; 2751 2752 fbnic_enable_twq0(&qt->sub0); 2753 fbnic_enable_twq1(&qt->sub1); 2754 fbnic_enable_tcq(nv, &qt->cmpl); 2755 } 2756 2757 /* Setup Rx Queue Triads */ 2758 for (j = 0; j < nv->rxt_count; j++, t++) { 2759 struct fbnic_q_triad *qt = &nv->qt[t]; 2760 2761 page_pool_enable_direct_recycling(qt->sub0.page_pool, 2762 &nv->napi); 2763 page_pool_enable_direct_recycling(qt->sub1.page_pool, 2764 &nv->napi); 2765 2766 fbnic_enable_bdq(&qt->sub0, &qt->sub1); 2767 fbnic_enable_rcq(nv, &qt->cmpl); 2768 } 2769 } 2770 2771 static void fbnic_nv_enable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) 2772 { 2773 __fbnic_nv_enable(nv); 2774 fbnic_wrfl(fbn->fbd); 2775 } 2776 2777 void fbnic_enable(struct fbnic_net *fbn) 2778 { 2779 struct fbnic_dev *fbd = fbn->fbd; 2780 int i; 2781 2782 for (i = 0; i < fbn->num_napi; i++) 2783 __fbnic_nv_enable(fbn->napi[i]); 2784 2785 fbnic_wrfl(fbd); 2786 } 2787 2788 static void fbnic_nv_irq_enable(struct fbnic_napi_vector *nv) 2789 { 2790 fbnic_config_txrx_usecs(nv, FBNIC_INTR_CQ_REARM_INTR_UNMASK); 2791 } 2792 2793 void fbnic_napi_enable(struct fbnic_net *fbn) 2794 { 2795 u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {}; 2796 struct fbnic_dev *fbd = fbn->fbd; 2797 int i; 2798 2799 for (i = 0; i < fbn->num_napi; i++) { 2800 struct fbnic_napi_vector *nv = fbn->napi[i]; 2801 2802 napi_enable_locked(&nv->napi); 2803 2804 fbnic_nv_irq_enable(nv); 2805 2806 /* Record bit used for NAPI IRQs so we can 2807 * set the mask appropriately 2808 */ 2809 irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); 2810 } 2811 2812 /* Force the first interrupt on the device to guarantee 2813 * that any packets that may have been enqueued during the 2814 * bringup are processed. 2815 */ 2816 for (i = 0; i < ARRAY_SIZE(irqs); i++) { 2817 if (!irqs[i]) 2818 continue; 2819 fbnic_wr32(fbd, FBNIC_INTR_SET(i), irqs[i]); 2820 } 2821 2822 fbnic_wrfl(fbd); 2823 } 2824 2825 void fbnic_napi_depletion_check(struct net_device *netdev) 2826 { 2827 struct fbnic_net *fbn = netdev_priv(netdev); 2828 u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {}; 2829 struct fbnic_dev *fbd = fbn->fbd; 2830 int i, j, t; 2831 2832 for (i = 0; i < fbn->num_napi; i++) { 2833 struct fbnic_napi_vector *nv = fbn->napi[i]; 2834 2835 /* Find RQs which are completely out of pages */ 2836 for (t = nv->txt_count, j = 0; j < nv->rxt_count; j++, t++) { 2837 /* Assume 4 pages is always enough to fit a packet 2838 * and therefore generate a completion and an IRQ. 2839 */ 2840 if (fbnic_desc_used(&nv->qt[t].sub0) < 4 || 2841 fbnic_desc_used(&nv->qt[t].sub1) < 4) 2842 irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); 2843 } 2844 } 2845 2846 for (i = 0; i < ARRAY_SIZE(irqs); i++) { 2847 if (!irqs[i]) 2848 continue; 2849 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i), irqs[i]); 2850 fbnic_wr32(fbd, FBNIC_INTR_SET(i), irqs[i]); 2851 } 2852 2853 fbnic_wrfl(fbd); 2854 } 2855 2856 static int fbnic_queue_mem_alloc(struct net_device *dev, 2857 struct netdev_queue_config *qcfg, 2858 void *qmem, int idx) 2859 { 2860 struct fbnic_net *fbn = netdev_priv(dev); 2861 const struct fbnic_q_triad *real; 2862 struct fbnic_q_triad *qt = qmem; 2863 struct fbnic_napi_vector *nv; 2864 2865 if (!netif_running(dev)) 2866 return fbnic_alloc_qt_page_pools(fbn, qt, idx); 2867 2868 real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 2869 nv = fbn->napi[idx % fbn->num_napi]; 2870 2871 fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx, 2872 real->sub0.flags); 2873 fbnic_ring_init(&qt->sub1, real->sub1.doorbell, real->sub1.q_idx, 2874 real->sub1.flags); 2875 fbnic_ring_init(&qt->cmpl, real->cmpl.doorbell, real->cmpl.q_idx, 2876 real->cmpl.flags); 2877 2878 return fbnic_alloc_rx_qt_resources(fbn, nv, qt); 2879 } 2880 2881 static void fbnic_queue_mem_free(struct net_device *dev, void *qmem) 2882 { 2883 struct fbnic_net *fbn = netdev_priv(dev); 2884 struct fbnic_q_triad *qt = qmem; 2885 2886 if (!netif_running(dev)) 2887 fbnic_free_qt_page_pools(qt); 2888 else 2889 fbnic_free_qt_resources(fbn, qt); 2890 } 2891 2892 static void __fbnic_nv_restart(struct fbnic_net *fbn, 2893 struct fbnic_napi_vector *nv) 2894 { 2895 struct fbnic_dev *fbd = fbn->fbd; 2896 int i; 2897 2898 fbnic_nv_enable(fbn, nv); 2899 fbnic_nv_fill(nv); 2900 2901 napi_enable_locked(&nv->napi); 2902 fbnic_nv_irq_enable(nv); 2903 fbnic_wr32(fbd, FBNIC_INTR_SET(nv->v_idx / 32), BIT(nv->v_idx % 32)); 2904 fbnic_wrfl(fbd); 2905 2906 for (i = 0; i < nv->txt_count; i++) 2907 netif_wake_subqueue(fbn->netdev, nv->qt[i].sub0.q_idx); 2908 fbnic_dbg_nv_init(nv); 2909 } 2910 2911 static int fbnic_queue_start(struct net_device *dev, 2912 struct netdev_queue_config *qcfg, 2913 void *qmem, int idx) 2914 { 2915 struct fbnic_net *fbn = netdev_priv(dev); 2916 struct fbnic_napi_vector *nv; 2917 struct fbnic_q_triad *real; 2918 2919 real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 2920 nv = fbn->napi[idx % fbn->num_napi]; 2921 2922 fbnic_aggregate_ring_bdq_counters(fbn, &real->sub0); 2923 fbnic_aggregate_ring_bdq_counters(fbn, &real->sub1); 2924 fbnic_aggregate_ring_rx_counters(fbn, &real->cmpl); 2925 2926 memcpy(real, qmem, sizeof(*real)); 2927 2928 __fbnic_nv_restart(fbn, nv); 2929 2930 return 0; 2931 } 2932 2933 static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx) 2934 { 2935 struct fbnic_net *fbn = netdev_priv(dev); 2936 const struct fbnic_q_triad *real; 2937 struct fbnic_napi_vector *nv; 2938 int i, t; 2939 int err; 2940 2941 real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 2942 nv = fbn->napi[idx % fbn->num_napi]; 2943 fbnic_dbg_nv_exit(nv); 2944 2945 napi_disable_locked(&nv->napi); 2946 fbnic_nv_irq_disable(nv); 2947 2948 for (i = 0; i < nv->txt_count; i++) 2949 netif_stop_subqueue(dev, nv->qt[i].sub0.q_idx); 2950 fbnic_nv_disable(fbn, nv); 2951 2952 for (t = 0; t < nv->txt_count + nv->rxt_count; t++) { 2953 err = fbnic_wait_queue_idle(fbn, t >= nv->txt_count, 2954 nv->qt[t].sub0.q_idx); 2955 if (err) 2956 goto err_restart; 2957 } 2958 2959 fbnic_synchronize_irq(fbn->fbd, nv->v_idx); 2960 fbnic_nv_flush(nv); 2961 2962 page_pool_disable_direct_recycling(real->sub0.page_pool); 2963 page_pool_disable_direct_recycling(real->sub1.page_pool); 2964 2965 memcpy(qmem, real, sizeof(*real)); 2966 2967 return 0; 2968 2969 err_restart: 2970 __fbnic_nv_restart(fbn, nv); 2971 return err; 2972 } 2973 2974 const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops = { 2975 .ndo_queue_mem_size = sizeof(struct fbnic_q_triad), 2976 .ndo_queue_mem_alloc = fbnic_queue_mem_alloc, 2977 .ndo_queue_mem_free = fbnic_queue_mem_free, 2978 .ndo_queue_start = fbnic_queue_start, 2979 .ndo_queue_stop = fbnic_queue_stop, 2980 }; 2981