1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/bpf.h> 6 #include <linux/bpf_trace.h> 7 #include <linux/iopoll.h> 8 #include <linux/pci.h> 9 #include <linux/time64.h> 10 #include <net/netdev_queues.h> 11 #include <net/netdev_rx_queue.h> 12 #include <net/page_pool/helpers.h> 13 #include <net/tcp.h> 14 #include <net/xdp.h> 15 16 #include "fbnic.h" 17 #include "fbnic_csr.h" 18 #include "fbnic_netdev.h" 19 #include "fbnic_txrx.h" 20 21 enum { 22 FBNIC_XDP_PASS = 0, 23 FBNIC_XDP_CONSUME, 24 FBNIC_XDP_TX, 25 FBNIC_XDP_LEN_ERR, 26 }; 27 28 enum { 29 FBNIC_XMIT_CB_TS = 0x01, 30 }; 31 32 struct fbnic_xmit_cb { 33 u32 bytecount; 34 u16 gso_segs; 35 u8 desc_count; 36 u8 flags; 37 int hw_head; 38 }; 39 40 #define FBNIC_XMIT_CB(__skb) ((struct fbnic_xmit_cb *)((__skb)->cb)) 41 42 #define FBNIC_XMIT_NOUNMAP ((void *)1) 43 44 u32 __iomem *fbnic_ring_csr_base(const struct fbnic_ring *ring) 45 { 46 unsigned long csr_base = (unsigned long)ring->doorbell; 47 48 csr_base &= ~(FBNIC_QUEUE_STRIDE * sizeof(u32) - 1); 49 50 return (u32 __iomem *)csr_base; 51 } 52 53 static u32 fbnic_ring_rd32(struct fbnic_ring *ring, unsigned int csr) 54 { 55 u32 __iomem *csr_base = fbnic_ring_csr_base(ring); 56 57 return readl(csr_base + csr); 58 } 59 60 static void fbnic_ring_wr32(struct fbnic_ring *ring, unsigned int csr, u32 val) 61 { 62 u32 __iomem *csr_base = fbnic_ring_csr_base(ring); 63 64 writel(val, csr_base + csr); 65 } 66 67 /** 68 * fbnic_ts40_to_ns() - convert descriptor timestamp to PHC time 69 * @fbn: netdev priv of the FB NIC 70 * @ts40: timestamp read from a descriptor 71 * 72 * Return: u64 value of PHC time in nanoseconds 73 * 74 * Convert truncated 40 bit device timestamp as read from a descriptor 75 * to the full PHC time in nanoseconds. 76 */ 77 static __maybe_unused u64 fbnic_ts40_to_ns(struct fbnic_net *fbn, u64 ts40) 78 { 79 unsigned int s; 80 u64 time_ns; 81 s64 offset; 82 u8 ts_top; 83 u32 high; 84 85 do { 86 s = u64_stats_fetch_begin(&fbn->time_seq); 87 offset = READ_ONCE(fbn->time_offset); 88 } while (u64_stats_fetch_retry(&fbn->time_seq, s)); 89 90 high = READ_ONCE(fbn->time_high); 91 92 /* Bits 63..40 from periodic clock reads, 39..0 from ts40 */ 93 time_ns = (u64)(high >> 8) << 40 | ts40; 94 95 /* Compare bits 32-39 between periodic reads and ts40, 96 * see if HW clock may have wrapped since last read. We are sure 97 * that periodic reads are always at least ~1 minute behind, so 98 * this logic works perfectly fine. 99 */ 100 ts_top = ts40 >> 32; 101 if (ts_top < (u8)high && (u8)high - ts_top > U8_MAX / 2) 102 time_ns += 1ULL << 40; 103 104 return time_ns + offset; 105 } 106 107 static unsigned int fbnic_desc_unused(struct fbnic_ring *ring) 108 { 109 return (ring->head - ring->tail - 1) & ring->size_mask; 110 } 111 112 static unsigned int fbnic_desc_used(struct fbnic_ring *ring) 113 { 114 return (ring->tail - ring->head) & ring->size_mask; 115 } 116 117 static struct netdev_queue *txring_txq(const struct net_device *dev, 118 const struct fbnic_ring *ring) 119 { 120 return netdev_get_tx_queue(dev, ring->q_idx); 121 } 122 123 static int fbnic_maybe_stop_tx(const struct net_device *dev, 124 struct fbnic_ring *ring, 125 const unsigned int size) 126 { 127 struct netdev_queue *txq = txring_txq(dev, ring); 128 int res; 129 130 res = netif_txq_maybe_stop(txq, fbnic_desc_unused(ring), size, 131 FBNIC_TX_DESC_WAKEUP); 132 if (!res) { 133 u64_stats_update_begin(&ring->stats.syncp); 134 ring->stats.twq.stop++; 135 u64_stats_update_end(&ring->stats.syncp); 136 } 137 138 return !res; 139 } 140 141 static bool fbnic_tx_sent_queue(struct sk_buff *skb, struct fbnic_ring *ring) 142 { 143 struct netdev_queue *dev_queue = txring_txq(skb->dev, ring); 144 unsigned int bytecount = FBNIC_XMIT_CB(skb)->bytecount; 145 bool xmit_more = netdev_xmit_more(); 146 147 /* TBD: Request completion more often if xmit_more becomes large */ 148 149 return __netdev_tx_sent_queue(dev_queue, bytecount, xmit_more); 150 } 151 152 static void fbnic_unmap_single_twd(struct device *dev, __le64 *twd) 153 { 154 u64 raw_twd = le64_to_cpu(*twd); 155 unsigned int len; 156 dma_addr_t dma; 157 158 dma = FIELD_GET(FBNIC_TWD_ADDR_MASK, raw_twd); 159 len = FIELD_GET(FBNIC_TWD_LEN_MASK, raw_twd); 160 161 dma_unmap_single(dev, dma, len, DMA_TO_DEVICE); 162 } 163 164 static void fbnic_unmap_page_twd(struct device *dev, __le64 *twd) 165 { 166 u64 raw_twd = le64_to_cpu(*twd); 167 unsigned int len; 168 dma_addr_t dma; 169 170 dma = FIELD_GET(FBNIC_TWD_ADDR_MASK, raw_twd); 171 len = FIELD_GET(FBNIC_TWD_LEN_MASK, raw_twd); 172 173 dma_unmap_page(dev, dma, len, DMA_TO_DEVICE); 174 } 175 176 #define FBNIC_TWD_TYPE(_type) \ 177 cpu_to_le64(FIELD_PREP(FBNIC_TWD_TYPE_MASK, FBNIC_TWD_TYPE_##_type)) 178 179 static bool fbnic_tx_tstamp(struct sk_buff *skb) 180 { 181 struct fbnic_net *fbn; 182 183 if (!unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 184 return false; 185 186 fbn = netdev_priv(skb->dev); 187 if (fbn->hwtstamp_config.tx_type == HWTSTAMP_TX_OFF) 188 return false; 189 190 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 191 FBNIC_XMIT_CB(skb)->flags |= FBNIC_XMIT_CB_TS; 192 FBNIC_XMIT_CB(skb)->hw_head = -1; 193 194 return true; 195 } 196 197 static bool 198 fbnic_tx_lso(struct fbnic_ring *ring, struct sk_buff *skb, 199 __le64 *meta, unsigned int *l2len, unsigned int *i3len) 200 { 201 unsigned int l3_type, l4_type, l4len, hdrlen; 202 struct skb_shared_info *shinfo; 203 unsigned char *l4hdr; 204 __be16 payload_len; 205 206 if (unlikely(skb_cow_head(skb, 0))) 207 return true; 208 209 shinfo = skb_shinfo(skb); 210 211 if (shinfo->gso_type & SKB_GSO_PARTIAL) { 212 l3_type = FBNIC_TWD_L3_TYPE_OTHER; 213 } else if (!skb->encapsulation) { 214 if (ip_hdr(skb)->version == 4) 215 l3_type = FBNIC_TWD_L3_TYPE_IPV4; 216 else 217 l3_type = FBNIC_TWD_L3_TYPE_IPV6; 218 } else { 219 unsigned int o3len; 220 221 o3len = skb_inner_network_header(skb) - skb_network_header(skb); 222 *i3len -= o3len; 223 *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L3_OHLEN_MASK, 224 o3len / 2)); 225 l3_type = FBNIC_TWD_L3_TYPE_V6V6; 226 } 227 228 l4hdr = skb_checksum_start(skb); 229 payload_len = cpu_to_be16(skb->len - (l4hdr - skb->data)); 230 231 if (shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) { 232 struct tcphdr *tcph = (struct tcphdr *)l4hdr; 233 234 l4_type = FBNIC_TWD_L4_TYPE_TCP; 235 l4len = __tcp_hdrlen((struct tcphdr *)l4hdr); 236 csum_replace_by_diff(&tcph->check, (__force __wsum)payload_len); 237 } else { 238 struct udphdr *udph = (struct udphdr *)l4hdr; 239 240 l4_type = FBNIC_TWD_L4_TYPE_UDP; 241 l4len = sizeof(struct udphdr); 242 csum_replace_by_diff(&udph->check, (__force __wsum)payload_len); 243 } 244 245 hdrlen = (l4hdr - skb->data) + l4len; 246 *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L3_TYPE_MASK, l3_type) | 247 FIELD_PREP(FBNIC_TWD_L4_TYPE_MASK, l4_type) | 248 FIELD_PREP(FBNIC_TWD_L4_HLEN_MASK, l4len / 4) | 249 FIELD_PREP(FBNIC_TWD_MSS_MASK, shinfo->gso_size) | 250 FBNIC_TWD_FLAG_REQ_LSO); 251 252 FBNIC_XMIT_CB(skb)->bytecount += (shinfo->gso_segs - 1) * hdrlen; 253 FBNIC_XMIT_CB(skb)->gso_segs = shinfo->gso_segs; 254 255 u64_stats_update_begin(&ring->stats.syncp); 256 ring->stats.twq.lso += shinfo->gso_segs; 257 u64_stats_update_end(&ring->stats.syncp); 258 259 return false; 260 } 261 262 static bool 263 fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) 264 { 265 unsigned int l2len, i3len; 266 267 if (fbnic_tx_tstamp(skb)) 268 *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_TS); 269 270 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) 271 return false; 272 273 l2len = skb_mac_header_len(skb); 274 i3len = skb_checksum_start(skb) - skb_network_header(skb); 275 276 *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_CSUM_OFFSET_MASK, 277 skb->csum_offset / 2)); 278 279 if (skb_is_gso(skb)) { 280 if (fbnic_tx_lso(ring, skb, meta, &l2len, &i3len)) 281 return true; 282 } else { 283 *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_CSO); 284 u64_stats_update_begin(&ring->stats.syncp); 285 ring->stats.twq.csum_partial++; 286 u64_stats_update_end(&ring->stats.syncp); 287 } 288 289 *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L2_HLEN_MASK, l2len / 2) | 290 FIELD_PREP(FBNIC_TWD_L3_IHLEN_MASK, i3len / 2)); 291 return false; 292 } 293 294 static void 295 fbnic_rx_csum(u64 rcd, struct sk_buff *skb, struct fbnic_ring *rcq, 296 u64 *csum_cmpl, u64 *csum_none) 297 { 298 skb_checksum_none_assert(skb); 299 300 if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) { 301 (*csum_none)++; 302 return; 303 } 304 305 if (FIELD_GET(FBNIC_RCD_META_L4_CSUM_UNNECESSARY, rcd)) { 306 skb->ip_summed = CHECKSUM_UNNECESSARY; 307 } else { 308 u16 csum = FIELD_GET(FBNIC_RCD_META_L2_CSUM_MASK, rcd); 309 310 skb->ip_summed = CHECKSUM_COMPLETE; 311 skb->csum = (__force __wsum)csum; 312 (*csum_cmpl)++; 313 } 314 } 315 316 static void fbnic_tx_doorbell(struct fbnic_ring *ring, __le64 *meta) 317 { 318 *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_COMPLETION); 319 ring->deferred_meta = -1; 320 321 /* Force DMA writes to flush before writing to tail */ 322 dma_wmb(); 323 324 writel(ring->tail, ring->doorbell); 325 } 326 327 /* Packets handed to us with xmit_more set are left in the ring without a 328 * doorbell, and without a completion request, in the expectation that the 329 * packet ending the burst will ring for all of them. If that packet gets 330 * dropped instead we have to ring here, otherwise the descriptors sit in 331 * the ring until the next transmit, which may never come. 332 */ 333 static void fbnic_tx_flush_doorbell(struct fbnic_ring *ring) 334 { 335 if (ring->deferred_meta >= 0) 336 fbnic_tx_doorbell(ring, &ring->desc[ring->deferred_meta]); 337 } 338 339 static bool 340 fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) 341 { 342 struct device *dev = skb->dev->dev.parent; 343 unsigned int tail = ring->tail, first; 344 unsigned int size, data_len; 345 skb_frag_t *frag; 346 bool is_net_iov; 347 dma_addr_t dma; 348 __le64 *twd; 349 350 ring->tx_buf[tail] = skb; 351 352 tail++; 353 tail &= ring->size_mask; 354 first = tail; 355 356 size = skb_headlen(skb); 357 data_len = skb->data_len; 358 359 if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) 360 goto dma_error; 361 362 is_net_iov = false; 363 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); 364 365 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 366 twd = &ring->desc[tail]; 367 368 if (dma_mapping_error(dev, dma)) 369 goto dma_error; 370 371 *twd = cpu_to_le64(FIELD_PREP(FBNIC_TWD_ADDR_MASK, dma) | 372 FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | 373 FIELD_PREP(FBNIC_TWD_TYPE_MASK, 374 FBNIC_TWD_TYPE_AL)); 375 if (is_net_iov) 376 ring->tx_buf[tail] = FBNIC_XMIT_NOUNMAP; 377 378 tail++; 379 tail &= ring->size_mask; 380 381 if (!data_len) 382 break; 383 384 size = skb_frag_size(frag); 385 data_len -= size; 386 387 if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) 388 goto dma_error; 389 390 is_net_iov = skb_frag_is_net_iov(frag); 391 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); 392 } 393 394 *twd |= FBNIC_TWD_TYPE(LAST_AL); 395 396 FBNIC_XMIT_CB(skb)->desc_count = ((twd - meta) + 1) & ring->size_mask; 397 398 ring->tail = tail; 399 400 /* Record SW timestamp */ 401 skb_tx_timestamp(skb); 402 403 /* Verify there is room for another packet */ 404 fbnic_maybe_stop_tx(skb->dev, ring, FBNIC_MAX_SKB_DESC); 405 406 if (fbnic_tx_sent_queue(skb, ring)) 407 fbnic_tx_doorbell(ring, meta); 408 else 409 ring->deferred_meta = meta - ring->desc; 410 411 return false; 412 dma_error: 413 if (net_ratelimit()) 414 netdev_err(skb->dev, "TX DMA map failed\n"); 415 416 while (tail != first) { 417 tail--; 418 tail &= ring->size_mask; 419 twd = &ring->desc[tail]; 420 if (tail == first) 421 fbnic_unmap_single_twd(dev, twd); 422 else if (ring->tx_buf[tail] == FBNIC_XMIT_NOUNMAP) 423 ring->tx_buf[tail] = NULL; 424 else 425 fbnic_unmap_page_twd(dev, twd); 426 } 427 428 return true; 429 } 430 431 #define FBNIC_MIN_FRAME_LEN 60 432 433 static netdev_tx_t 434 fbnic_xmit_frame_ring(struct sk_buff *skb, struct fbnic_ring *ring) 435 { 436 __le64 *meta = &ring->desc[ring->tail]; 437 u16 desc_needed; 438 439 if (skb_put_padto(skb, FBNIC_MIN_FRAME_LEN)) 440 goto err_count; 441 442 /* Need: 1 descriptor per page, 443 * + 1 desc for skb_head, 444 * + 2 desc for metadata and timestamp metadata 445 * + 7 desc gap to keep tail from touching head 446 * otherwise try next time 447 */ 448 desc_needed = skb_shinfo(skb)->nr_frags + 10; 449 if (fbnic_maybe_stop_tx(skb->dev, ring, desc_needed)) { 450 fbnic_tx_flush_doorbell(ring); 451 return NETDEV_TX_BUSY; 452 } 453 454 *meta = cpu_to_le64(FBNIC_TWD_FLAG_DEST_MAC); 455 456 /* Write all members within DWORD to condense this into 2 4B writes */ 457 FBNIC_XMIT_CB(skb)->bytecount = skb->len; 458 FBNIC_XMIT_CB(skb)->gso_segs = 1; 459 FBNIC_XMIT_CB(skb)->desc_count = 0; 460 FBNIC_XMIT_CB(skb)->flags = 0; 461 462 if (fbnic_tx_offloads(ring, skb, meta)) 463 goto err_free; 464 465 if (fbnic_tx_map(ring, skb, meta)) 466 goto err_free; 467 468 return NETDEV_TX_OK; 469 470 err_free: 471 dev_kfree_skb_any(skb); 472 err_count: 473 fbnic_tx_flush_doorbell(ring); 474 475 u64_stats_update_begin(&ring->stats.syncp); 476 ring->stats.dropped++; 477 u64_stats_update_end(&ring->stats.syncp); 478 return NETDEV_TX_OK; 479 } 480 481 netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev) 482 { 483 struct fbnic_net *fbn = netdev_priv(dev); 484 unsigned int q_map = skb->queue_mapping; 485 486 return fbnic_xmit_frame_ring(skb, fbn->tx[q_map]); 487 } 488 489 static netdev_features_t 490 fbnic_features_check_encap_gso(struct sk_buff *skb, struct net_device *dev, 491 netdev_features_t features, unsigned int l3len) 492 { 493 netdev_features_t skb_gso_features; 494 struct ipv6hdr *ip6_hdr; 495 unsigned char l4_hdr; 496 unsigned int start; 497 __be16 frag_off; 498 499 /* Require MANGLEID for GSO_PARTIAL of IPv4. 500 * In theory we could support TSO with single, innermost v4 header 501 * by pretending everything before it is L2, but that needs to be 502 * parsed case by case.. so leaving it for when the need arises. 503 */ 504 if (!(features & NETIF_F_TSO_MANGLEID)) 505 features &= ~NETIF_F_TSO; 506 507 skb_gso_features = skb_shinfo(skb)->gso_type; 508 skb_gso_features <<= NETIF_F_GSO_SHIFT; 509 510 /* We'd only clear the native GSO features, so don't bother validating 511 * if the match can only be on those supported thru GSO_PARTIAL. 512 */ 513 if (!(skb_gso_features & FBNIC_TUN_GSO_FEATURES)) 514 return features; 515 516 /* We can only do IPv6-in-IPv6, not v4-in-v6. It'd be nice 517 * to fall back to partial for this, or any failure below. 518 * This is just an optimization, UDPv4 will be caught later on. 519 */ 520 if (skb_gso_features & NETIF_F_TSO) 521 return features & ~FBNIC_TUN_GSO_FEATURES; 522 523 /* Inner headers multiple of 2 */ 524 if ((skb_inner_network_header(skb) - skb_network_header(skb)) % 2) 525 return features & ~FBNIC_TUN_GSO_FEATURES; 526 527 /* Encapsulated GSO packet, make 100% sure it's IPv6-in-IPv6. */ 528 ip6_hdr = ipv6_hdr(skb); 529 if (ip6_hdr->version != 6) 530 return features & ~FBNIC_TUN_GSO_FEATURES; 531 532 l4_hdr = ip6_hdr->nexthdr; 533 start = (unsigned char *)ip6_hdr - skb->data + sizeof(struct ipv6hdr); 534 start = ipv6_skip_exthdr(skb, start, &l4_hdr, &frag_off); 535 if (frag_off || l4_hdr != IPPROTO_IPV6 || 536 skb->data + start != skb_inner_network_header(skb)) 537 return features & ~FBNIC_TUN_GSO_FEATURES; 538 539 return features; 540 } 541 542 netdev_features_t 543 fbnic_features_check(struct sk_buff *skb, struct net_device *dev, 544 netdev_features_t features) 545 { 546 unsigned int l2len, l3len; 547 548 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) 549 return features; 550 551 l2len = skb_mac_header_len(skb); 552 l3len = skb_checksum_start(skb) - skb_network_header(skb); 553 554 /* Check header lengths are multiple of 2. 555 * In case of 6in6 we support longer headers (IHLEN + OHLEN) 556 * but keep things simple for now, 512B is plenty. 557 */ 558 if ((l2len | l3len | skb->csum_offset) % 2 || 559 !FIELD_FIT(FBNIC_TWD_L2_HLEN_MASK, l2len / 2) || 560 !FIELD_FIT(FBNIC_TWD_L3_IHLEN_MASK, l3len / 2) || 561 !FIELD_FIT(FBNIC_TWD_CSUM_OFFSET_MASK, skb->csum_offset / 2)) 562 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 563 564 if (likely(!skb->encapsulation) || !skb_is_gso(skb)) 565 return features; 566 567 return fbnic_features_check_encap_gso(skb, dev, features, l3len); 568 } 569 570 static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, 571 struct fbnic_ring *ring, bool discard, 572 unsigned int hw_head) 573 { 574 u64 total_bytes = 0, total_packets = 0, ts_lost = 0; 575 unsigned int head = ring->head; 576 struct netdev_queue *txq; 577 unsigned int clean_desc; 578 579 clean_desc = (hw_head - head) & ring->size_mask; 580 581 while (clean_desc) { 582 struct sk_buff *skb = ring->tx_buf[head]; 583 unsigned int desc_cnt; 584 585 desc_cnt = FBNIC_XMIT_CB(skb)->desc_count; 586 if (desc_cnt > clean_desc) 587 break; 588 589 if (unlikely(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)) { 590 FBNIC_XMIT_CB(skb)->hw_head = hw_head; 591 if (likely(!discard)) 592 break; 593 ts_lost++; 594 } 595 596 ring->tx_buf[head] = NULL; 597 598 clean_desc -= desc_cnt; 599 600 while (!(ring->desc[head] & FBNIC_TWD_TYPE(AL))) { 601 head++; 602 head &= ring->size_mask; 603 desc_cnt--; 604 } 605 606 fbnic_unmap_single_twd(nv->dev, &ring->desc[head]); 607 head++; 608 head &= ring->size_mask; 609 desc_cnt--; 610 611 while (desc_cnt--) { 612 if (ring->tx_buf[head] != FBNIC_XMIT_NOUNMAP) 613 fbnic_unmap_page_twd(nv->dev, 614 &ring->desc[head]); 615 else 616 ring->tx_buf[head] = NULL; 617 head++; 618 head &= ring->size_mask; 619 } 620 621 total_bytes += FBNIC_XMIT_CB(skb)->bytecount; 622 total_packets += FBNIC_XMIT_CB(skb)->gso_segs; 623 624 napi_consume_skb(skb, napi_budget); 625 } 626 627 if (!total_bytes) 628 return; 629 630 ring->head = head; 631 632 txq = txring_txq(nv->napi.dev, ring); 633 634 if (unlikely(discard)) { 635 u64_stats_update_begin(&ring->stats.syncp); 636 ring->stats.dropped += total_packets; 637 ring->stats.twq.ts_lost += ts_lost; 638 u64_stats_update_end(&ring->stats.syncp); 639 640 netdev_tx_completed_queue(txq, total_packets, total_bytes); 641 return; 642 } 643 644 u64_stats_update_begin(&ring->stats.syncp); 645 ring->stats.bytes += total_bytes; 646 ring->stats.packets += total_packets; 647 u64_stats_update_end(&ring->stats.syncp); 648 649 if (!netif_txq_completed_wake(txq, total_packets, total_bytes, 650 fbnic_desc_unused(ring), 651 FBNIC_TX_DESC_WAKEUP)) { 652 u64_stats_update_begin(&ring->stats.syncp); 653 ring->stats.twq.wake++; 654 u64_stats_update_end(&ring->stats.syncp); 655 } 656 } 657 658 static void fbnic_clean_twq1(struct fbnic_napi_vector *nv, bool pp_allow_direct, 659 struct fbnic_ring *ring, bool discard, 660 unsigned int hw_head) 661 { 662 u64 total_bytes = 0, total_packets = 0; 663 unsigned int head = ring->head; 664 665 while (hw_head != head) { 666 struct page *page; 667 u64 twd; 668 669 if (unlikely(!(ring->desc[head] & FBNIC_TWD_TYPE(AL)))) 670 goto next_desc; 671 672 twd = le64_to_cpu(ring->desc[head]); 673 page = ring->tx_buf[head]; 674 675 /* TYPE_AL is 2, TYPE_LAST_AL is 3. So this trick gives 676 * us one increment per packet, with no branches. 677 */ 678 total_packets += FIELD_GET(FBNIC_TWD_TYPE_MASK, twd) - 679 FBNIC_TWD_TYPE_AL; 680 total_bytes += FIELD_GET(FBNIC_TWD_LEN_MASK, twd); 681 682 page_pool_put_page(pp_page_to_nmdesc(page)->pp, page, -1, 683 pp_allow_direct); 684 next_desc: 685 head++; 686 head &= ring->size_mask; 687 } 688 689 if (!total_bytes) 690 return; 691 692 ring->head = head; 693 694 if (discard) { 695 u64_stats_update_begin(&ring->stats.syncp); 696 ring->stats.dropped += total_packets; 697 u64_stats_update_end(&ring->stats.syncp); 698 return; 699 } 700 701 u64_stats_update_begin(&ring->stats.syncp); 702 ring->stats.bytes += total_bytes; 703 ring->stats.packets += total_packets; 704 u64_stats_update_end(&ring->stats.syncp); 705 } 706 707 static void fbnic_clean_tsq(struct fbnic_napi_vector *nv, 708 struct fbnic_ring *ring, 709 u64 tcd, int *ts_head, int *head0) 710 { 711 struct skb_shared_hwtstamps hwtstamp; 712 struct fbnic_net *fbn; 713 struct sk_buff *skb; 714 int head; 715 u64 ns; 716 717 head = (*ts_head < 0) ? ring->head : *ts_head; 718 719 do { 720 unsigned int desc_cnt; 721 722 if (head == ring->tail) { 723 if (unlikely(net_ratelimit())) 724 netdev_err(nv->napi.dev, 725 "Tx timestamp without matching packet\n"); 726 return; 727 } 728 729 skb = ring->tx_buf[head]; 730 desc_cnt = FBNIC_XMIT_CB(skb)->desc_count; 731 732 head += desc_cnt; 733 head &= ring->size_mask; 734 } while (!(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)); 735 736 fbn = netdev_priv(nv->napi.dev); 737 ns = fbnic_ts40_to_ns(fbn, FIELD_GET(FBNIC_TCD_TYPE1_TS_MASK, tcd)); 738 739 memset(&hwtstamp, 0, sizeof(hwtstamp)); 740 hwtstamp.hwtstamp = ns_to_ktime(ns); 741 742 *ts_head = head; 743 744 FBNIC_XMIT_CB(skb)->flags &= ~FBNIC_XMIT_CB_TS; 745 if (*head0 < 0) { 746 head = FBNIC_XMIT_CB(skb)->hw_head; 747 if (head >= 0) 748 *head0 = head; 749 } 750 751 skb_tstamp_tx(skb, &hwtstamp); 752 u64_stats_update_begin(&ring->stats.syncp); 753 ring->stats.twq.ts_packets++; 754 u64_stats_update_end(&ring->stats.syncp); 755 } 756 757 static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx, 758 netmem_ref netmem) 759 { 760 struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; 761 762 page_pool_fragment_netmem(netmem, FBNIC_PAGECNT_BIAS_MAX); 763 rx_buf->pagecnt_bias = FBNIC_PAGECNT_BIAS_MAX; 764 rx_buf->netmem = netmem; 765 } 766 767 static struct page * 768 fbnic_page_pool_get_head(struct fbnic_q_triad *qt, unsigned int idx) 769 { 770 struct fbnic_rx_buf *rx_buf = &qt->sub0.rx_buf[idx]; 771 772 rx_buf->pagecnt_bias--; 773 774 /* sub0 is always fed system pages, from the NAPI-level page_pool */ 775 return netmem_to_page(rx_buf->netmem); 776 } 777 778 static netmem_ref 779 fbnic_page_pool_get_data(struct fbnic_q_triad *qt, unsigned int idx) 780 { 781 struct fbnic_rx_buf *rx_buf = &qt->sub1.rx_buf[idx]; 782 783 rx_buf->pagecnt_bias--; 784 785 return rx_buf->netmem; 786 } 787 788 static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx, 789 int budget) 790 { 791 struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; 792 netmem_ref netmem = rx_buf->netmem; 793 794 if (!page_pool_unref_netmem(netmem, rx_buf->pagecnt_bias)) 795 page_pool_put_unrefed_netmem(ring->page_pool, netmem, -1, 796 !!budget); 797 798 rx_buf->netmem = 0; 799 } 800 801 static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget, 802 struct fbnic_q_triad *qt, s32 ts_head, s32 head0, 803 s32 head1) 804 { 805 if (head0 >= 0) 806 fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0); 807 else if (ts_head >= 0) 808 fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, ts_head); 809 810 if (head1 >= 0) { 811 qt->cmpl.deferred_head = -1; 812 if (napi_budget) 813 fbnic_clean_twq1(nv, true, &qt->sub1, false, head1); 814 else 815 qt->cmpl.deferred_head = head1; 816 } 817 } 818 819 static void 820 fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, 821 int napi_budget) 822 { 823 struct fbnic_ring *cmpl = &qt->cmpl; 824 s32 head1 = cmpl->deferred_head; 825 s32 head0 = -1, ts_head = -1; 826 __le64 *raw_tcd, done; 827 u32 head = cmpl->head; 828 829 done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(FBNIC_TCD_DONE); 830 raw_tcd = &cmpl->desc[head & cmpl->size_mask]; 831 832 /* Walk the completion queue collecting the heads reported by NIC */ 833 while ((*raw_tcd & cpu_to_le64(FBNIC_TCD_DONE)) == done) { 834 u64 tcd; 835 836 dma_rmb(); 837 838 tcd = le64_to_cpu(*raw_tcd); 839 840 switch (FIELD_GET(FBNIC_TCD_TYPE_MASK, tcd)) { 841 case FBNIC_TCD_TYPE_0: 842 if (tcd & FBNIC_TCD_TWQ1) 843 head1 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD1_MASK, 844 tcd); 845 else 846 head0 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD0_MASK, 847 tcd); 848 /* Currently all err status bits are related to 849 * timestamps and as those have yet to be added 850 * they are skipped for now. 851 */ 852 break; 853 case FBNIC_TCD_TYPE_1: 854 if (WARN_ON_ONCE(tcd & FBNIC_TCD_TWQ1)) 855 break; 856 857 fbnic_clean_tsq(nv, &qt->sub0, tcd, &ts_head, &head0); 858 break; 859 default: 860 break; 861 } 862 863 raw_tcd++; 864 head++; 865 if (!(head & cmpl->size_mask)) { 866 done ^= cpu_to_le64(FBNIC_TCD_DONE); 867 raw_tcd = &cmpl->desc[0]; 868 } 869 } 870 871 /* Record the current head/tail of the queue */ 872 if (cmpl->head != head) { 873 cmpl->head = head; 874 writel(head & cmpl->size_mask, cmpl->doorbell); 875 } 876 877 /* Unmap and free processed buffers */ 878 fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0, head1); 879 } 880 881 static void fbnic_clean_bdq(struct fbnic_ring *ring, unsigned int hw_head, 882 int napi_budget) 883 { 884 unsigned int head = ring->head; 885 886 if (head == hw_head) 887 return; 888 889 do { 890 fbnic_page_pool_drain(ring, head, napi_budget); 891 892 head++; 893 head &= ring->size_mask; 894 } while (head != hw_head); 895 896 ring->head = head; 897 } 898 899 static u16 fbnic_rcd_bd_idx(const struct fbnic_ring *bdq, u64 rcd) 900 { 901 return FIELD_GET(FBNIC_RCD_AL_BUFF_ID_MASK, rcd) >> bdq->bd_page_shift; 902 } 903 904 static unsigned int fbnic_rcd_bd_page_offset(const struct fbnic_ring *bdq, 905 u64 rcd) 906 { 907 u16 id = FIELD_GET(FBNIC_RCD_AL_BUFF_ID_MASK, rcd); 908 u16 page_id = id & (fbnic_bd_page_count(bdq) - 1); 909 910 return page_id * FBNIC_BD_PAGE_SIZE; 911 } 912 913 static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, netmem_ref netmem) 914 { 915 __le64 *bdq_desc = &bdq->desc[id * fbnic_bd_page_count(bdq)]; 916 dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem); 917 u64 bd, i = fbnic_bd_page_count(bdq); 918 919 bd = (FBNIC_BD_DESC_ADDR_MASK & dma) | 920 FIELD_PREP(FBNIC_BD_DESC_ID_MASK, (u64)id << bdq->bd_page_shift); 921 922 /* Posted pages larger than 4 KiB use consecutive device-page IDs in 923 * the low bits of the software page ID. 924 */ 925 do { 926 *bdq_desc = cpu_to_le64(bd); 927 bd += FIELD_PREP(FBNIC_BD_DESC_ADDR_MASK, 1) | 928 FIELD_PREP(FBNIC_BD_DESC_ID_MASK, 1); 929 bdq_desc++; 930 } while (--i); 931 } 932 933 static void fbnic_fill_bdq(struct fbnic_ring *bdq) 934 { 935 unsigned int count = fbnic_desc_unused(bdq); 936 unsigned int i = bdq->tail; 937 938 if (!count) 939 return; 940 941 do { 942 netmem_ref netmem; 943 944 netmem = page_pool_dev_alloc_netmems(bdq->page_pool); 945 if (!netmem) { 946 u64_stats_update_begin(&bdq->stats.syncp); 947 bdq->stats.bdq.alloc_failed++; 948 u64_stats_update_end(&bdq->stats.syncp); 949 950 break; 951 } 952 953 fbnic_page_pool_init(bdq, i, netmem); 954 fbnic_bd_prep(bdq, i, netmem); 955 956 i++; 957 i &= bdq->size_mask; 958 959 count--; 960 } while (count); 961 962 if (bdq->tail != i) { 963 bdq->tail = i; 964 965 /* Force DMA writes to flush before writing to tail */ 966 dma_wmb(); 967 968 writel(i * fbnic_bd_page_count(bdq), bdq->doorbell); 969 } 970 } 971 972 static unsigned int fbnic_hdr_pg_start(unsigned int pg_off) 973 { 974 /* The headroom of the first header may be larger than FBNIC_RX_HROOM 975 * due to alignment. So account for that by just making the page 976 * offset 0 if we are starting at the first header. 977 */ 978 if (ALIGN(FBNIC_RX_HROOM, 128) > FBNIC_RX_HROOM && 979 pg_off == ALIGN(FBNIC_RX_HROOM, 128)) 980 return 0; 981 982 return pg_off - FBNIC_RX_HROOM; 983 } 984 985 static unsigned int fbnic_hdr_pg_end(unsigned int pg_off, unsigned int len) 986 { 987 /* Determine the end of the buffer by finding the start of the next 988 * and then subtracting the headroom from that frame. 989 */ 990 pg_off += len + FBNIC_RX_TROOM + FBNIC_RX_HROOM; 991 992 return ALIGN(pg_off, 128) - FBNIC_RX_HROOM; 993 } 994 995 static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, 996 struct fbnic_pkt_buff *pkt, 997 struct fbnic_q_triad *qt) 998 { 999 unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); 1000 unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); 1001 unsigned int hdr_pg_idx = fbnic_rcd_bd_idx(&qt->sub0, rcd); 1002 unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom; 1003 unsigned char *hdr_start; 1004 struct page *page; 1005 1006 /* data_hard_start should always be NULL when this is called */ 1007 WARN_ON_ONCE(pkt->buff.data_hard_start); 1008 1009 page = fbnic_page_pool_get_head(qt, hdr_pg_idx); 1010 1011 /* Short-cut the end calculation if we know page is fully consumed */ 1012 hdr_pg_end = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? 1013 FBNIC_BD_PAGE_SIZE : fbnic_hdr_pg_end(hdr_pg_off, len); 1014 hdr_pg_start = fbnic_hdr_pg_start(hdr_pg_off); 1015 1016 headroom = hdr_pg_off - hdr_pg_start + FBNIC_RX_PAD; 1017 frame_sz = hdr_pg_end - hdr_pg_start; 1018 xdp_init_buff(&pkt->buff, frame_sz, &qt->xdp_rxq); 1019 hdr_pg_start += fbnic_rcd_bd_page_offset(&qt->sub0, rcd); 1020 1021 /* Sync DMA buffer */ 1022 dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), 1023 hdr_pg_start, frame_sz, 1024 DMA_BIDIRECTIONAL); 1025 1026 /* Build frame around buffer */ 1027 hdr_start = page_address(page) + hdr_pg_start; 1028 net_prefetch(pkt->buff.data); 1029 xdp_prepare_buff(&pkt->buff, hdr_start, headroom, 1030 len - FBNIC_RX_PAD, true); 1031 1032 pkt->hwtstamp = 0; 1033 pkt->add_frag_failed = false; 1034 } 1035 1036 static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, 1037 struct fbnic_pkt_buff *pkt, 1038 struct fbnic_q_triad *qt) 1039 { 1040 unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); 1041 unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); 1042 unsigned int pg_idx = fbnic_rcd_bd_idx(&qt->sub1, rcd); 1043 unsigned int truesize; 1044 netmem_ref netmem; 1045 bool added; 1046 1047 netmem = fbnic_page_pool_get_data(qt, pg_idx); 1048 1049 truesize = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? 1050 FBNIC_BD_PAGE_SIZE - pg_off : 1051 ALIGN(len, FBNIC_RX_PAYLD_ALIGN); 1052 1053 pg_off += fbnic_rcd_bd_page_offset(&qt->sub1, rcd); 1054 1055 /* Sync DMA buffer */ 1056 page_pool_dma_sync_netmem_for_cpu(qt->sub1.page_pool, netmem, 1057 pg_off, truesize); 1058 1059 /* Consecutive device-page completions from one PPQ page are adjacent 1060 * ranges in the same netmem. 1061 */ 1062 added = xdp_buff_add_frag(&pkt->buff, netmem, pg_off, len, truesize); 1063 if (unlikely(!added)) { 1064 pkt->add_frag_failed = true; 1065 netdev_err_once(nv->napi.dev, 1066 "Failed to add fragment to xdp_buff\n"); 1067 page_pool_put_full_netmem(qt->sub1.page_pool, netmem, true); 1068 } 1069 } 1070 1071 static void fbnic_put_pkt_buff(struct fbnic_q_triad *qt, 1072 struct fbnic_pkt_buff *pkt, int budget) 1073 { 1074 struct page *page; 1075 1076 if (!pkt->buff.data_hard_start) 1077 return; 1078 1079 if (xdp_buff_has_frags(&pkt->buff)) { 1080 struct skb_shared_info *shinfo; 1081 netmem_ref netmem; 1082 int nr_frags; 1083 1084 shinfo = xdp_get_shared_info_from_buff(&pkt->buff); 1085 nr_frags = shinfo->nr_frags; 1086 1087 while (nr_frags--) { 1088 netmem = skb_frag_netmem(&shinfo->frags[nr_frags]); 1089 page_pool_put_full_netmem(qt->sub1.page_pool, netmem, 1090 !!budget); 1091 } 1092 } 1093 1094 page = virt_to_page(pkt->buff.data_hard_start); 1095 page_pool_put_full_page(qt->sub0.page_pool, page, !!budget); 1096 } 1097 1098 static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv, 1099 struct fbnic_pkt_buff *pkt) 1100 { 1101 struct sk_buff *skb; 1102 1103 skb = xdp_build_skb_from_buff(&pkt->buff); 1104 if (!skb) 1105 return NULL; 1106 1107 /* Add timestamp if present */ 1108 if (pkt->hwtstamp) 1109 skb_hwtstamps(skb)->hwtstamp = pkt->hwtstamp; 1110 1111 return skb; 1112 } 1113 1114 static long fbnic_pkt_tx(struct fbnic_napi_vector *nv, 1115 struct fbnic_pkt_buff *pkt) 1116 { 1117 struct fbnic_ring *ring = &nv->qt[0].sub1; 1118 int size, offset, nsegs = 1, data_len = 0; 1119 unsigned int tail = ring->tail; 1120 struct skb_shared_info *shinfo; 1121 skb_frag_t *frag = NULL; 1122 struct page *page; 1123 dma_addr_t dma; 1124 __le64 *twd; 1125 1126 if (unlikely(xdp_buff_has_frags(&pkt->buff))) { 1127 shinfo = xdp_get_shared_info_from_buff(&pkt->buff); 1128 nsegs += shinfo->nr_frags; 1129 data_len = shinfo->xdp_frags_size; 1130 frag = &shinfo->frags[0]; 1131 } 1132 1133 if (fbnic_desc_unused(ring) < nsegs) { 1134 u64_stats_update_begin(&ring->stats.syncp); 1135 ring->stats.dropped++; 1136 u64_stats_update_end(&ring->stats.syncp); 1137 return -FBNIC_XDP_CONSUME; 1138 } 1139 1140 page = virt_to_page(pkt->buff.data_hard_start); 1141 offset = offset_in_page(pkt->buff.data); 1142 dma = page_pool_get_dma_addr(page); 1143 1144 size = pkt->buff.data_end - pkt->buff.data; 1145 1146 while (nsegs--) { 1147 dma_sync_single_range_for_device(nv->dev, dma, offset, size, 1148 DMA_BIDIRECTIONAL); 1149 dma += offset; 1150 1151 ring->tx_buf[tail] = page; 1152 1153 twd = &ring->desc[tail]; 1154 *twd = cpu_to_le64(FIELD_PREP(FBNIC_TWD_ADDR_MASK, dma) | 1155 FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | 1156 FIELD_PREP(FBNIC_TWD_TYPE_MASK, 1157 FBNIC_TWD_TYPE_AL)); 1158 1159 tail++; 1160 tail &= ring->size_mask; 1161 1162 if (!data_len) 1163 break; 1164 1165 offset = skb_frag_off(frag); 1166 page = skb_frag_page(frag); 1167 dma = page_pool_get_dma_addr(page); 1168 1169 size = skb_frag_size(frag); 1170 data_len -= size; 1171 frag++; 1172 } 1173 1174 *twd |= FBNIC_TWD_TYPE(LAST_AL); 1175 1176 ring->tail = tail; 1177 1178 return -FBNIC_XDP_TX; 1179 } 1180 1181 static void fbnic_pkt_commit_tail(struct fbnic_napi_vector *nv, 1182 unsigned int pkt_tail) 1183 { 1184 struct fbnic_ring *ring = &nv->qt[0].sub1; 1185 1186 /* Force DMA writes to flush before writing to tail */ 1187 dma_wmb(); 1188 1189 writel(pkt_tail, ring->doorbell); 1190 } 1191 1192 static struct sk_buff *fbnic_run_xdp(struct fbnic_napi_vector *nv, 1193 struct fbnic_pkt_buff *pkt) 1194 { 1195 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 1196 struct bpf_prog *xdp_prog; 1197 int act; 1198 1199 xdp_prog = READ_ONCE(fbn->xdp_prog); 1200 if (!xdp_prog) 1201 goto xdp_pass; 1202 1203 /* Should never happen, config paths enforce HDS threshold > MTU */ 1204 if (xdp_buff_has_frags(&pkt->buff) && !xdp_prog->aux->xdp_has_frags) 1205 return ERR_PTR(-FBNIC_XDP_LEN_ERR); 1206 1207 act = bpf_prog_run_xdp(xdp_prog, &pkt->buff); 1208 switch (act) { 1209 case XDP_PASS: 1210 xdp_pass: 1211 return fbnic_build_skb(nv, pkt); 1212 case XDP_TX: 1213 return ERR_PTR(fbnic_pkt_tx(nv, pkt)); 1214 default: 1215 bpf_warn_invalid_xdp_action(nv->napi.dev, xdp_prog, act); 1216 fallthrough; 1217 case XDP_ABORTED: 1218 trace_xdp_exception(nv->napi.dev, xdp_prog, act); 1219 fallthrough; 1220 case XDP_DROP: 1221 break; 1222 } 1223 1224 return ERR_PTR(-FBNIC_XDP_CONSUME); 1225 } 1226 1227 static enum pkt_hash_types fbnic_skb_hash_type(u64 rcd) 1228 { 1229 return (FBNIC_RCD_META_L4_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L4 : 1230 (FBNIC_RCD_META_L3_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L3 : 1231 PKT_HASH_TYPE_L2; 1232 } 1233 1234 static void fbnic_rx_tstamp(struct fbnic_napi_vector *nv, u64 rcd, 1235 struct fbnic_pkt_buff *pkt) 1236 { 1237 struct fbnic_net *fbn; 1238 u64 ns, ts; 1239 1240 if (!FIELD_GET(FBNIC_RCD_OPT_META_TS, rcd)) 1241 return; 1242 1243 fbn = netdev_priv(nv->napi.dev); 1244 ts = FIELD_GET(FBNIC_RCD_OPT_META_TS_MASK, rcd); 1245 ns = fbnic_ts40_to_ns(fbn, ts); 1246 1247 /* Add timestamp to shared info */ 1248 pkt->hwtstamp = ns_to_ktime(ns); 1249 } 1250 1251 static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv, 1252 u64 rcd, struct sk_buff *skb, 1253 struct fbnic_q_triad *qt, 1254 u64 *csum_cmpl, u64 *csum_none) 1255 { 1256 struct net_device *netdev = nv->napi.dev; 1257 struct fbnic_ring *rcq = &qt->cmpl; 1258 1259 fbnic_rx_csum(rcd, skb, rcq, csum_cmpl, csum_none); 1260 1261 if (netdev->features & NETIF_F_RXHASH) 1262 skb_set_hash(skb, 1263 FIELD_GET(FBNIC_RCD_META_RSS_HASH_MASK, rcd), 1264 fbnic_skb_hash_type(rcd)); 1265 1266 skb_record_rx_queue(skb, rcq->q_idx); 1267 } 1268 1269 static bool fbnic_rcd_metadata_err(u64 rcd) 1270 { 1271 return !!(FBNIC_RCD_META_UNCORRECTABLE_ERR_MASK & rcd); 1272 } 1273 1274 static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, 1275 struct fbnic_q_triad *qt, int budget) 1276 { 1277 unsigned int packets = 0, bytes = 0, dropped = 0, alloc_failed = 0; 1278 u64 csum_complete = 0, csum_none = 0, length_errors = 0; 1279 s32 head0 = -1, head1 = -1, pkt_tail = -1; 1280 struct fbnic_ring *rcq = &qt->cmpl; 1281 struct fbnic_pkt_buff *pkt; 1282 __le64 *raw_rcd, done; 1283 u32 head = rcq->head; 1284 1285 done = (head & (rcq->size_mask + 1)) ? cpu_to_le64(FBNIC_RCD_DONE) : 0; 1286 raw_rcd = &rcq->desc[head & rcq->size_mask]; 1287 pkt = rcq->pkt; 1288 1289 /* Walk the completion queue collecting the heads reported by NIC */ 1290 while (likely(packets < budget)) { 1291 struct sk_buff *skb = ERR_PTR(-EINVAL); 1292 u32 pkt_bytes; 1293 u64 rcd; 1294 1295 if ((*raw_rcd & cpu_to_le64(FBNIC_RCD_DONE)) == done) 1296 break; 1297 1298 dma_rmb(); 1299 1300 rcd = le64_to_cpu(*raw_rcd); 1301 1302 switch (FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd)) { 1303 case FBNIC_RCD_TYPE_HDR_AL: 1304 head0 = fbnic_rcd_bd_idx(&qt->sub0, rcd); 1305 fbnic_pkt_prepare(nv, rcd, pkt, qt); 1306 1307 break; 1308 case FBNIC_RCD_TYPE_PAY_AL: 1309 head1 = fbnic_rcd_bd_idx(&qt->sub1, rcd); 1310 fbnic_add_rx_frag(nv, rcd, pkt, qt); 1311 1312 break; 1313 case FBNIC_RCD_TYPE_OPT_META: 1314 /* Only type 0 is currently supported */ 1315 if (FIELD_GET(FBNIC_RCD_OPT_META_TYPE_MASK, rcd)) 1316 break; 1317 1318 fbnic_rx_tstamp(nv, rcd, pkt); 1319 1320 /* We currently ignore the action table index */ 1321 break; 1322 case FBNIC_RCD_TYPE_META: 1323 if (likely(!fbnic_rcd_metadata_err(rcd) && 1324 !pkt->add_frag_failed)) { 1325 pkt_bytes = xdp_get_buff_len(&pkt->buff); 1326 skb = fbnic_run_xdp(nv, pkt); 1327 } 1328 1329 /* Populate skb and invalidate XDP */ 1330 if (!IS_ERR_OR_NULL(skb)) { 1331 fbnic_populate_skb_fields(nv, rcd, skb, qt, 1332 &csum_complete, 1333 &csum_none); 1334 napi_gro_receive(&nv->napi, skb); 1335 } else if (skb == ERR_PTR(-FBNIC_XDP_TX)) { 1336 pkt_tail = nv->qt[0].sub1.tail; 1337 } else if (PTR_ERR(skb) == -FBNIC_XDP_CONSUME) { 1338 fbnic_put_pkt_buff(qt, pkt, 1); 1339 } else { 1340 if (!skb) 1341 alloc_failed++; 1342 1343 if (skb == ERR_PTR(-FBNIC_XDP_LEN_ERR)) 1344 length_errors++; 1345 else 1346 dropped++; 1347 1348 fbnic_put_pkt_buff(qt, pkt, 1); 1349 goto next_dont_count; 1350 } 1351 1352 packets++; 1353 bytes += pkt_bytes; 1354 next_dont_count: 1355 pkt->buff.data_hard_start = NULL; 1356 1357 break; 1358 } 1359 1360 raw_rcd++; 1361 head++; 1362 if (!(head & rcq->size_mask)) { 1363 done ^= cpu_to_le64(FBNIC_RCD_DONE); 1364 raw_rcd = &rcq->desc[0]; 1365 } 1366 } 1367 1368 u64_stats_update_begin(&rcq->stats.syncp); 1369 rcq->stats.packets += packets; 1370 rcq->stats.bytes += bytes; 1371 rcq->stats.dropped += dropped; 1372 rcq->stats.rx.alloc_failed += alloc_failed; 1373 rcq->stats.rx.csum_complete += csum_complete; 1374 rcq->stats.rx.csum_none += csum_none; 1375 rcq->stats.rx.length_errors += length_errors; 1376 u64_stats_update_end(&rcq->stats.syncp); 1377 1378 if (pkt_tail >= 0) 1379 fbnic_pkt_commit_tail(nv, pkt_tail); 1380 1381 /* Unmap and free processed buffers */ 1382 if (head0 >= 0) 1383 fbnic_clean_bdq(&qt->sub0, head0, budget); 1384 fbnic_fill_bdq(&qt->sub0); 1385 1386 if (head1 >= 0) 1387 fbnic_clean_bdq(&qt->sub1, head1, budget); 1388 fbnic_fill_bdq(&qt->sub1); 1389 1390 /* Record the current head/tail of the queue */ 1391 if (rcq->head != head) { 1392 rcq->head = head; 1393 writel(head & rcq->size_mask, rcq->doorbell); 1394 } 1395 1396 return packets; 1397 } 1398 1399 static void fbnic_nv_irq_disable(struct fbnic_napi_vector *nv) 1400 { 1401 struct fbnic_dev *fbd = nv->fbd; 1402 u32 v_idx = nv->v_idx; 1403 1404 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(v_idx / 32), 1 << (v_idx % 32)); 1405 } 1406 1407 static void fbnic_nv_irq_rearm(struct fbnic_napi_vector *nv) 1408 { 1409 struct fbnic_dev *fbd = nv->fbd; 1410 u32 v_idx = nv->v_idx; 1411 1412 fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(v_idx), 1413 FBNIC_INTR_CQ_REARM_INTR_UNMASK); 1414 } 1415 1416 static int fbnic_poll(struct napi_struct *napi, int budget) 1417 { 1418 struct fbnic_napi_vector *nv = container_of(napi, 1419 struct fbnic_napi_vector, 1420 napi); 1421 int i, j, work_done = 0; 1422 1423 for (i = 0; i < nv->txt_count; i++) 1424 fbnic_clean_tcq(nv, &nv->qt[i], budget); 1425 1426 for (j = 0; j < nv->rxt_count; j++, i++) 1427 work_done += fbnic_clean_rcq(nv, &nv->qt[i], budget); 1428 1429 if (work_done >= budget) 1430 return budget; 1431 1432 if (likely(napi_complete_done(napi, work_done))) 1433 fbnic_nv_irq_rearm(nv); 1434 1435 return work_done; 1436 } 1437 1438 irqreturn_t fbnic_msix_clean_rings(int __always_unused irq, void *data) 1439 { 1440 struct fbnic_napi_vector *nv = *(void **)data; 1441 1442 napi_schedule_irqoff(&nv->napi); 1443 1444 return IRQ_HANDLED; 1445 } 1446 1447 void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn, 1448 struct fbnic_ring *rxr) 1449 { 1450 struct fbnic_queue_stats *stats = &rxr->stats; 1451 1452 /* Capture stats from queues before dissasociating them */ 1453 fbn->rx_stats.bytes += stats->bytes; 1454 fbn->rx_stats.packets += stats->packets; 1455 fbn->rx_stats.dropped += stats->dropped; 1456 fbn->rx_stats.rx.alloc_failed += stats->rx.alloc_failed; 1457 fbn->rx_stats.rx.csum_complete += stats->rx.csum_complete; 1458 fbn->rx_stats.rx.csum_none += stats->rx.csum_none; 1459 fbn->rx_stats.rx.length_errors += stats->rx.length_errors; 1460 /* Remember to add new stats here */ 1461 BUILD_BUG_ON(sizeof(fbn->rx_stats.rx) / 8 != 4); 1462 } 1463 1464 void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn, 1465 struct fbnic_ring *bdq) 1466 { 1467 struct fbnic_queue_stats *stats = &bdq->stats; 1468 1469 /* Capture stats from queues before dissasociating them */ 1470 fbn->bdq_stats.bdq.alloc_failed += stats->bdq.alloc_failed; 1471 /* Remember to add new stats here */ 1472 BUILD_BUG_ON(sizeof(fbn->rx_stats.bdq) / 8 != 1); 1473 } 1474 1475 void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn, 1476 struct fbnic_ring *txr) 1477 { 1478 struct fbnic_queue_stats *stats = &txr->stats; 1479 1480 /* Capture stats from queues before dissasociating them */ 1481 fbn->tx_stats.bytes += stats->bytes; 1482 fbn->tx_stats.packets += stats->packets; 1483 fbn->tx_stats.dropped += stats->dropped; 1484 fbn->tx_stats.twq.csum_partial += stats->twq.csum_partial; 1485 fbn->tx_stats.twq.lso += stats->twq.lso; 1486 fbn->tx_stats.twq.ts_lost += stats->twq.ts_lost; 1487 fbn->tx_stats.twq.ts_packets += stats->twq.ts_packets; 1488 fbn->tx_stats.twq.stop += stats->twq.stop; 1489 fbn->tx_stats.twq.wake += stats->twq.wake; 1490 /* Remember to add new stats here */ 1491 BUILD_BUG_ON(sizeof(fbn->tx_stats.twq) / 8 != 6); 1492 } 1493 1494 void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn, 1495 struct fbnic_ring *xdpr) 1496 { 1497 struct fbnic_queue_stats *stats = &xdpr->stats; 1498 1499 if (!(xdpr->flags & FBNIC_RING_F_STATS)) 1500 return; 1501 1502 /* Capture stats from queues before dissasociating them */ 1503 fbn->tx_stats.dropped += stats->dropped; 1504 fbn->tx_stats.bytes += stats->bytes; 1505 fbn->tx_stats.packets += stats->packets; 1506 } 1507 1508 static void fbnic_remove_tx_ring(struct fbnic_net *fbn, 1509 struct fbnic_ring *txr) 1510 { 1511 if (!(txr->flags & FBNIC_RING_F_STATS)) 1512 return; 1513 1514 fbnic_aggregate_ring_tx_counters(fbn, txr); 1515 1516 /* Remove pointer to the Tx ring */ 1517 WARN_ON(fbn->tx[txr->q_idx] && fbn->tx[txr->q_idx] != txr); 1518 fbn->tx[txr->q_idx] = NULL; 1519 } 1520 1521 static void fbnic_remove_xdp_ring(struct fbnic_net *fbn, 1522 struct fbnic_ring *xdpr) 1523 { 1524 if (!(xdpr->flags & FBNIC_RING_F_STATS)) 1525 return; 1526 1527 fbnic_aggregate_ring_xdp_counters(fbn, xdpr); 1528 1529 /* Remove pointer to the Tx ring */ 1530 WARN_ON(fbn->tx[xdpr->q_idx] && fbn->tx[xdpr->q_idx] != xdpr); 1531 fbn->tx[xdpr->q_idx] = NULL; 1532 } 1533 1534 static void fbnic_remove_rx_ring(struct fbnic_net *fbn, 1535 struct fbnic_ring *rxr) 1536 { 1537 if (!(rxr->flags & FBNIC_RING_F_STATS)) 1538 return; 1539 1540 fbnic_aggregate_ring_rx_counters(fbn, rxr); 1541 1542 /* Remove pointer to the Rx ring */ 1543 WARN_ON(fbn->rx[rxr->q_idx] && fbn->rx[rxr->q_idx] != rxr); 1544 fbn->rx[rxr->q_idx] = NULL; 1545 } 1546 1547 static void fbnic_remove_bdq_ring(struct fbnic_net *fbn, 1548 struct fbnic_ring *bdq) 1549 { 1550 if (!(bdq->flags & FBNIC_RING_F_STATS)) 1551 return; 1552 1553 fbnic_aggregate_ring_bdq_counters(fbn, bdq); 1554 } 1555 1556 static void fbnic_free_qt_page_pools(struct fbnic_q_triad *qt) 1557 { 1558 page_pool_destroy(qt->sub0.page_pool); 1559 page_pool_destroy(qt->sub1.page_pool); 1560 } 1561 1562 static void fbnic_free_napi_vector(struct fbnic_net *fbn, 1563 struct fbnic_napi_vector *nv) 1564 { 1565 struct fbnic_dev *fbd = nv->fbd; 1566 int i, j; 1567 1568 for (i = 0; i < nv->txt_count; i++) { 1569 fbnic_remove_tx_ring(fbn, &nv->qt[i].sub0); 1570 fbnic_remove_xdp_ring(fbn, &nv->qt[i].sub1); 1571 fbnic_remove_tx_ring(fbn, &nv->qt[i].cmpl); 1572 } 1573 1574 for (j = 0; j < nv->rxt_count; j++, i++) { 1575 fbnic_remove_bdq_ring(fbn, &nv->qt[i].sub0); 1576 fbnic_remove_bdq_ring(fbn, &nv->qt[i].sub1); 1577 fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl); 1578 } 1579 1580 fbnic_napi_free_irq(fbd, nv); 1581 netif_napi_del_locked(&nv->napi); 1582 fbn->napi[fbnic_napi_idx(nv)] = NULL; 1583 kfree(nv); 1584 } 1585 1586 void fbnic_free_napi_vectors(struct fbnic_net *fbn) 1587 { 1588 int i; 1589 1590 for (i = 0; i < fbn->num_napi; i++) 1591 if (fbn->napi[i]) 1592 fbnic_free_napi_vector(fbn, fbn->napi[i]); 1593 } 1594 1595 static int 1596 fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, 1597 unsigned int rxq_idx, u32 rx_page_size) 1598 { 1599 struct page_pool_params pp_params = { 1600 .order = 0, 1601 .flags = PP_FLAG_DMA_MAP | 1602 PP_FLAG_DMA_SYNC_DEV, 1603 .pool_size = fbn->hpq_size + fbn->ppq_size, 1604 .nid = NUMA_NO_NODE, 1605 .dev = fbn->netdev->dev.parent, 1606 .dma_dir = DMA_BIDIRECTIONAL, 1607 .offset = 0, 1608 .max_len = PAGE_SIZE, 1609 .netdev = fbn->netdev, 1610 .queue_idx = rxq_idx, 1611 }; 1612 struct page_pool *pp; 1613 1614 /* Page pool cannot exceed a size of 32768. This doesn't limit the 1615 * pages on the ring but the number we can have cached waiting on 1616 * the next use. 1617 * 1618 * TBD: Can this be reduced further? Would a multiple of 1619 * NAPI_POLL_WEIGHT possibly make more sense? The question is how 1620 * may pages do we need to hold in reserve to get the best return 1621 * without hogging too much system memory. 1622 */ 1623 if (pp_params.pool_size > 32768) 1624 pp_params.pool_size = 32768; 1625 1626 pp = page_pool_create(&pp_params); 1627 if (IS_ERR(pp)) 1628 return PTR_ERR(pp); 1629 1630 qt->sub0.page_pool = pp; 1631 if (netif_rxq_has_unreadable_mp(fbn->netdev, rxq_idx)) { 1632 pp_params.order = get_order(rx_page_size); 1633 pp_params.max_len = rx_page_size; 1634 pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; 1635 pp_params.dma_dir = DMA_FROM_DEVICE; 1636 1637 pp = page_pool_create(&pp_params); 1638 if (IS_ERR(pp)) 1639 goto err_destroy_sub0; 1640 } else { 1641 page_pool_get(pp); 1642 } 1643 qt->sub1.page_pool = pp; 1644 1645 return 0; 1646 1647 err_destroy_sub0: 1648 page_pool_destroy(qt->sub0.page_pool); 1649 return PTR_ERR(pp); 1650 } 1651 1652 static u8 fbnic_bdq_page_shift(u32 page_size) 1653 { 1654 return ilog2(page_size / FBNIC_BD_PAGE_SIZE); 1655 } 1656 1657 static void fbnic_bdq_set_page_size(struct fbnic_ring *bdq, u32 page_size) 1658 { 1659 bdq->bd_page_shift = fbnic_bdq_page_shift(page_size); 1660 } 1661 1662 static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, 1663 int q_idx, u8 flags) 1664 { 1665 u64_stats_init(&ring->stats.syncp); 1666 ring->doorbell = doorbell; 1667 ring->q_idx = q_idx; 1668 ring->flags = flags; 1669 fbnic_bdq_set_page_size(ring, PAGE_SIZE); 1670 ring->deferred_head = -1; 1671 } 1672 1673 static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, 1674 unsigned int v_count, unsigned int v_idx, 1675 unsigned int txq_count, unsigned int txq_idx, 1676 unsigned int rxq_count, unsigned int rxq_idx) 1677 { 1678 int txt_count = txq_count, rxt_count = rxq_count; 1679 u32 __iomem *uc_addr = fbd->uc_addr0; 1680 int xdp_count = 0, qt_count, err; 1681 struct fbnic_napi_vector *nv; 1682 struct fbnic_q_triad *qt; 1683 u32 __iomem *db; 1684 1685 /* We need to reserve at least one Tx Queue Triad for an XDP ring */ 1686 if (rxq_count) { 1687 xdp_count = 1; 1688 if (!txt_count) 1689 txt_count = 1; 1690 } 1691 1692 qt_count = txt_count + rxq_count; 1693 if (!qt_count) 1694 return -EINVAL; 1695 1696 /* If MMIO has already failed there are no rings to initialize */ 1697 if (!uc_addr) 1698 return -EIO; 1699 1700 /* Allocate NAPI vector and queue triads */ 1701 nv = kzalloc_flex(*nv, qt, qt_count); 1702 if (!nv) 1703 return -ENOMEM; 1704 1705 /* Record queue triad counts */ 1706 nv->txt_count = txt_count; 1707 nv->rxt_count = rxt_count; 1708 1709 /* Provide pointer back to fbnic and MSI-X vectors */ 1710 nv->fbd = fbd; 1711 nv->v_idx = v_idx; 1712 1713 /* Tie napi to netdev */ 1714 fbn->napi[fbnic_napi_idx(nv)] = nv; 1715 netif_napi_add_config_locked(fbn->netdev, &nv->napi, fbnic_poll, 1716 fbnic_napi_idx(nv)); 1717 1718 /* Record IRQ to NAPI struct */ 1719 netif_napi_set_irq_locked(&nv->napi, 1720 pci_irq_vector(to_pci_dev(fbd->dev), 1721 nv->v_idx)); 1722 1723 /* Tie nv back to PCIe dev */ 1724 nv->dev = fbd->dev; 1725 1726 /* Request the IRQ for napi vector */ 1727 err = fbnic_napi_request_irq(fbd, nv); 1728 if (err) 1729 goto napi_del; 1730 1731 /* Initialize queue triads */ 1732 qt = nv->qt; 1733 1734 while (txt_count) { 1735 u8 flags = FBNIC_RING_F_CTX | FBNIC_RING_F_STATS; 1736 1737 /* Configure Tx queue */ 1738 db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ0_TAIL]; 1739 1740 /* Assign Tx queue to netdev if applicable */ 1741 if (txq_count > 0) { 1742 1743 fbnic_ring_init(&qt->sub0, db, txq_idx, flags); 1744 fbn->tx[txq_idx] = &qt->sub0; 1745 txq_count--; 1746 } else { 1747 fbnic_ring_init(&qt->sub0, db, 0, 1748 FBNIC_RING_F_DISABLED); 1749 } 1750 1751 /* Configure XDP queue */ 1752 db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ1_TAIL]; 1753 1754 /* Assign XDP queue to netdev if applicable 1755 * 1756 * The setup for this is in itself a bit different. 1757 * 1. We only need one XDP Tx queue per NAPI vector. 1758 * 2. We associate it to the first Rx queue index. 1759 * 3. The hardware side is associated based on the Tx Queue. 1760 * 4. The netdev queue is offset by FBNIC_MAX_TXQs. 1761 */ 1762 if (xdp_count > 0) { 1763 unsigned int xdp_idx = FBNIC_MAX_TXQS + rxq_idx; 1764 1765 fbnic_ring_init(&qt->sub1, db, xdp_idx, flags); 1766 fbn->tx[xdp_idx] = &qt->sub1; 1767 xdp_count--; 1768 } else { 1769 fbnic_ring_init(&qt->sub1, db, 0, 1770 FBNIC_RING_F_DISABLED); 1771 } 1772 1773 /* Configure Tx completion queue */ 1774 db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TCQ_HEAD]; 1775 fbnic_ring_init(&qt->cmpl, db, 0, 0); 1776 1777 /* Update Tx queue index */ 1778 txt_count--; 1779 txq_idx += v_count; 1780 1781 /* Move to next queue triad */ 1782 qt++; 1783 } 1784 1785 while (rxt_count) { 1786 /* Configure header queue */ 1787 db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_HPQ_TAIL]; 1788 fbnic_ring_init(&qt->sub0, db, 0, 1789 FBNIC_RING_F_CTX | FBNIC_RING_F_STATS); 1790 1791 /* Configure payload queue */ 1792 db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_PPQ_TAIL]; 1793 fbnic_ring_init(&qt->sub1, db, 0, 1794 FBNIC_RING_F_CTX | FBNIC_RING_F_STATS); 1795 1796 /* Configure Rx completion queue */ 1797 db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_RCQ_HEAD]; 1798 fbnic_ring_init(&qt->cmpl, db, rxq_idx, FBNIC_RING_F_STATS); 1799 fbn->rx[rxq_idx] = &qt->cmpl; 1800 1801 /* Update Rx queue index */ 1802 rxt_count--; 1803 rxq_idx += v_count; 1804 1805 /* Move to next queue triad */ 1806 qt++; 1807 } 1808 1809 return 0; 1810 1811 napi_del: 1812 netif_napi_del_locked(&nv->napi); 1813 fbn->napi[fbnic_napi_idx(nv)] = NULL; 1814 kfree(nv); 1815 return err; 1816 } 1817 1818 int fbnic_alloc_napi_vectors(struct fbnic_net *fbn) 1819 { 1820 unsigned int txq_idx = 0, rxq_idx = 0, v_idx = FBNIC_NON_NAPI_VECTORS; 1821 unsigned int num_tx = fbn->num_tx_queues; 1822 unsigned int num_rx = fbn->num_rx_queues; 1823 unsigned int num_napi = fbn->num_napi; 1824 struct fbnic_dev *fbd = fbn->fbd; 1825 int err; 1826 1827 /* Allocate 1 Tx queue per napi vector */ 1828 if (num_napi <= FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { 1829 while (num_tx) { 1830 err = fbnic_alloc_napi_vector(fbd, fbn, 1831 num_napi, v_idx, 1832 1, txq_idx, 0, 0); 1833 if (err) 1834 goto free_vectors; 1835 1836 /* Update counts and index */ 1837 num_tx--; 1838 txq_idx++; 1839 1840 v_idx++; 1841 } 1842 } 1843 1844 /* Allocate Tx/Rx queue pairs per vector, or allocate remaining Rx */ 1845 while (num_rx | num_tx) { 1846 int tqpv = DIV_ROUND_UP(num_tx, num_napi - txq_idx); 1847 int rqpv = DIV_ROUND_UP(num_rx, num_napi - rxq_idx); 1848 1849 err = fbnic_alloc_napi_vector(fbd, fbn, num_napi, v_idx, 1850 tqpv, txq_idx, rqpv, rxq_idx); 1851 if (err) 1852 goto free_vectors; 1853 1854 /* Update counts and index */ 1855 num_tx -= tqpv; 1856 txq_idx++; 1857 1858 num_rx -= rqpv; 1859 rxq_idx++; 1860 1861 v_idx++; 1862 } 1863 1864 return 0; 1865 1866 free_vectors: 1867 fbnic_free_napi_vectors(fbn); 1868 1869 return err; 1870 } 1871 1872 static void fbnic_free_ring_resources(struct device *dev, 1873 struct fbnic_ring *ring) 1874 { 1875 kvfree(ring->buffer); 1876 ring->buffer = NULL; 1877 1878 /* If size is not set there are no descriptors present */ 1879 if (!ring->size) 1880 return; 1881 1882 dma_free_coherent(dev, ring->size, ring->desc, ring->dma); 1883 ring->size_mask = 0; 1884 ring->size = 0; 1885 } 1886 1887 static int fbnic_alloc_tx_ring_desc(struct fbnic_net *fbn, 1888 struct fbnic_ring *txr) 1889 { 1890 struct device *dev = fbn->netdev->dev.parent; 1891 size_t size; 1892 1893 /* Round size up to nearest 4K */ 1894 size = ALIGN(array_size(sizeof(*txr->desc), fbn->txq_size), 4096); 1895 1896 txr->desc = dma_alloc_coherent(dev, size, &txr->dma, 1897 GFP_KERNEL | __GFP_NOWARN); 1898 if (!txr->desc) 1899 return -ENOMEM; 1900 1901 /* txq_size should be a power of 2, so mask is just that -1 */ 1902 txr->size_mask = fbn->txq_size - 1; 1903 txr->size = size; 1904 1905 return 0; 1906 } 1907 1908 static int fbnic_alloc_tx_ring_buffer(struct fbnic_ring *txr) 1909 { 1910 size_t size = array_size(sizeof(*txr->tx_buf), txr->size_mask + 1); 1911 1912 txr->tx_buf = kvzalloc(size, GFP_KERNEL | __GFP_NOWARN); 1913 1914 return txr->tx_buf ? 0 : -ENOMEM; 1915 } 1916 1917 static int fbnic_alloc_tx_ring_resources(struct fbnic_net *fbn, 1918 struct fbnic_ring *txr) 1919 { 1920 struct device *dev = fbn->netdev->dev.parent; 1921 int err; 1922 1923 if (txr->flags & FBNIC_RING_F_DISABLED) 1924 return 0; 1925 1926 err = fbnic_alloc_tx_ring_desc(fbn, txr); 1927 if (err) 1928 return err; 1929 1930 if (!(txr->flags & FBNIC_RING_F_CTX)) 1931 return 0; 1932 1933 err = fbnic_alloc_tx_ring_buffer(txr); 1934 if (err) 1935 goto free_desc; 1936 1937 return 0; 1938 1939 free_desc: 1940 fbnic_free_ring_resources(dev, txr); 1941 return err; 1942 } 1943 1944 static int fbnic_alloc_rx_ring_desc(struct fbnic_net *fbn, 1945 struct fbnic_ring *rxr) 1946 { 1947 struct device *dev = fbn->netdev->dev.parent; 1948 size_t desc_size = sizeof(*rxr->desc); 1949 u32 rxq_size; 1950 size_t size; 1951 1952 switch (rxr->doorbell - fbnic_ring_csr_base(rxr)) { 1953 case FBNIC_QUEUE_BDQ_HPQ_TAIL: 1954 rxq_size = fbn->hpq_size / fbnic_bd_page_count(rxr); 1955 desc_size *= fbnic_bd_page_count(rxr); 1956 break; 1957 case FBNIC_QUEUE_BDQ_PPQ_TAIL: 1958 rxq_size = fbn->ppq_size / fbnic_bd_page_count(rxr); 1959 desc_size *= fbnic_bd_page_count(rxr); 1960 break; 1961 case FBNIC_QUEUE_RCQ_HEAD: 1962 rxq_size = fbn->rcq_size; 1963 break; 1964 default: 1965 return -EINVAL; 1966 } 1967 1968 /* Round size up to nearest 4K */ 1969 size = ALIGN(array_size(desc_size, rxq_size), 4096); 1970 1971 rxr->desc = dma_alloc_coherent(dev, size, &rxr->dma, 1972 GFP_KERNEL | __GFP_NOWARN); 1973 if (!rxr->desc) 1974 return -ENOMEM; 1975 1976 /* rxq_size should be a power of 2, so mask is just that -1 */ 1977 rxr->size_mask = rxq_size - 1; 1978 rxr->size = size; 1979 1980 return 0; 1981 } 1982 1983 static int fbnic_alloc_rx_ring_buffer(struct fbnic_ring *rxr) 1984 { 1985 size_t size = array_size(sizeof(*rxr->rx_buf), rxr->size_mask + 1); 1986 1987 if (rxr->flags & FBNIC_RING_F_CTX) 1988 size = sizeof(*rxr->rx_buf) * (rxr->size_mask + 1); 1989 else 1990 size = sizeof(*rxr->pkt); 1991 1992 rxr->rx_buf = kvzalloc(size, GFP_KERNEL | __GFP_NOWARN); 1993 1994 return rxr->rx_buf ? 0 : -ENOMEM; 1995 } 1996 1997 static int fbnic_alloc_rx_ring_resources(struct fbnic_net *fbn, 1998 struct fbnic_ring *rxr) 1999 { 2000 struct device *dev = fbn->netdev->dev.parent; 2001 int err; 2002 2003 err = fbnic_alloc_rx_ring_desc(fbn, rxr); 2004 if (err) 2005 return err; 2006 2007 err = fbnic_alloc_rx_ring_buffer(rxr); 2008 if (err) 2009 goto free_desc; 2010 2011 return 0; 2012 2013 free_desc: 2014 fbnic_free_ring_resources(dev, rxr); 2015 return err; 2016 } 2017 2018 static void fbnic_free_qt_resources(struct fbnic_net *fbn, 2019 struct fbnic_q_triad *qt) 2020 { 2021 struct device *dev = fbn->netdev->dev.parent; 2022 2023 fbnic_free_ring_resources(dev, &qt->cmpl); 2024 fbnic_free_ring_resources(dev, &qt->sub1); 2025 fbnic_free_ring_resources(dev, &qt->sub0); 2026 2027 if (xdp_rxq_info_is_reg(&qt->xdp_rxq)) { 2028 xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); 2029 xdp_rxq_info_unreg(&qt->xdp_rxq); 2030 fbnic_free_qt_page_pools(qt); 2031 } 2032 } 2033 2034 static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn, 2035 struct fbnic_q_triad *qt) 2036 { 2037 struct device *dev = fbn->netdev->dev.parent; 2038 int err; 2039 2040 err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub0); 2041 if (err) 2042 return err; 2043 2044 err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub1); 2045 if (err) 2046 goto free_sub0; 2047 2048 err = fbnic_alloc_tx_ring_resources(fbn, &qt->cmpl); 2049 if (err) 2050 goto free_sub1; 2051 2052 return 0; 2053 2054 free_sub1: 2055 fbnic_free_ring_resources(dev, &qt->sub1); 2056 free_sub0: 2057 fbnic_free_ring_resources(dev, &qt->sub0); 2058 return err; 2059 } 2060 2061 static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn, 2062 struct fbnic_napi_vector *nv, 2063 struct fbnic_q_triad *qt, 2064 u32 rx_page_size) 2065 { 2066 struct device *dev = fbn->netdev->dev.parent; 2067 int err; 2068 2069 err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx, rx_page_size); 2070 if (err) 2071 return err; 2072 2073 fbnic_bdq_set_page_size(&qt->sub1, rx_page_size); 2074 2075 err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, qt->sub0.q_idx, 2076 nv->napi.napi_id); 2077 if (err) 2078 goto free_page_pools; 2079 2080 err = xdp_rxq_info_reg_mem_model(&qt->xdp_rxq, MEM_TYPE_PAGE_POOL, 2081 qt->sub0.page_pool); 2082 if (err) 2083 goto unreg_rxq; 2084 2085 err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0); 2086 if (err) 2087 goto unreg_mm; 2088 2089 err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub1); 2090 if (err) 2091 goto free_sub0; 2092 2093 err = fbnic_alloc_rx_ring_resources(fbn, &qt->cmpl); 2094 if (err) 2095 goto free_sub1; 2096 2097 return 0; 2098 2099 free_sub1: 2100 fbnic_free_ring_resources(dev, &qt->sub1); 2101 free_sub0: 2102 fbnic_free_ring_resources(dev, &qt->sub0); 2103 unreg_mm: 2104 xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); 2105 unreg_rxq: 2106 xdp_rxq_info_unreg(&qt->xdp_rxq); 2107 free_page_pools: 2108 fbnic_free_qt_page_pools(qt); 2109 return err; 2110 } 2111 2112 static void fbnic_free_nv_resources(struct fbnic_net *fbn, 2113 struct fbnic_napi_vector *nv) 2114 { 2115 int i; 2116 2117 for (i = 0; i < nv->txt_count + nv->rxt_count; i++) 2118 fbnic_free_qt_resources(fbn, &nv->qt[i]); 2119 } 2120 2121 static int fbnic_alloc_nv_resources(struct fbnic_net *fbn, 2122 struct fbnic_napi_vector *nv) 2123 { 2124 int i, j, err; 2125 2126 /* Allocate Tx Resources */ 2127 for (i = 0; i < nv->txt_count; i++) { 2128 err = fbnic_alloc_tx_qt_resources(fbn, &nv->qt[i]); 2129 if (err) 2130 goto free_qt_resources; 2131 } 2132 2133 /* Allocate Rx Resources */ 2134 for (j = 0; j < nv->rxt_count; j++, i++) { 2135 struct netdev_queue_config qcfg; 2136 2137 netdev_queue_config(fbn->netdev, nv->qt[i].cmpl.q_idx, &qcfg); 2138 err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i], 2139 qcfg.rx_page_size); 2140 if (err) 2141 goto free_qt_resources; 2142 } 2143 2144 return 0; 2145 2146 free_qt_resources: 2147 while (i--) 2148 fbnic_free_qt_resources(fbn, &nv->qt[i]); 2149 return err; 2150 } 2151 2152 void fbnic_free_resources(struct fbnic_net *fbn) 2153 { 2154 int i; 2155 2156 for (i = 0; i < fbn->num_napi; i++) 2157 fbnic_free_nv_resources(fbn, fbn->napi[i]); 2158 } 2159 2160 int fbnic_alloc_resources(struct fbnic_net *fbn) 2161 { 2162 int i, err = -ENODEV; 2163 2164 for (i = 0; i < fbn->num_napi; i++) { 2165 err = fbnic_alloc_nv_resources(fbn, fbn->napi[i]); 2166 if (err) 2167 goto free_resources; 2168 } 2169 2170 return 0; 2171 2172 free_resources: 2173 while (i--) 2174 fbnic_free_nv_resources(fbn, fbn->napi[i]); 2175 2176 return err; 2177 } 2178 2179 static void fbnic_set_netif_napi(struct fbnic_napi_vector *nv) 2180 { 2181 int i, j; 2182 2183 /* Associate Tx queue with NAPI */ 2184 for (i = 0; i < nv->txt_count; i++) { 2185 struct fbnic_q_triad *qt = &nv->qt[i]; 2186 2187 netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, 2188 NETDEV_QUEUE_TYPE_TX, &nv->napi); 2189 } 2190 2191 /* Associate Rx queue with NAPI */ 2192 for (j = 0; j < nv->rxt_count; j++, i++) { 2193 struct fbnic_q_triad *qt = &nv->qt[i]; 2194 2195 netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, 2196 NETDEV_QUEUE_TYPE_RX, &nv->napi); 2197 } 2198 } 2199 2200 static void fbnic_reset_netif_napi(struct fbnic_napi_vector *nv) 2201 { 2202 int i, j; 2203 2204 /* Disassociate Tx queue from NAPI */ 2205 for (i = 0; i < nv->txt_count; i++) { 2206 struct fbnic_q_triad *qt = &nv->qt[i]; 2207 2208 netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, 2209 NETDEV_QUEUE_TYPE_TX, NULL); 2210 } 2211 2212 /* Disassociate Rx queue from NAPI */ 2213 for (j = 0; j < nv->rxt_count; j++, i++) { 2214 struct fbnic_q_triad *qt = &nv->qt[i]; 2215 2216 netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, 2217 NETDEV_QUEUE_TYPE_RX, NULL); 2218 } 2219 } 2220 2221 int fbnic_set_netif_queues(struct fbnic_net *fbn) 2222 { 2223 int i, err; 2224 2225 err = netif_set_real_num_queues(fbn->netdev, fbn->num_tx_queues, 2226 fbn->num_rx_queues); 2227 if (err) 2228 return err; 2229 2230 for (i = 0; i < fbn->num_napi; i++) 2231 fbnic_set_netif_napi(fbn->napi[i]); 2232 2233 return 0; 2234 } 2235 2236 void fbnic_reset_netif_queues(struct fbnic_net *fbn) 2237 { 2238 int i; 2239 2240 for (i = 0; i < fbn->num_napi; i++) 2241 fbnic_reset_netif_napi(fbn->napi[i]); 2242 } 2243 2244 static void fbnic_disable_twq0(struct fbnic_ring *txr) 2245 { 2246 u32 twq_ctl = fbnic_ring_rd32(txr, FBNIC_QUEUE_TWQ0_CTL); 2247 2248 twq_ctl &= ~FBNIC_QUEUE_TWQ_CTL_ENABLE; 2249 2250 fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ0_CTL, twq_ctl); 2251 } 2252 2253 static void fbnic_disable_twq1(struct fbnic_ring *txr) 2254 { 2255 u32 twq_ctl = fbnic_ring_rd32(txr, FBNIC_QUEUE_TWQ1_CTL); 2256 2257 twq_ctl &= ~FBNIC_QUEUE_TWQ_CTL_ENABLE; 2258 2259 fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ1_CTL, twq_ctl); 2260 } 2261 2262 static void fbnic_disable_tcq(struct fbnic_ring *txr) 2263 { 2264 fbnic_ring_wr32(txr, FBNIC_QUEUE_TCQ_CTL, 0); 2265 fbnic_ring_wr32(txr, FBNIC_QUEUE_TIM_MASK, FBNIC_QUEUE_TIM_MASK_MASK); 2266 } 2267 2268 static void fbnic_disable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) 2269 { 2270 u32 bdq_ctl = fbnic_ring_rd32(hpq, FBNIC_QUEUE_BDQ_CTL); 2271 2272 bdq_ctl &= ~FBNIC_QUEUE_BDQ_CTL_ENABLE; 2273 2274 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, bdq_ctl); 2275 } 2276 2277 static void fbnic_disable_rcq(struct fbnic_ring *rxr) 2278 { 2279 fbnic_ring_wr32(rxr, FBNIC_QUEUE_RCQ_CTL, 0); 2280 fbnic_ring_wr32(rxr, FBNIC_QUEUE_RIM_MASK, FBNIC_QUEUE_RIM_MASK_MASK); 2281 } 2282 2283 void fbnic_napi_disable(struct fbnic_net *fbn) 2284 { 2285 int i; 2286 2287 for (i = 0; i < fbn->num_napi; i++) { 2288 napi_disable_locked(&fbn->napi[i]->napi); 2289 2290 fbnic_nv_irq_disable(fbn->napi[i]); 2291 } 2292 } 2293 2294 static void __fbnic_nv_disable(struct fbnic_napi_vector *nv) 2295 { 2296 int i, t; 2297 2298 /* Disable Tx queue triads */ 2299 for (t = 0; t < nv->txt_count; t++) { 2300 struct fbnic_q_triad *qt = &nv->qt[t]; 2301 2302 fbnic_disable_twq0(&qt->sub0); 2303 fbnic_disable_twq1(&qt->sub1); 2304 fbnic_disable_tcq(&qt->cmpl); 2305 } 2306 2307 /* Disable Rx queue triads */ 2308 for (i = 0; i < nv->rxt_count; i++, t++) { 2309 struct fbnic_q_triad *qt = &nv->qt[t]; 2310 2311 fbnic_disable_bdq(&qt->sub0, &qt->sub1); 2312 fbnic_disable_rcq(&qt->cmpl); 2313 } 2314 } 2315 2316 static void 2317 fbnic_nv_disable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) 2318 { 2319 __fbnic_nv_disable(nv); 2320 fbnic_wrfl(fbn->fbd); 2321 } 2322 2323 void fbnic_dbg_down(struct fbnic_net *fbn) 2324 { 2325 int i; 2326 2327 for (i = 0; i < fbn->num_napi; i++) 2328 fbnic_dbg_nv_exit(fbn->napi[i]); 2329 } 2330 2331 void fbnic_dbg_up(struct fbnic_net *fbn) 2332 { 2333 int i; 2334 2335 for (i = 0; i < fbn->num_napi; i++) 2336 fbnic_dbg_nv_init(fbn->napi[i]); 2337 } 2338 2339 void fbnic_disable(struct fbnic_net *fbn) 2340 { 2341 struct fbnic_dev *fbd = fbn->fbd; 2342 int i; 2343 2344 for (i = 0; i < fbn->num_napi; i++) 2345 __fbnic_nv_disable(fbn->napi[i]); 2346 2347 fbnic_wrfl(fbd); 2348 } 2349 2350 static void fbnic_tx_flush(struct fbnic_dev *fbd) 2351 { 2352 netdev_warn(fbd->netdev, "triggering Tx flush\n"); 2353 2354 fbnic_rmw32(fbd, FBNIC_TMI_DROP_CTRL, FBNIC_TMI_DROP_CTRL_EN, 2355 FBNIC_TMI_DROP_CTRL_EN); 2356 } 2357 2358 static void fbnic_tx_flush_off(struct fbnic_dev *fbd) 2359 { 2360 fbnic_rmw32(fbd, FBNIC_TMI_DROP_CTRL, FBNIC_TMI_DROP_CTRL_EN, 0); 2361 } 2362 2363 struct fbnic_idle_regs { 2364 u32 reg_base; 2365 u8 reg_cnt; 2366 }; 2367 2368 static bool fbnic_all_idle(struct fbnic_dev *fbd, 2369 const struct fbnic_idle_regs *regs, 2370 unsigned int nregs) 2371 { 2372 unsigned int i, j; 2373 2374 for (i = 0; i < nregs; i++) { 2375 for (j = 0; j < regs[i].reg_cnt; j++) { 2376 if (fbnic_rd32(fbd, regs[i].reg_base + j) != ~0U) 2377 return false; 2378 } 2379 } 2380 return true; 2381 } 2382 2383 static void fbnic_idle_dump(struct fbnic_dev *fbd, 2384 const struct fbnic_idle_regs *regs, 2385 unsigned int nregs, const char *dir, int err) 2386 { 2387 unsigned int i, j; 2388 2389 netdev_err(fbd->netdev, "error waiting for %s idle %d\n", dir, err); 2390 for (i = 0; i < nregs; i++) 2391 for (j = 0; j < regs[i].reg_cnt; j++) 2392 netdev_err(fbd->netdev, "0x%04x: %08x\n", 2393 regs[i].reg_base + j, 2394 fbnic_rd32(fbd, regs[i].reg_base + j)); 2395 } 2396 2397 int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail) 2398 { 2399 static const struct fbnic_idle_regs tx[] = { 2400 { FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TWQ_IDLE_CNT, }, 2401 { FBNIC_QM_TQS_IDLE(0), FBNIC_QM_TQS_IDLE_CNT, }, 2402 { FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TDE_IDLE_CNT, }, 2403 { FBNIC_QM_TCQ_IDLE(0), FBNIC_QM_TCQ_IDLE_CNT, }, 2404 }, rx[] = { 2405 { FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_HPQ_IDLE_CNT, }, 2406 { FBNIC_QM_PPQ_IDLE(0), FBNIC_QM_PPQ_IDLE_CNT, }, 2407 { FBNIC_QM_RCQ_IDLE(0), FBNIC_QM_RCQ_IDLE_CNT, }, 2408 }; 2409 bool idle; 2410 int err; 2411 2412 err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2, 500000, 2413 false, fbd, tx, ARRAY_SIZE(tx)); 2414 if (err == -ETIMEDOUT) { 2415 fbnic_tx_flush(fbd); 2416 err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2417 2, 500000, false, 2418 fbd, tx, ARRAY_SIZE(tx)); 2419 fbnic_tx_flush_off(fbd); 2420 } 2421 if (err) { 2422 fbnic_idle_dump(fbd, tx, ARRAY_SIZE(tx), "Tx", err); 2423 if (may_fail) 2424 return err; 2425 } 2426 2427 err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2, 500000, 2428 false, fbd, rx, ARRAY_SIZE(rx)); 2429 if (err) 2430 fbnic_idle_dump(fbd, rx, ARRAY_SIZE(rx), "Rx", err); 2431 return err; 2432 } 2433 2434 static int 2435 fbnic_wait_queue_idle(struct fbnic_net *fbn, bool rx, unsigned int idx) 2436 { 2437 static const unsigned int tx_regs[] = { 2438 FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TQS_IDLE(0), 2439 FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TCQ_IDLE(0), 2440 }, rx_regs[] = { 2441 FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_PPQ_IDLE(0), 2442 FBNIC_QM_RCQ_IDLE(0), 2443 }; 2444 struct fbnic_dev *fbd = fbn->fbd; 2445 unsigned int val, mask, off; 2446 const unsigned int *regs; 2447 unsigned int reg_cnt; 2448 int i, err; 2449 2450 regs = rx ? rx_regs : tx_regs; 2451 reg_cnt = rx ? ARRAY_SIZE(rx_regs) : ARRAY_SIZE(tx_regs); 2452 2453 off = idx / 32; 2454 mask = BIT(idx % 32); 2455 2456 for (i = 0; i < reg_cnt; i++) { 2457 err = read_poll_timeout_atomic(fbnic_rd32, val, val & mask, 2458 2, 500000, false, 2459 fbd, regs[i] + off); 2460 if (err) { 2461 netdev_err(fbd->netdev, 2462 "wait for queue %s%d idle failed 0x%04x(%d): %08x (mask: %08x)\n", 2463 rx ? "Rx" : "Tx", idx, regs[i] + off, i, 2464 val, mask); 2465 return err; 2466 } 2467 } 2468 2469 return 0; 2470 } 2471 2472 static void fbnic_nv_flush(struct fbnic_napi_vector *nv) 2473 { 2474 int j, t; 2475 2476 /* Flush any processed Tx Queue Triads and drop the rest */ 2477 for (t = 0; t < nv->txt_count; t++) { 2478 struct fbnic_q_triad *qt = &nv->qt[t]; 2479 struct netdev_queue *tx_queue; 2480 2481 /* Clean the work queues of unprocessed work */ 2482 fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); 2483 fbnic_clean_twq1(nv, false, &qt->sub1, true, 2484 qt->sub1.tail); 2485 2486 /* Reset completion queue descriptor ring */ 2487 memset(qt->cmpl.desc, 0, qt->cmpl.size); 2488 2489 /* Nothing else to do if Tx queue is disabled */ 2490 if (qt->sub0.flags & FBNIC_RING_F_DISABLED) 2491 continue; 2492 2493 /* Reset BQL associated with Tx queue */ 2494 tx_queue = netdev_get_tx_queue(nv->napi.dev, 2495 qt->sub0.q_idx); 2496 netdev_tx_reset_queue(tx_queue); 2497 } 2498 2499 /* Flush any processed Rx Queue Triads and drop the rest */ 2500 for (j = 0; j < nv->rxt_count; j++, t++) { 2501 struct fbnic_q_triad *qt = &nv->qt[t]; 2502 2503 /* Clean the work queues of unprocessed work */ 2504 fbnic_clean_bdq(&qt->sub0, qt->sub0.tail, 0); 2505 fbnic_clean_bdq(&qt->sub1, qt->sub1.tail, 0); 2506 2507 /* Reset completion queue descriptor ring */ 2508 memset(qt->cmpl.desc, 0, qt->cmpl.size); 2509 2510 fbnic_put_pkt_buff(qt, qt->cmpl.pkt, 0); 2511 memset(qt->cmpl.pkt, 0, sizeof(struct fbnic_pkt_buff)); 2512 } 2513 } 2514 2515 void fbnic_flush(struct fbnic_net *fbn) 2516 { 2517 int i; 2518 2519 for (i = 0; i < fbn->num_napi; i++) 2520 fbnic_nv_flush(fbn->napi[i]); 2521 } 2522 2523 static void fbnic_nv_fill(struct fbnic_napi_vector *nv) 2524 { 2525 int j, t; 2526 2527 /* Configure NAPI mapping and populate pages 2528 * in the BDQ rings to use for Rx 2529 */ 2530 for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { 2531 struct fbnic_q_triad *qt = &nv->qt[t]; 2532 2533 /* Populate the header and payload BDQs */ 2534 fbnic_fill_bdq(&qt->sub0); 2535 fbnic_fill_bdq(&qt->sub1); 2536 } 2537 } 2538 2539 void fbnic_fill(struct fbnic_net *fbn) 2540 { 2541 int i; 2542 2543 for (i = 0; i < fbn->num_napi; i++) 2544 fbnic_nv_fill(fbn->napi[i]); 2545 } 2546 2547 static void fbnic_enable_twq0(struct fbnic_ring *twq) 2548 { 2549 u32 log_size = fls(twq->size_mask); 2550 2551 if (!twq->size_mask) 2552 return; 2553 2554 /* Reset head/tail */ 2555 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); 2556 twq->tail = 0; 2557 twq->head = 0; 2558 twq->deferred_meta = -1; 2559 2560 /* Store descriptor ring address and size */ 2561 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAL, lower_32_bits(twq->dma)); 2562 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAH, upper_32_bits(twq->dma)); 2563 2564 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2565 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_SIZE, log_size & 0xf); 2566 2567 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); 2568 } 2569 2570 static void fbnic_enable_twq1(struct fbnic_ring *twq) 2571 { 2572 u32 log_size = fls(twq->size_mask); 2573 2574 if (!twq->size_mask) 2575 return; 2576 2577 /* Reset head/tail */ 2578 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); 2579 twq->tail = 0; 2580 twq->head = 0; 2581 2582 /* Store descriptor ring address and size */ 2583 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_BAL, lower_32_bits(twq->dma)); 2584 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_BAH, upper_32_bits(twq->dma)); 2585 2586 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2587 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_SIZE, log_size & 0xf); 2588 2589 fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); 2590 } 2591 2592 static void fbnic_enable_tcq(struct fbnic_napi_vector *nv, 2593 struct fbnic_ring *tcq) 2594 { 2595 u32 log_size = fls(tcq->size_mask); 2596 2597 if (!tcq->size_mask) 2598 return; 2599 2600 /* Reset head/tail */ 2601 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_RESET); 2602 tcq->tail = 0; 2603 tcq->head = 0; 2604 2605 /* Store descriptor ring address and size */ 2606 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_BAL, lower_32_bits(tcq->dma)); 2607 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_BAH, upper_32_bits(tcq->dma)); 2608 2609 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2610 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_SIZE, log_size & 0xf); 2611 2612 /* Store interrupt information for the completion queue */ 2613 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_CTL, nv->v_idx); 2614 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_THRESHOLD, tcq->size_mask / 2); 2615 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_MASK, 0); 2616 2617 /* Enable queue */ 2618 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_ENABLE); 2619 } 2620 2621 static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) 2622 { 2623 u32 bdq_ctl = FBNIC_QUEUE_BDQ_CTL_ENABLE; 2624 u32 log_size; 2625 2626 /* Reset head/tail */ 2627 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, FBNIC_QUEUE_BDQ_CTL_RESET); 2628 ppq->tail = 0; 2629 ppq->head = 0; 2630 hpq->tail = 0; 2631 hpq->head = 0; 2632 2633 log_size = fls(hpq->size_mask) + hpq->bd_page_shift; 2634 2635 /* Store descriptor ring address and size */ 2636 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAL, lower_32_bits(hpq->dma)); 2637 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAH, upper_32_bits(hpq->dma)); 2638 2639 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2640 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_SIZE, log_size & 0xf); 2641 2642 if (!ppq->size_mask) 2643 goto write_ctl; 2644 2645 log_size = fls(ppq->size_mask) + ppq->bd_page_shift; 2646 2647 /* Add enabling of PPQ to BDQ control */ 2648 bdq_ctl |= FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE; 2649 2650 /* Store descriptor ring address and size */ 2651 fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_BAL, lower_32_bits(ppq->dma)); 2652 fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_BAH, upper_32_bits(ppq->dma)); 2653 fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_SIZE, log_size & 0xf); 2654 2655 write_ctl: 2656 fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, bdq_ctl); 2657 } 2658 2659 static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, 2660 struct fbnic_ring *rcq, bool tx_pause, 2661 bool hdr_split) 2662 { 2663 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 2664 u32 drop_mode, rcq_ctl; 2665 2666 if (!tx_pause && fbn->num_rx_queues > 1) 2667 drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE; 2668 else 2669 drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_NEVER; 2670 2671 /* Specify packet layout */ 2672 rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK, drop_mode) | 2673 FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK, FBNIC_RX_HROOM) | 2674 FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM) | 2675 FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT, hdr_split); 2676 2677 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL0, rcq_ctl); 2678 } 2679 2680 void fbnic_config_drop_mode(struct fbnic_net *fbn, bool txp) 2681 { 2682 bool hds; 2683 int i, t; 2684 2685 hds = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; 2686 2687 for (i = 0; i < fbn->num_napi; i++) { 2688 struct fbnic_napi_vector *nv = fbn->napi[i]; 2689 2690 for (t = 0; t < nv->rxt_count; t++) { 2691 struct fbnic_q_triad *qt = &nv->qt[nv->txt_count + t]; 2692 2693 fbnic_config_drop_mode_rcq(nv, &qt->cmpl, txp, hds); 2694 } 2695 } 2696 } 2697 2698 static void fbnic_config_rim_threshold(struct fbnic_ring *rcq, u16 nv_idx, u32 rx_desc) 2699 { 2700 u32 threshold; 2701 2702 /* Set the threhsold to half the ring size if rx_frames 2703 * is not configured 2704 */ 2705 threshold = rx_desc ? : rcq->size_mask / 2; 2706 2707 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_CTL, nv_idx); 2708 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_THRESHOLD, threshold); 2709 } 2710 2711 void fbnic_config_rx_cqe_nsecs(struct fbnic_dev *fbd) 2712 { 2713 u32 coal_wait; 2714 2715 coal_wait = DIV_ROUND_CLOSEST_ULL((u64)fbd->rx_cqe_nsecs * 2716 FBNIC_CLOCK_FREQ, NSEC_PER_SEC); 2717 2718 /* TICK_CYCLES controls the interrupt threshold timer. COAL_WAIT is 2719 * independent and measured in core clock cycles. 2720 */ 2721 wr32(fbd, FBNIC_QM_RCQ_CTL0, 2722 FIELD_PREP(FBNIC_QM_RCQ_CTL0_TICK_CYCLES, 2723 FBNIC_CLOCK_FREQ / USEC_PER_SEC) | 2724 FIELD_PREP(FBNIC_QM_RCQ_CTL0_COAL_WAIT, coal_wait)); 2725 } 2726 2727 void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm) 2728 { 2729 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 2730 struct fbnic_dev *fbd = nv->fbd; 2731 u32 val = arm; 2732 2733 val |= FIELD_PREP(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT, fbn->rx_usecs) | 2734 FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT_UPD_EN; 2735 val |= FIELD_PREP(FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT, fbn->tx_usecs) | 2736 FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT_UPD_EN; 2737 2738 fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(nv->v_idx), val); 2739 } 2740 2741 void fbnic_config_rx_frames(struct fbnic_napi_vector *nv) 2742 { 2743 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 2744 int i; 2745 2746 for (i = nv->txt_count; i < nv->rxt_count + nv->txt_count; i++) { 2747 struct fbnic_q_triad *qt = &nv->qt[i]; 2748 2749 fbnic_config_rim_threshold(&qt->cmpl, nv->v_idx, 2750 fbn->rx_max_frames * 2751 FBNIC_MIN_RXD_PER_FRAME); 2752 } 2753 } 2754 2755 static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, 2756 struct fbnic_ring *rcq) 2757 { 2758 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); 2759 u32 log_size = fls(rcq->size_mask); 2760 u32 rcq_ctl = 0; 2761 bool hdr_split; 2762 u32 hds_thresh; 2763 2764 /* Force lower bound on MAX_HEADER_BYTES. Below this, all frames should 2765 * be split at L4. It would also result in the frames being split at 2766 * L2/L3 depending on the frame size. 2767 */ 2768 hdr_split = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; 2769 fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause, hdr_split); 2770 2771 hds_thresh = max(fbn->hds_thresh, FBNIC_HDR_BYTES_MIN); 2772 rcq_ctl |= FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | 2773 FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, hds_thresh) | 2774 FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, 2775 FBNIC_RX_PAYLD_OFFSET) | 2776 FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_PG_CL_MASK, 2777 FBNIC_RX_PAYLD_PG_CL); 2778 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL1, rcq_ctl); 2779 2780 /* Reset head/tail */ 2781 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_RESET); 2782 rcq->head = 0; 2783 rcq->tail = 0; 2784 2785 /* Store descriptor ring address and size */ 2786 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_BAL, lower_32_bits(rcq->dma)); 2787 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_BAH, upper_32_bits(rcq->dma)); 2788 2789 /* Write lower 4 bits of log size as 64K ring size is 0 */ 2790 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_SIZE, log_size & 0xf); 2791 2792 /* Store interrupt information for the completion queue */ 2793 fbnic_config_rim_threshold(rcq, nv->v_idx, fbn->rx_max_frames * 2794 FBNIC_MIN_RXD_PER_FRAME); 2795 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_MASK, 0); 2796 2797 /* Enable queue */ 2798 fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_ENABLE); 2799 } 2800 2801 static void __fbnic_nv_enable(struct fbnic_napi_vector *nv) 2802 { 2803 int j, t; 2804 2805 /* Setup Tx Queue Triads */ 2806 for (t = 0; t < nv->txt_count; t++) { 2807 struct fbnic_q_triad *qt = &nv->qt[t]; 2808 2809 fbnic_enable_twq0(&qt->sub0); 2810 fbnic_enable_twq1(&qt->sub1); 2811 fbnic_enable_tcq(nv, &qt->cmpl); 2812 } 2813 2814 /* Setup Rx Queue Triads */ 2815 for (j = 0; j < nv->rxt_count; j++, t++) { 2816 struct fbnic_q_triad *qt = &nv->qt[t]; 2817 2818 page_pool_enable_direct_recycling(qt->sub0.page_pool, 2819 &nv->napi); 2820 page_pool_enable_direct_recycling(qt->sub1.page_pool, 2821 &nv->napi); 2822 2823 fbnic_enable_bdq(&qt->sub0, &qt->sub1); 2824 fbnic_enable_rcq(nv, &qt->cmpl); 2825 } 2826 } 2827 2828 static void fbnic_nv_enable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) 2829 { 2830 __fbnic_nv_enable(nv); 2831 fbnic_wrfl(fbn->fbd); 2832 } 2833 2834 void fbnic_enable(struct fbnic_net *fbn) 2835 { 2836 struct fbnic_dev *fbd = fbn->fbd; 2837 int i; 2838 2839 for (i = 0; i < fbn->num_napi; i++) 2840 __fbnic_nv_enable(fbn->napi[i]); 2841 2842 fbnic_wrfl(fbd); 2843 } 2844 2845 static void fbnic_nv_irq_enable(struct fbnic_napi_vector *nv) 2846 { 2847 fbnic_config_txrx_usecs(nv, FBNIC_INTR_CQ_REARM_INTR_UNMASK); 2848 } 2849 2850 void fbnic_napi_enable(struct fbnic_net *fbn) 2851 { 2852 u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {}; 2853 struct fbnic_dev *fbd = fbn->fbd; 2854 int i; 2855 2856 for (i = 0; i < fbn->num_napi; i++) { 2857 struct fbnic_napi_vector *nv = fbn->napi[i]; 2858 2859 napi_enable_locked(&nv->napi); 2860 2861 fbnic_nv_irq_enable(nv); 2862 2863 /* Record bit used for NAPI IRQs so we can 2864 * set the mask appropriately 2865 */ 2866 irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); 2867 } 2868 2869 /* Force the first interrupt on the device to guarantee 2870 * that any packets that may have been enqueued during the 2871 * bringup are processed. 2872 */ 2873 for (i = 0; i < ARRAY_SIZE(irqs); i++) { 2874 if (!irqs[i]) 2875 continue; 2876 fbnic_wr32(fbd, FBNIC_INTR_SET(i), irqs[i]); 2877 } 2878 2879 fbnic_wrfl(fbd); 2880 } 2881 2882 void fbnic_napi_depletion_check(struct net_device *netdev) 2883 { 2884 struct fbnic_net *fbn = netdev_priv(netdev); 2885 u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {}; 2886 struct fbnic_dev *fbd = fbn->fbd; 2887 int i, j, t; 2888 2889 for (i = 0; i < fbn->num_napi; i++) { 2890 struct fbnic_napi_vector *nv = fbn->napi[i]; 2891 2892 /* Find RQs which are completely out of pages */ 2893 for (t = nv->txt_count, j = 0; j < nv->rxt_count; j++, t++) { 2894 /* Assume 4 pages is always enough to fit a packet 2895 * and therefore generate a completion and an IRQ. 2896 */ 2897 if (fbnic_desc_used(&nv->qt[t].sub0) < 4 || 2898 fbnic_desc_used(&nv->qt[t].sub1) < 4) 2899 irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); 2900 } 2901 } 2902 2903 for (i = 0; i < ARRAY_SIZE(irqs); i++) { 2904 if (!irqs[i]) 2905 continue; 2906 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i), irqs[i]); 2907 fbnic_wr32(fbd, FBNIC_INTR_SET(i), irqs[i]); 2908 } 2909 2910 fbnic_wrfl(fbd); 2911 } 2912 2913 /* Returns the napi vector servicing an Rx queue, or NULL if the datapath 2914 * is torn down. The association is published by fbnic_set_netif_napi() 2915 * and cleared by fbnic_reset_netif_napi(), both under the instance lock. 2916 */ 2917 static struct fbnic_napi_vector *fbnic_rxq_nv(struct net_device *dev, int idx) 2918 { 2919 struct napi_struct *napi = __netif_get_rx_queue(dev, idx)->napi; 2920 2921 return napi ? container_of(napi, struct fbnic_napi_vector, napi) : NULL; 2922 } 2923 2924 static int fbnic_queue_mem_alloc(struct net_device *dev, 2925 struct netdev_queue_config *qcfg, 2926 void *qmem, int idx) 2927 { 2928 struct fbnic_net *fbn = netdev_priv(dev); 2929 const struct fbnic_q_triad *real; 2930 struct fbnic_q_triad *qt = qmem; 2931 struct fbnic_napi_vector *nv; 2932 2933 if (!netif_running(dev)) 2934 return fbnic_alloc_qt_page_pools(fbn, qt, idx, 2935 qcfg->rx_page_size); 2936 2937 /* A failed PCIe recovery or resume can leave the datapath torn down 2938 * while netif_running() is still true. This ndo runs before 2939 * netdev_rx_queue_restart() checks netif_running(), so bail out 2940 * rather than touching rings and vectors that are already freed. 2941 */ 2942 nv = fbnic_rxq_nv(dev, idx); 2943 if (!nv) 2944 return -ENETDOWN; 2945 2946 real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 2947 2948 fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx, 2949 real->sub0.flags); 2950 fbnic_ring_init(&qt->sub1, real->sub1.doorbell, real->sub1.q_idx, 2951 real->sub1.flags); 2952 fbnic_ring_init(&qt->cmpl, real->cmpl.doorbell, real->cmpl.q_idx, 2953 real->cmpl.flags); 2954 2955 return fbnic_alloc_rx_qt_resources(fbn, nv, qt, qcfg->rx_page_size); 2956 } 2957 2958 static void fbnic_default_qcfg(struct net_device *dev, 2959 struct netdev_queue_config *qcfg) 2960 { 2961 qcfg->rx_page_size = PAGE_SIZE; 2962 } 2963 2964 static int fbnic_validate_qcfg(struct net_device *dev, 2965 struct netdev_queue_config *qcfg, 2966 struct netlink_ext_ack *extack) 2967 { 2968 u32 bd_page_count, ppq_entries, frag_count; 2969 u32 rx_page_size = qcfg->rx_page_size; 2970 u32 ppq_size; 2971 2972 if (!qcfg->rx_jumbo_ring_size) { 2973 NL_SET_ERR_MSG_MOD(extack, 2974 "rx-jumbo ring size must be nonzero"); 2975 return -EINVAL; 2976 } 2977 2978 ppq_size = roundup_pow_of_two(qcfg->rx_jumbo_ring_size); 2979 2980 if (!is_power_of_2(rx_page_size)) { 2981 NL_SET_ERR_MSG_MOD(extack, 2982 "rx_page_size must be a power of 2"); 2983 return -EINVAL; 2984 } 2985 2986 if (rx_page_size < FBNIC_BD_PAGE_SIZE) { 2987 NL_SET_ERR_MSG_MOD(extack, 2988 "rx_page_size must be at least 4K"); 2989 return -EINVAL; 2990 } 2991 2992 /* Payload fragments occupy multiples of FBNIC_RX_PAYLD_ALIGN bytes. 2993 * Keep at least one reference in the bias until fbnic_clean_bdq() 2994 * observes a completion from a subsequent allocation. 2995 */ 2996 frag_count = rx_page_size / FBNIC_RX_PAYLD_ALIGN; 2997 if (frag_count >= FBNIC_PAGECNT_BIAS_MAX) { 2998 NL_SET_ERR_MSG_MOD(extack, 2999 "rx_page_size can produce too many fragments"); 3000 return -EINVAL; 3001 } 3002 3003 bd_page_count = rx_page_size / FBNIC_BD_PAGE_SIZE; 3004 ppq_entries = ppq_size / bd_page_count; 3005 /* The PPQ is sized in 4 KiB device pages. One software entry tracks 3006 * each page-pool allocation. In addition to the unused entry for 3007 * empty/full accounting, cleanup retains the current allocation 3008 * until a completion identifies a subsequent allocation. A two-entry 3009 * ring can only post one allocation and cannot make progress. 3010 * Require at least four entries, since ring sizes are powers of two. 3011 */ 3012 if (ppq_entries < 4) { 3013 NL_SET_ERR_MSG_MOD(extack, 3014 "rx-jumbo ring size too small for rx_page_size"); 3015 return -EINVAL; 3016 } 3017 3018 return 0; 3019 } 3020 3021 static void fbnic_queue_mem_free(struct net_device *dev, void *qmem) 3022 { 3023 struct fbnic_net *fbn = netdev_priv(dev); 3024 struct fbnic_q_triad *qt = qmem; 3025 3026 if (!netif_running(dev)) 3027 fbnic_free_qt_page_pools(qt); 3028 else 3029 fbnic_free_qt_resources(fbn, qt); 3030 } 3031 3032 static void __fbnic_nv_restart(struct fbnic_net *fbn, 3033 struct fbnic_napi_vector *nv) 3034 { 3035 struct fbnic_dev *fbd = fbn->fbd; 3036 int i; 3037 3038 fbnic_nv_enable(fbn, nv); 3039 fbnic_nv_fill(nv); 3040 3041 napi_enable_locked(&nv->napi); 3042 fbnic_nv_irq_enable(nv); 3043 fbnic_wr32(fbd, FBNIC_INTR_SET(nv->v_idx / 32), BIT(nv->v_idx % 32)); 3044 fbnic_wrfl(fbd); 3045 3046 for (i = 0; i < nv->txt_count; i++) 3047 netif_wake_subqueue(fbn->netdev, nv->qt[i].sub0.q_idx); 3048 fbnic_dbg_nv_init(nv); 3049 } 3050 3051 static int fbnic_queue_start(struct net_device *dev, 3052 struct netdev_queue_config *qcfg, 3053 void *qmem, int idx) 3054 { 3055 struct fbnic_net *fbn = netdev_priv(dev); 3056 struct fbnic_napi_vector *nv; 3057 struct fbnic_q_triad *real; 3058 3059 real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 3060 nv = fbnic_rxq_nv(dev, idx); 3061 3062 fbnic_aggregate_ring_bdq_counters(fbn, &real->sub0); 3063 fbnic_aggregate_ring_bdq_counters(fbn, &real->sub1); 3064 fbnic_aggregate_ring_rx_counters(fbn, &real->cmpl); 3065 3066 memcpy(real, qmem, sizeof(*real)); 3067 3068 __fbnic_nv_restart(fbn, nv); 3069 3070 return 0; 3071 } 3072 3073 static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx) 3074 { 3075 struct fbnic_net *fbn = netdev_priv(dev); 3076 const struct fbnic_q_triad *real; 3077 struct fbnic_napi_vector *nv; 3078 int i, t; 3079 int err; 3080 3081 real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); 3082 nv = fbnic_rxq_nv(dev, idx); 3083 fbnic_dbg_nv_exit(nv); 3084 3085 napi_disable_locked(&nv->napi); 3086 fbnic_nv_irq_disable(nv); 3087 3088 for (i = 0; i < nv->txt_count; i++) 3089 netif_stop_subqueue(dev, nv->qt[i].sub0.q_idx); 3090 fbnic_nv_disable(fbn, nv); 3091 3092 for (t = 0; t < nv->txt_count + nv->rxt_count; t++) { 3093 err = fbnic_wait_queue_idle(fbn, t >= nv->txt_count, 3094 nv->qt[t].sub0.q_idx); 3095 if (err) 3096 goto err_restart; 3097 } 3098 3099 fbnic_synchronize_irq(fbn->fbd, nv->v_idx); 3100 fbnic_nv_flush(nv); 3101 3102 page_pool_disable_direct_recycling(real->sub0.page_pool); 3103 page_pool_disable_direct_recycling(real->sub1.page_pool); 3104 3105 memcpy(qmem, real, sizeof(*real)); 3106 3107 return 0; 3108 3109 err_restart: 3110 __fbnic_nv_restart(fbn, nv); 3111 return err; 3112 } 3113 3114 const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops = { 3115 .ndo_queue_mem_size = sizeof(struct fbnic_q_triad), 3116 .ndo_queue_mem_alloc = fbnic_queue_mem_alloc, 3117 .ndo_queue_mem_free = fbnic_queue_mem_free, 3118 .ndo_queue_start = fbnic_queue_start, 3119 .ndo_queue_stop = fbnic_queue_stop, 3120 .ndo_default_qcfg = fbnic_default_qcfg, 3121 .ndo_validate_qcfg = fbnic_validate_qcfg, 3122 .supported_params = QCFG_RX_PAGE_SIZE, 3123 }; 3124