1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #ifndef _FBNIC_TXRX_H_ 5 #define _FBNIC_TXRX_H_ 6 7 #include <linux/netdevice.h> 8 #include <linux/skbuff.h> 9 #include <linux/types.h> 10 #include <linux/u64_stats_sync.h> 11 #include <net/xdp.h> 12 13 struct fbnic_net; 14 15 /* Guarantee we have space needed for storing the buffer 16 * To store the buffer we need: 17 * 1 descriptor per page 18 * + 1 descriptor for skb head 19 * + 2 descriptors for metadata and optional metadata 20 * + 7 descriptors to keep tail out of the same cacheline as head 21 * If we cannot guarantee that then we should return TX_BUSY 22 */ 23 #define FBNIC_MAX_SKB_DESC (MAX_SKB_FRAGS + 10) 24 #define FBNIC_TX_DESC_WAKEUP (FBNIC_MAX_SKB_DESC * 2) 25 #define FBNIC_TX_DESC_MIN roundup_pow_of_two(FBNIC_TX_DESC_WAKEUP) 26 27 /* To receive the worst case packet we need: 28 * 1 descriptor for primary metadata 29 * + 1 descriptor for optional metadata 30 * + 1 descriptor for headers 31 * + 4 descriptors for payload 32 */ 33 #define FBNIC_MAX_RX_PKT_DESC 7 34 #define FBNIC_RX_DESC_MIN roundup_pow_of_two(FBNIC_MAX_RX_PKT_DESC * 2) 35 36 #define FBNIC_MAX_TXQS 128u 37 #define FBNIC_MAX_RXQS 128u 38 #define FBNIC_MAX_XDPQS 128u 39 40 /* These apply to TWQs, TCQ, RCQ */ 41 #define FBNIC_QUEUE_SIZE_MIN 64u 42 #define FBNIC_QUEUE_SIZE_MAX SZ_64K 43 44 #define FBNIC_TXQ_SIZE_DEFAULT 1024 45 #define FBNIC_HPQ_SIZE_DEFAULT 256 46 #define FBNIC_PPQ_SIZE_DEFAULT 256 47 #define FBNIC_RCQ_SIZE_DEFAULT 1024 48 #define FBNIC_TX_USECS_DEFAULT 35 49 #define FBNIC_RX_USECS_DEFAULT 30 50 #define FBNIC_RX_FRAMES_DEFAULT 0 51 52 #define FBNIC_RX_TROOM \ 53 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) 54 #define FBNIC_RX_HROOM_PAD 128 55 #define FBNIC_RX_HROOM \ 56 (ALIGN(FBNIC_RX_TROOM + FBNIC_RX_HROOM_PAD, 128) - FBNIC_RX_TROOM) 57 #define FBNIC_RX_PAD 0 58 #define FBNIC_RX_PAYLD_OFFSET 0 59 #define FBNIC_RX_PAYLD_PG_CL 0 60 61 #define FBNIC_RING_F_DISABLED BIT(0) 62 #define FBNIC_RING_F_CTX BIT(1) 63 #define FBNIC_RING_F_STATS BIT(2) /* Ring's stats may be used */ 64 65 #define FBNIC_HDS_THRESH_MAX \ 66 (4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD) 67 #define FBNIC_HDS_THRESH_DEFAULT \ 68 (1536 - FBNIC_RX_PAD) 69 #define FBNIC_HDR_BYTES_MIN 256 70 71 struct fbnic_pkt_buff { 72 struct xdp_buff buff; 73 ktime_t hwtstamp; 74 bool add_frag_failed; 75 }; 76 77 struct fbnic_queue_stats { 78 u64 packets; 79 u64 bytes; 80 union { 81 struct { 82 u64 csum_partial; 83 u64 lso; 84 u64 ts_packets; 85 u64 ts_lost; 86 u64 stop; 87 u64 wake; 88 } twq; 89 struct { 90 u64 alloc_failed; 91 u64 csum_complete; 92 u64 csum_none; 93 u64 length_errors; 94 } rx; 95 struct { 96 u64 alloc_failed; 97 } bdq; 98 }; 99 u64 dropped; 100 struct u64_stats_sync syncp; 101 }; 102 103 #define FBNIC_PAGECNT_BIAS_MAX PAGE_SIZE 104 105 struct fbnic_rx_buf { 106 netmem_ref netmem; 107 long pagecnt_bias; 108 }; 109 110 struct fbnic_ring { 111 /* Pointer to buffer specific info */ 112 union { 113 struct fbnic_pkt_buff *pkt; /* RCQ */ 114 struct fbnic_rx_buf *rx_buf; /* BDQ */ 115 void **tx_buf; /* TWQ */ 116 void *buffer; /* Generic pointer */ 117 }; 118 119 u32 __iomem *doorbell; /* Pointer to CSR space for ring */ 120 __le64 *desc; /* Descriptor ring memory */ 121 u16 size_mask; /* Size of ring in descriptors - 1 */ 122 u8 q_idx; /* Logical netdev ring index */ 123 u8 flags; /* Ring flags (FBNIC_RING_F_*) */ 124 125 u32 head, tail; /* Head/Tail of ring */ 126 127 union { 128 /* Rx BDQs only */ 129 struct page_pool *page_pool; 130 131 /* TWQ0 only, index of the meta descriptor of the last packet 132 * placed in the ring without ringing the doorbell, -1 if the 133 * doorbell is in sync with the tail. 134 */ 135 s32 deferred_meta; 136 137 /* TCQ only, used to cache the head for TWQ1 if 138 * an attempt is made to clean TWQ1 with zero napi_budget. 139 */ 140 s32 deferred_head; 141 }; 142 143 struct fbnic_queue_stats stats; 144 145 /* Slow path fields follow */ 146 dma_addr_t dma; /* Phys addr of descriptor memory */ 147 size_t size; /* Size of descriptor ring in memory */ 148 }; 149 150 struct fbnic_q_triad { 151 struct fbnic_ring sub0, sub1, cmpl; 152 struct xdp_rxq_info xdp_rxq; 153 }; 154 155 struct fbnic_napi_vector { 156 struct napi_struct napi; 157 struct device *dev; /* Device for DMA unmapping */ 158 struct fbnic_dev *fbd; 159 struct dentry *dbg_nv; 160 161 u16 v_idx; 162 u8 txt_count; 163 u8 rxt_count; 164 165 struct fbnic_q_triad qt[]; 166 }; 167 168 extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops; 169 170 netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev); 171 netdev_features_t 172 fbnic_features_check(struct sk_buff *skb, struct net_device *dev, 173 netdev_features_t features); 174 175 void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn, 176 struct fbnic_ring *rxr); 177 void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn, 178 struct fbnic_ring *rxr); 179 void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn, 180 struct fbnic_ring *txr); 181 void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn, 182 struct fbnic_ring *xdpr); 183 184 int fbnic_alloc_napi_vectors(struct fbnic_net *fbn); 185 void fbnic_free_napi_vectors(struct fbnic_net *fbn); 186 int fbnic_alloc_resources(struct fbnic_net *fbn); 187 void fbnic_free_resources(struct fbnic_net *fbn); 188 int fbnic_set_netif_queues(struct fbnic_net *fbn); 189 void fbnic_reset_netif_queues(struct fbnic_net *fbn); 190 irqreturn_t fbnic_msix_clean_rings(int irq, void *data); 191 void fbnic_napi_enable(struct fbnic_net *fbn); 192 void fbnic_napi_disable(struct fbnic_net *fbn); 193 void fbnic_config_drop_mode(struct fbnic_net *fbn, bool tx_pause); 194 void fbnic_enable(struct fbnic_net *fbn); 195 void fbnic_disable(struct fbnic_net *fbn); 196 void fbnic_dbg_up(struct fbnic_net *fbn); 197 void fbnic_dbg_down(struct fbnic_net *fbn); 198 void fbnic_flush(struct fbnic_net *fbn); 199 void fbnic_fill(struct fbnic_net *fbn); 200 201 u32 __iomem *fbnic_ring_csr_base(const struct fbnic_ring *ring); 202 void fbnic_napi_depletion_check(struct net_device *netdev); 203 int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail); 204 205 static inline int fbnic_napi_idx(const struct fbnic_napi_vector *nv) 206 { 207 return nv->v_idx - FBNIC_NON_NAPI_VECTORS; 208 } 209 210 void fbnic_dbg_nv_init(struct fbnic_napi_vector *nv); 211 void fbnic_dbg_nv_exit(struct fbnic_napi_vector *nv); 212 #endif /* _FBNIC_TXRX_H_ */ 213