1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Atlantic Network Driver 3 * 4 * Copyright (C) 2014-2019 aQuantia Corporation 5 * Copyright (C) 2019-2020 Marvell International Ltd. 6 */ 7 8 /* File aq_ring.h: Declaration of functions for Rx/Tx rings. */ 9 10 #ifndef AQ_RING_H 11 #define AQ_RING_H 12 13 #include "aq_common.h" 14 #include "aq_vec.h" 15 16 #define AQ_XDP_HEADROOM ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) 17 #define AQ_XDP_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) 18 19 struct page; 20 struct aq_nic_cfg_s; 21 22 struct aq_rxpage { 23 struct page *page; 24 dma_addr_t daddr; 25 unsigned int order; 26 unsigned int pg_off; 27 }; 28 29 /* TxC SOP DX EOP 30 * +----------+----------+----------+----------- 31 * 8bytes|len l3,l4 | pa | pa | pa 32 * +----------+----------+----------+----------- 33 * 4/8bytes|len pkt |len pkt | | skb 34 * +----------+----------+----------+----------- 35 * 4/8bytes|is_gso |len,flags |len |len,is_eop 36 * +----------+----------+----------+----------- 37 * 38 * This aq_ring_buff_s doesn't have endianness dependency. 39 * It is __packed for cache line optimizations. 40 */ 41 struct __packed aq_ring_buff_s { 42 union { 43 /* RX/TX */ 44 dma_addr_t pa; 45 /* RX */ 46 struct { 47 u32 rss_hash; 48 u16 next; 49 u8 is_hash_l4; 50 u8 rsvd1; 51 struct aq_rxpage rxdata; 52 u16 vlan_rx_tag; 53 }; 54 /* EOP */ 55 struct { 56 dma_addr_t pa_eop; 57 struct sk_buff *skb; 58 struct xdp_frame *xdpf; 59 }; 60 /* TxC */ 61 struct { 62 u32 mss; 63 u8 len_l2; 64 u8 len_l3; 65 u8 len_l4; 66 u8 is_ipv6:1; 67 u8 rsvd2:7; 68 u32 len_pkt; 69 u16 vlan_tx_tag; 70 }; 71 }; 72 union { 73 struct { 74 u32 len:16; 75 u32 is_ip_cso:1; 76 u32 is_udp_cso:1; 77 u32 is_tcp_cso:1; 78 u32 is_cso_err:1; 79 u32 is_sop:1; 80 u32 is_eop:1; 81 u32 is_gso_tcp:1; 82 u32 is_gso_udp:1; 83 u32 is_mapped:1; 84 u32 is_cleaned:1; 85 u32 is_error:1; 86 u32 is_vlan:1; 87 u32 is_lro:1; 88 u32 request_ts:1; 89 u32 clk_sel:1; 90 u32 rsvd3:1; 91 u16 eop_index; 92 u16 rsvd4; 93 }; 94 u64 flags; 95 }; 96 }; 97 98 struct aq_ring_stats_rx_s { 99 struct u64_stats_sync syncp; /* must be first */ 100 u64 errors; 101 u64 packets; 102 u64 bytes; 103 u64 lro_packets; 104 u64 jumbo_packets; 105 u64 alloc_fails; 106 u64 skb_alloc_fails; 107 u64 polls; 108 u64 pg_losts; 109 u64 pg_flips; 110 u64 pg_reuses; 111 u64 xdp_aborted; 112 u64 xdp_drop; 113 u64 xdp_pass; 114 u64 xdp_tx; 115 u64 xdp_invalid; 116 u64 xdp_redirect; 117 }; 118 119 struct aq_ring_stats_tx_s { 120 struct u64_stats_sync syncp; /* must be first */ 121 u64 errors; 122 u64 packets; 123 u64 bytes; 124 u64 queue_restarts; 125 }; 126 127 union aq_ring_stats_s { 128 struct aq_ring_stats_rx_s rx; 129 struct aq_ring_stats_tx_s tx; 130 }; 131 132 enum atl_ring_type { 133 ATL_RING_TX, 134 ATL_RING_RX, 135 }; 136 137 struct aq_ring_s { 138 struct aq_ring_buff_s *buff_ring; 139 u8 *dx_ring; /* descriptors ring, dma shared mem */ 140 struct aq_nic_s *aq_nic; 141 unsigned int idx; /* for HW layer registers operations */ 142 unsigned int hw_head; 143 unsigned int sw_head; 144 unsigned int sw_tail; 145 unsigned int size; /* descriptors number */ 146 unsigned int dx_size; /* TX or RX descriptor size, */ 147 /* stored here for fater math */ 148 u16 page_order; 149 u16 page_offset; 150 u16 frame_max; 151 u16 tail_size; 152 union aq_ring_stats_s stats; 153 dma_addr_t dx_ring_pa; 154 struct bpf_prog *xdp_prog; 155 enum atl_ring_type ring_type; 156 struct xdp_rxq_info xdp_rxq; 157 unsigned long ptp_ts_deadline; 158 }; 159 160 struct aq_ring_param_s { 161 unsigned int vec_idx; 162 unsigned int cpu; 163 cpumask_t affinity_mask; 164 }; 165 166 static inline void *aq_buf_vaddr(struct aq_rxpage *rxpage) 167 { 168 return page_to_virt(rxpage->page) + rxpage->pg_off; 169 } 170 171 static inline dma_addr_t aq_buf_daddr(struct aq_rxpage *rxpage) 172 { 173 return rxpage->daddr + rxpage->pg_off; 174 } 175 176 static inline unsigned int aq_ring_next_dx(struct aq_ring_s *self, 177 unsigned int dx) 178 { 179 return (++dx >= self->size) ? 0U : dx; 180 } 181 182 static inline unsigned int aq_ring_avail_dx(struct aq_ring_s *self) 183 { 184 return (((self->sw_tail >= self->sw_head)) ? 185 (self->size - 1) - self->sw_tail + self->sw_head : 186 self->sw_head - self->sw_tail - 1); 187 } 188 189 int aq_ring_tx_alloc(struct aq_ring_s *self, 190 struct aq_nic_s *aq_nic, 191 unsigned int idx, 192 struct aq_nic_cfg_s *aq_nic_cfg); 193 int aq_ring_rx_alloc(struct aq_ring_s *self, 194 struct aq_nic_s *aq_nic, 195 unsigned int idx, 196 struct aq_nic_cfg_s *aq_nic_cfg); 197 198 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type); 199 void aq_ring_rx_deinit(struct aq_ring_s *self); 200 void aq_ring_free(struct aq_ring_s *self); 201 void aq_ring_update_queue_state(struct aq_ring_s *ring); 202 void aq_ring_queue_wake(struct aq_ring_s *ring); 203 void aq_ring_queue_stop(struct aq_ring_s *ring); 204 bool aq_ring_tx_clean(struct aq_ring_s *self); 205 int aq_xdp_xmit(struct net_device *dev, int num_frames, 206 struct xdp_frame **frames, u32 flags); 207 int aq_ring_rx_clean(struct aq_ring_s *self, 208 struct napi_struct *napi, 209 int *work_done, 210 int budget); 211 int aq_ring_rx_fill(struct aq_ring_s *self); 212 213 int aq_ring_hwts_rx_alloc(struct aq_ring_s *self, 214 struct aq_nic_s *aq_nic, unsigned int idx, 215 unsigned int size, unsigned int dx_size); 216 void aq_ring_hwts_rx_free(struct aq_ring_s *self); 217 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic); 218 219 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data); 220 221 #endif /* AQ_RING_H */ 222