1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Atlantic Network Driver
3 *
4 * Copyright (C) 2014-2019 aQuantia Corporation
5 * Copyright (C) 2019-2020 Marvell International Ltd.
6 */
7
8 /* File aq_ring.h: Declaration of functions for Rx/Tx rings. */
9
10 #ifndef AQ_RING_H
11 #define AQ_RING_H
12
13 #include "aq_common.h"
14 #include "aq_vec.h"
15
16 #define AQ_XDP_HEADROOM ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8)
17 #define AQ_XDP_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
18
19 struct page;
20 struct aq_nic_cfg_s;
21
22 struct aq_rxpage {
23 struct page *page;
24 dma_addr_t daddr;
25 unsigned int order;
26 unsigned int pg_off;
27 };
28
29 /* TxC SOP DX EOP
30 * +----------+----------+----------+-----------
31 * 8bytes|len l3,l4 | pa | pa | pa
32 * +----------+----------+----------+-----------
33 * 4/8bytes|len pkt |len pkt | | skb
34 * +----------+----------+----------+-----------
35 * 4/8bytes|is_gso |len,flags |len |len,is_eop
36 * +----------+----------+----------+-----------
37 *
38 * This aq_ring_buff_s doesn't have endianness dependency.
39 * It is __packed for cache line optimizations.
40 */
41 struct __packed aq_ring_buff_s {
42 union {
43 /* RX/TX */
44 dma_addr_t pa;
45 /* RX */
46 struct {
47 u32 rss_hash;
48 u16 next;
49 u8 is_hash_l4;
50 u8 rsvd1;
51 struct aq_rxpage rxdata;
52 u16 vlan_rx_tag;
53 };
54 /* EOP */
55 struct {
56 dma_addr_t pa_eop;
57 struct sk_buff *skb;
58 struct xdp_frame *xdpf;
59 };
60 /* TxC */
61 struct {
62 u32 mss;
63 u8 len_l2;
64 u8 len_l3;
65 u8 len_l4;
66 u8 is_ipv6:1;
67 u8 rsvd2:7;
68 u32 len_pkt;
69 u16 vlan_tx_tag;
70 };
71 };
72 union {
73 struct {
74 u32 len:16;
75 u32 is_ip_cso:1;
76 u32 is_udp_cso:1;
77 u32 is_tcp_cso:1;
78 u32 is_cso_err:1;
79 u32 is_sop:1;
80 u32 is_eop:1;
81 u32 is_gso_tcp:1;
82 u32 is_gso_udp:1;
83 u32 is_mapped:1;
84 u32 is_cleaned:1;
85 u32 is_error:1;
86 u32 is_vlan:1;
87 u32 is_lro:1;
88 u32 request_ts:1;
89 u32 clk_sel:1;
90 u32 rsvd3:1;
91 u16 eop_index;
92 u16 rsvd4;
93 };
94 u64 flags;
95 };
96 };
97
98 struct aq_ring_stats_rx_s {
99 struct u64_stats_sync syncp; /* must be first */
100 u64 errors;
101 u64 packets;
102 u64 bytes;
103 u64 lro_packets;
104 u64 jumbo_packets;
105 u64 alloc_fails;
106 u64 skb_alloc_fails;
107 u64 polls;
108 u64 pg_losts;
109 u64 pg_flips;
110 u64 pg_reuses;
111 u64 xdp_aborted;
112 u64 xdp_drop;
113 u64 xdp_pass;
114 u64 xdp_tx;
115 u64 xdp_invalid;
116 u64 xdp_redirect;
117 };
118
119 struct aq_ring_stats_tx_s {
120 struct u64_stats_sync syncp; /* must be first */
121 u64 errors;
122 u64 packets;
123 u64 bytes;
124 u64 queue_restarts;
125 };
126
127 union aq_ring_stats_s {
128 struct aq_ring_stats_rx_s rx;
129 struct aq_ring_stats_tx_s tx;
130 };
131
132 enum atl_ring_type {
133 ATL_RING_TX,
134 ATL_RING_RX,
135 };
136
137 struct aq_ring_s {
138 struct aq_ring_buff_s *buff_ring;
139 u8 *dx_ring; /* descriptors ring, dma shared mem */
140 struct aq_nic_s *aq_nic;
141 unsigned int idx; /* for HW layer registers operations */
142 unsigned int hw_head;
143 unsigned int sw_head;
144 unsigned int sw_tail;
145 unsigned int size; /* descriptors number */
146 unsigned int dx_size; /* TX or RX descriptor size, */
147 /* stored here for fater math */
148 u16 page_order;
149 u16 page_offset;
150 u16 frame_max;
151 u16 tail_size;
152 union aq_ring_stats_s stats;
153 dma_addr_t dx_ring_pa;
154 struct bpf_prog *xdp_prog;
155 enum atl_ring_type ring_type;
156 struct xdp_rxq_info xdp_rxq;
157 unsigned long ptp_ts_deadline;
158 };
159
160 struct aq_ring_param_s {
161 unsigned int vec_idx;
162 unsigned int cpu;
163 cpumask_t affinity_mask;
164 };
165
aq_buf_vaddr(struct aq_rxpage * rxpage)166 static inline void *aq_buf_vaddr(struct aq_rxpage *rxpage)
167 {
168 return page_to_virt(rxpage->page) + rxpage->pg_off;
169 }
170
aq_buf_daddr(struct aq_rxpage * rxpage)171 static inline dma_addr_t aq_buf_daddr(struct aq_rxpage *rxpage)
172 {
173 return rxpage->daddr + rxpage->pg_off;
174 }
175
aq_ring_next_dx(struct aq_ring_s * self,unsigned int dx)176 static inline unsigned int aq_ring_next_dx(struct aq_ring_s *self,
177 unsigned int dx)
178 {
179 return (++dx >= self->size) ? 0U : dx;
180 }
181
aq_ring_avail_dx(struct aq_ring_s * self)182 static inline unsigned int aq_ring_avail_dx(struct aq_ring_s *self)
183 {
184 return (((self->sw_tail >= self->sw_head)) ?
185 (self->size - 1) - self->sw_tail + self->sw_head :
186 self->sw_head - self->sw_tail - 1);
187 }
188
189 int aq_ring_tx_alloc(struct aq_ring_s *self,
190 struct aq_nic_s *aq_nic,
191 unsigned int idx,
192 struct aq_nic_cfg_s *aq_nic_cfg);
193 int aq_ring_rx_alloc(struct aq_ring_s *self,
194 struct aq_nic_s *aq_nic,
195 unsigned int idx,
196 struct aq_nic_cfg_s *aq_nic_cfg);
197
198 int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type);
199 void aq_ring_rx_deinit(struct aq_ring_s *self);
200 void aq_ring_free(struct aq_ring_s *self);
201 void aq_ring_update_queue_state(struct aq_ring_s *ring);
202 void aq_ring_queue_wake(struct aq_ring_s *ring);
203 void aq_ring_queue_stop(struct aq_ring_s *ring);
204 bool aq_ring_tx_clean(struct aq_ring_s *self);
205 void aq_ring_tx_deinit(struct aq_ring_s *self);
206 int aq_xdp_xmit(struct net_device *dev, int num_frames,
207 struct xdp_frame **frames, u32 flags);
208 int aq_ring_rx_clean(struct aq_ring_s *self,
209 struct napi_struct *napi,
210 int *work_done,
211 int budget);
212 int aq_ring_rx_fill(struct aq_ring_s *self);
213
214 int aq_ring_hwts_rx_alloc(struct aq_ring_s *self,
215 struct aq_nic_s *aq_nic, unsigned int idx,
216 unsigned int size, unsigned int dx_size);
217 void aq_ring_hwts_rx_free(struct aq_ring_s *self);
218 void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic);
219
220 unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data);
221
222 #endif /* AQ_RING_H */
223